From 07e1f2e4401328e78beaee96f9f8c5e605a9c962 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 30 Mar 2023 09:18:16 -0300 Subject: [PATCH 001/279] docs(kafka_producer): update ZH translation of field (rv5.0) Fixes https://emqx.atlassian.net/browse/EMQX-9411 --- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index df32c1cae..076619e91 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -62,7 +62,7 @@ emqx_ee_bridge_kafka { mqtt_topic { desc { en: "MQTT topic or topic as data source (bridge input). Should not configure this if the bridge is used as a rule action." - zh: "指定 MQTT 主题作为桥接的数据源。 若该桥接用于规则的动作，则必须将该配置项删除。" + zh: "桥接指定 MQTT 主题数据，也可以留空由规则动作指定数据源。" } label { en: "Source MQTT Topic" From aabbbf5deb069f5fcf584d5d298f5efbc1b72e24 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 31 Mar 2023 09:07:00 -0300 Subject: [PATCH 002/279] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index 076619e91..566d1889e 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -61,8 +61,8 @@ emqx_ee_bridge_kafka { } mqtt_topic { desc { - en: "MQTT topic or topic as data source (bridge input). Should not configure this if the bridge is used as a rule action." - zh: "桥接指定 MQTT 主题数据，也可以留空由规则动作指定数据源。" + en: "MQTT topic or topic filter as data source (bridge input). If rule action is used as data source, this config should be left empty, otherwise messages will be duplicated in Kafka." + zh: "指定源 MQTT 主题，若由规则动作指定数据源，则该配置应留空，否则消息会被重复发送到 Kafka。" } label { en: "Source MQTT Topic" From 159bcf329cecfcae9ea8764f5a15ad8eb45b8fef Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 29 Mar 2023 20:31:58 +0800 Subject: [PATCH 003/279] refactor: make Stomp and MQTT-SN gateway as an independent apps --- .../i18n/emqx_gateway_schema_i18n.conf | 92 --------- apps/emqx_gateway/src/emqx_gateway_api.erl | 4 +- apps/emqx_gateway/src/emqx_gateway_app.erl | 66 +++--- apps/emqx_gateway/src/emqx_gateway_schema.erl | 155 +++----------- apps/emqx_gateway/src/emqx_gateway_utils.erl | 81 +++++++- apps/emqx_mqttsn/.gitignore | 19 ++ apps/emqx_mqttsn/LICENSE | 191 ++++++++++++++++++ .../src/mqttsn => emqx_mqttsn}/README.md | 0 apps/emqx_mqttsn/i18n/emqx_mqttsn_schema.conf | 64 ++++++ .../include/emqx_mqttsn.hrl} | 0 apps/emqx_mqttsn/rebar.config | 2 + apps/emqx_mqttsn/src/emqx_mqttsn.app.src | 10 + .../src/emqx_mqttsn.erl} | 64 +++--- .../src/emqx_mqttsn_broadcast.erl} | 14 +- .../src/emqx_mqttsn_channel.erl} | 26 +-- .../src/emqx_mqttsn_frame.erl} | 6 +- .../src/emqx_mqttsn_registry.erl} | 30 ++- apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl | 107 ++++++++++ apps/emqx_stomp/.gitignore | 19 ++ apps/emqx_stomp/LICENSE | 191 ++++++++++++++++++ .../src/stomp => emqx_stomp}/README.md | 0 apps/emqx_stomp/i18n/emqx_stomp_schema.conf | 32 +++ .../include/emqx_stomp.hrl | 0 apps/emqx_stomp/rebar.config | 2 + apps/emqx_stomp/src/emqx_stomp.app.src | 10 + .../src/emqx_stomp.erl} | 51 ++--- .../src}/emqx_stomp_channel.erl | 2 +- .../src}/emqx_stomp_frame.erl | 2 +- .../src}/emqx_stomp_heartbeat.erl | 2 +- apps/emqx_stomp/src/emqx_stomp_schema.erl | 80 ++++++++ rebar.config.erl | 2 + 31 files changed, 968 insertions(+), 356 deletions(-) create mode 100644 apps/emqx_mqttsn/.gitignore create mode 100644 apps/emqx_mqttsn/LICENSE rename apps/{emqx_gateway/src/mqttsn => emqx_mqttsn}/README.md (100%) create mode 100644 apps/emqx_mqttsn/i18n/emqx_mqttsn_schema.conf rename apps/{emqx_gateway/src/mqttsn/include/emqx_sn.hrl => emqx_mqttsn/include/emqx_mqttsn.hrl} (100%) create mode 100644 apps/emqx_mqttsn/rebar.config create mode 100644 apps/emqx_mqttsn/src/emqx_mqttsn.app.src rename apps/{emqx_gateway/src/mqttsn/emqx_sn_impl.erl => emqx_mqttsn/src/emqx_mqttsn.erl} (76%) rename apps/{emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl => emqx_mqttsn/src/emqx_mqttsn_broadcast.erl} (89%) rename apps/{emqx_gateway/src/mqttsn/emqx_sn_channel.erl => emqx_mqttsn/src/emqx_mqttsn_channel.erl} (98%) rename apps/{emqx_gateway/src/mqttsn/emqx_sn_frame.erl => emqx_mqttsn/src/emqx_mqttsn_frame.erl} (99%) rename apps/{emqx_gateway/src/mqttsn/emqx_sn_registry.erl => emqx_mqttsn/src/emqx_mqttsn_registry.erl} (91%) create mode 100644 apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl create mode 100644 apps/emqx_stomp/.gitignore create mode 100644 apps/emqx_stomp/LICENSE rename apps/{emqx_gateway/src/stomp => emqx_stomp}/README.md (100%) create mode 100644 apps/emqx_stomp/i18n/emqx_stomp_schema.conf rename apps/{emqx_gateway/src/stomp => emqx_stomp}/include/emqx_stomp.hrl (100%) create mode 100644 apps/emqx_stomp/rebar.config create mode 100644 apps/emqx_stomp/src/emqx_stomp.app.src rename apps/{emqx_gateway/src/stomp/emqx_stomp_impl.erl => emqx_stomp/src/emqx_stomp.erl} (83%) rename apps/{emqx_gateway/src/stomp => emqx_stomp/src}/emqx_stomp_channel.erl (99%) rename apps/{emqx_gateway/src/stomp => emqx_stomp/src}/emqx_stomp_frame.erl (99%) rename apps/{emqx_gateway/src/stomp => emqx_stomp/src}/emqx_stomp_heartbeat.erl (98%) create mode 100644 apps/emqx_stomp/src/emqx_stomp_schema.erl diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf index 74a70eb73..aaa5007ee 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf @@ -1,97 +1,5 @@ emqx_gateway_schema { - stomp { - desc { - en: """The Stomp Gateway configuration. -This gateway supports v1.2/1.1/1.0""" - zh: """Stomp 网关配置。当前实现支持 v1.2/1.1/1.0 协议版本""" - } - } - - stom_frame_max_headers { - desc { - en: """The maximum number of Header""" - zh: """允许的 Header 最大数量""" - } - } - - stomp_frame_max_headers_length { - desc { - en: """The maximum string length of the Header Value""" - zh: """允许的 Header 字符串的最大长度""" - } - } - - stom_frame_max_body_length { - desc { - en: """Maximum number of bytes of Body allowed per Stomp packet""" - zh: """允许的 Stomp 报文 Body 的最大字节数""" - } - } - - mqttsn { - desc { - en: """The MQTT-SN Gateway configuration. -This gateway only supports the v1.2 protocol""" - zh: """MQTT-SN 网关配置。当前实现仅支持 v1.2 版本""" - } - } - - mqttsn_gateway_id { - desc { - en: """MQTT-SN Gateway ID. -When the broadcast option is enabled, the gateway will broadcast ADVERTISE message with this value""" - zh: """MQTT-SN 网关 ID。 -当 broadcast 打开时，MQTT-SN 网关会使用该 ID 来广播 ADVERTISE 消息""" - } - } - - mqttsn_broadcast { - desc { - en: """Whether to periodically broadcast ADVERTISE messages""" - zh: """是否周期性广播 ADVERTISE 消息""" - } - } - - mqttsn_enable_qos3 { - desc { - en: """Allows connectionless clients to publish messages with a Qos of -1. -This feature is defined for very simple client implementations which do not support any other features except this one. There is no connection setup nor tear down, no registration nor subscription. The client just sends its 'PUBLISH' messages to a GW""" - zh: """是否允许无连接的客户端发送 QoS 等于 -1 的消息。 -该功能主要用于支持轻量的 MQTT-SN 客户端实现，它不会向网关建立连接，注册主题，也不会发起订阅；它只使用 QoS 为 -1 来发布消息""" - } - } - - mqttsn_subs_resume { - desc { - en: """Whether to initiate all subscribed topic name registration messages to the client after the Session has been taken over by a new channel""" - zh: """在会话被重用后，网关是否主动向客户端注册对已订阅主题名称""" - } - } - - mqttsn_predefined { - desc { - en: """The pre-defined topic IDs and topic names. -A 'pre-defined' topic ID is a topic ID whose mapping to a topic name is known in advance by both the client's application and the gateway""" - zh: """预定义主题列表。 -预定义的主题列表，是一组 主题 ID 和 主题名称 的映射关系。使用预先定义的主题列表，可以减少 MQTT-SN 客户端和网关对于固定主题的注册请求""" - } - } - - mqttsn_predefined_id { - desc { - en: """Topic ID. Range: 1-65535""" - zh: """主题 ID。范围：1-65535""" - } - } - - mqttsn_predefined_topic { - desc { - en: """Topic Name""" - zh: """主题名称。注：不支持通配符""" - } - } - coap { desc { en: """The CoAP Gateway configuration. diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 62f723d59..bc44daca8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -395,7 +395,7 @@ fields(Gw) when Gw == exproto -> [{name, mk(Gw, #{desc => ?DESC(gateway_name)})}] ++ - convert_listener_struct(emqx_gateway_schema:fields(Gw)); + convert_listener_struct(emqx_gateway_schema:gateway_schema(Gw)); fields(Gw) when Gw == update_stomp; Gw == update_mqttsn; @@ -405,7 +405,7 @@ fields(Gw) when -> "update_" ++ GwStr = atom_to_list(Gw), Gw1 = list_to_existing_atom(GwStr), - remove_listener_and_authn(emqx_gateway_schema:fields(Gw1)); + remove_listener_and_authn(emqx_gateway_schema:gateway_schema(Gw1)); fields(Listener) when Listener == tcp_listener; Listener == ssl_listener; diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index cb5a16fde..a805a0ceb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -41,33 +41,49 @@ stop(_State) -> %% Internal funcs load_default_gateway_applications() -> - Apps = gateway_type_searching(), - lists:foreach(fun reg/1, Apps). + BuiltInGateways = [ + #{ + name => lwm2m, + callback_module => emqx_lwm2m_impl, + config_schema_module => emqx_lwm2m_schema + }, + #{ + name => coap, + callback_module => emqx_coap_impl, + config_schema_module => emqx_gateway_schema + }, + #{ + name => exproto, + callback_module => emqx_exproto_impl, + config_schema_module => emqx_gateway_schema + } + ], + lists:foreach( + fun(Def) -> + load_gateway_application(Def) + end, + emqx_gateway_utils:find_gateway_definations() ++ BuiltInGateways + ). -gateway_type_searching() -> - %% FIXME: Hardcoded apps - [ - emqx_stomp_impl, - emqx_sn_impl, - emqx_exproto_impl, - emqx_coap_impl, - emqx_lwm2m_impl - ]. - -reg(Mod) -> - try - Mod:reg(), - ?SLOG(debug, #{ - msg => "register_gateway_succeed", - callback_module => Mod - }) - catch - Class:Reason:Stk -> +load_gateway_application( + #{ + name := Name, + callback_module := CbMod, + config_schema_module := SchemaMod + } +) -> + RegistryOptions = [{cbkmod, CbMod}, {schema, SchemaMod}], + case emqx_gateway_registry:reg(Name, RegistryOptions) of + ok -> + ?SLOG(debug, #{ + msg => "register_gateway_succeed", + callback_module => CbMod + }); + {error, already_registered} -> ?SLOG(error, #{ - msg => "failed_to_register_gateway", - callback_module => Mod, - reason => {Class, Reason}, - stacktrace => Stk + msg => "gateway_already_registered", + name => Name, + callback_module => CbMod }) end. diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 2034a40eb..f7ce3d05c 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -53,6 +53,8 @@ -export([proxy_protocol_opts/0]). +-export([mountpoint/0, mountpoint/1, gateway_common_options/0, gateway_schema/1]). + namespace() -> gateway. tags() -> @@ -62,22 +64,6 @@ roots() -> [gateway]. fields(gateway) -> [ - {stomp, - sc( - ref(stomp), - #{ - required => {false, recursively}, - desc => ?DESC(stomp) - } - )}, - {mqttsn, - sc( - ref(mqttsn), - #{ - required => {false, recursively}, - desc => ?DESC(mqttsn) - } - )}, {coap, sc( ref(coap), @@ -102,102 +88,7 @@ fields(gateway) -> desc => ?DESC(exproto) } )} - ]; -fields(stomp) -> - [ - {frame, sc(ref(stomp_frame))}, - {mountpoint, mountpoint()}, - {listeners, sc(ref(tcp_listeners), #{desc => ?DESC(tcp_listeners)})} - ] ++ gateway_common_options(); -fields(stomp_frame) -> - [ - {max_headers, - sc( - non_neg_integer(), - #{ - default => 10, - desc => ?DESC(stom_frame_max_headers) - } - )}, - {max_headers_length, - sc( - non_neg_integer(), - #{ - default => 1024, - desc => ?DESC(stomp_frame_max_headers_length) - } - )}, - {max_body_length, - sc( - integer(), - #{ - default => 65536, - desc => ?DESC(stom_frame_max_body_length) - } - )} - ]; -fields(mqttsn) -> - [ - {gateway_id, - sc( - integer(), - #{ - default => 1, - required => true, - desc => ?DESC(mqttsn_gateway_id) - } - )}, - {broadcast, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(mqttsn_broadcast) - } - )}, - %% TODO: rename - {enable_qos3, - sc( - boolean(), - #{ - default => true, - desc => ?DESC(mqttsn_enable_qos3) - } - )}, - {subs_resume, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(mqttsn_subs_resume) - } - )}, - {predefined, - sc( - hoconsc:array(ref(mqttsn_predefined)), - #{ - default => [], - required => {false, recursively}, - desc => ?DESC(mqttsn_predefined) - } - )}, - {mountpoint, mountpoint()}, - {listeners, sc(ref(udp_listeners), #{desc => ?DESC(udp_listeners)})} - ] ++ gateway_common_options(); -fields(mqttsn_predefined) -> - [ - {id, - sc(integer(), #{ - required => true, - desc => ?DESC(mqttsn_predefined_id) - })}, - - {topic, - sc(binary(), #{ - required => true, - desc => ?DESC(mqttsn_predefined_topic) - })} - ]; + ] ++ gateway_schemas(); fields(coap) -> [ {heartbeat, @@ -522,17 +413,6 @@ fields(dtls_opts) -> desc(gateway) -> "EMQX Gateway configuration root."; -desc(stomp) -> - "The STOMP protocol gateway provides EMQX with the ability to access STOMP\n" - "(Simple (or Streaming) Text Orientated Messaging Protocol) protocol."; -desc(stomp_frame) -> - "Size limits for the STOMP frames."; -desc(mqttsn) -> - "The MQTT-SN (MQTT for Sensor Networks) protocol gateway."; -desc(mqttsn_predefined) -> - "The pre-defined topic name corresponding to the pre-defined topic\n" - "ID of N.\n\n" - "Note: the pre-defined topic ID of 0 is reserved."; desc(coap) -> "The CoAP protocol gateway provides EMQX with the access capability of the CoAP protocol.\n" "It allows publishing, subscribing, and receiving messages to EMQX in accordance\n" @@ -713,8 +593,33 @@ proxy_protocol_opts() -> )} ]. -sc(Type) -> - sc(Type, #{}). +%%-------------------------------------------------------------------- +%% dynamic schemas + +%% FIXME: don't hardcode the gateway names +gateway_schema(coap) -> fields(coap); +gateway_schema(lwm2m) -> fields(lwm2m); +gateway_schema(exproto) -> fields(exproto); +gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); +gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn). + +gateway_schemas() -> + lists:map( + fun(#{name := Name, config_schema_module := Mod}) -> + {Name, + sc( + ref(Mod, Name), + #{ + required => {false, recursively}, + desc => ?DESC(Name) + } + )} + end, + emqx_gateway_utils:find_gateway_definations() + ). + +%%-------------------------------------------------------------------- +%% helpers sc(Type, Meta) -> hoconsc:mk(Type, Meta). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index cee5baaa8..94c7490cc 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -46,7 +46,8 @@ global_chain/1, listener_chain/3, make_deprecated_paths/1, - make_compatible_schema/2 + make_compatible_schema/2, + find_gateway_definations/0 ]). -export([stringfy/1]). @@ -562,3 +563,81 @@ make_compatible_schema2(Path, SchemaFun) -> end, Schema ). + +find_gateway_definations() -> + lists:flatten( + lists:map( + fun(App) -> + gateways(find_attrs(App, gateway)) + end, + ignore_lib_apps(application:loaded_applications()) + ) + ). + +gateways([]) -> + []; +gateways([ + {_App, _Mod, + Defination = + #{ + name := Name, + callback_module := CbMod, + config_schema_module := SchemaMod + }} + | More +]) when is_atom(Name), is_atom(CbMod), is_atom(SchemaMod) -> + [Defination | gateways(More)]. + +find_attrs(App, Def) -> + [ + {App, Mod, Attr} + || {ok, Modules} <- [application:get_key(App, modules)], + Mod <- Modules, + {Name, Attrs} <- module_attributes(Mod), + Name =:= Def, + Attr <- Attrs + ]. + +module_attributes(Module) -> + try + Module:module_info(attributes) + catch + error:undef -> [] + end. + +ignore_lib_apps(Apps) -> + LibApps = [ + kernel, + stdlib, + sasl, + appmon, + eldap, + erts, + syntax_tools, + ssl, + crypto, + mnesia, + os_mon, + inets, + goldrush, + gproc, + runtime_tools, + snmp, + otp_mibs, + public_key, + asn1, + ssh, + hipe, + common_test, + observer, + webtool, + xmerl, + tools, + test_server, + compiler, + debugger, + eunit, + et, + wx + ], + [AppName || {AppName, _, _} <- Apps, not lists:member(AppName, LibApps)]. diff --git a/apps/emqx_mqttsn/.gitignore b/apps/emqx_mqttsn/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_mqttsn/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_mqttsn/LICENSE b/apps/emqx_mqttsn/LICENSE new file mode 100644 index 000000000..5a5418f0f --- /dev/null +++ b/apps/emqx_mqttsn/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/src/mqttsn/README.md b/apps/emqx_mqttsn/README.md similarity index 100% rename from apps/emqx_gateway/src/mqttsn/README.md rename to apps/emqx_mqttsn/README.md diff --git a/apps/emqx_mqttsn/i18n/emqx_mqttsn_schema.conf b/apps/emqx_mqttsn/i18n/emqx_mqttsn_schema.conf new file mode 100644 index 000000000..20c160b11 --- /dev/null +++ b/apps/emqx_mqttsn/i18n/emqx_mqttsn_schema.conf @@ -0,0 +1,64 @@ +emqx_mqttsn_schema { + mqttsn { + desc { + en: """The MQTT-SN Gateway configuration. +This gateway only supports the v1.2 protocol""" + zh: """MQTT-SN 网关配置。当前实现仅支持 v1.2 版本""" + } + } + + mqttsn_gateway_id { + desc { + en: """MQTT-SN Gateway ID. +When the broadcast option is enabled, the gateway will broadcast ADVERTISE message with this value""" + zh: """MQTT-SN 网关 ID。 +当 broadcast 打开时，MQTT-SN 网关会使用该 ID 来广播 ADVERTISE 消息""" + } + } + + mqttsn_broadcast { + desc { + en: """Whether to periodically broadcast ADVERTISE messages""" + zh: """是否周期性广播 ADVERTISE 消息""" + } + } + + mqttsn_enable_qos3 { + desc { + en: """Allows connectionless clients to publish messages with a Qos of -1. +This feature is defined for very simple client implementations which do not support any other features except this one. There is no connection setup nor tear down, no registration nor subscription. The client just sends its 'PUBLISH' messages to a GW""" + zh: """是否允许无连接的客户端发送 QoS 等于 -1 的消息。 +该功能主要用于支持轻量的 MQTT-SN 客户端实现，它不会向网关建立连接，注册主题，也不会发起订阅；它只使用 QoS 为 -1 来发布消息""" + } + } + + mqttsn_subs_resume { + desc { + en: """Whether to initiate all subscribed topic name registration messages to the client after the Session has been taken over by a new channel""" + zh: """在会话被重用后，网关是否主动向客户端注册对已订阅主题名称""" + } + } + + mqttsn_predefined { + desc { + en: """The pre-defined topic IDs and topic names. +A 'pre-defined' topic ID is a topic ID whose mapping to a topic name is known in advance by both the client's application and the gateway""" + zh: """预定义主题列表。 +预定义的主题列表，是一组 主题 ID 和 主题名称 的映射关系。使用预先定义的主题列表，可以减少 MQTT-SN 客户端和网关对于固定主题的注册请求""" + } + } + + mqttsn_predefined_id { + desc { + en: """Topic ID. Range: 1-65535""" + zh: """主题 ID。范围：1-65535""" + } + } + + mqttsn_predefined_topic { + desc { + en: """Topic Name""" + zh: """主题名称。注：不支持通配符""" + } + } +} diff --git a/apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl b/apps/emqx_mqttsn/include/emqx_mqttsn.hrl similarity index 100% rename from apps/emqx_gateway/src/mqttsn/include/emqx_sn.hrl rename to apps/emqx_mqttsn/include/emqx_mqttsn.hrl diff --git a/apps/emqx_mqttsn/rebar.config b/apps/emqx_mqttsn/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_mqttsn/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src new file mode 100644 index 000000000..e58cf5147 --- /dev/null +++ b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src @@ -0,0 +1,10 @@ +{application, emqx_mqttsn, + [{description, "MQTT-SN Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env,[]}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_mqttsn/src/emqx_mqttsn.erl similarity index 76% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl rename to apps/emqx_mqttsn/src/emqx_mqttsn.erl index db730aee1..5d6a94df4 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,13 +14,28 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The MQTT-SN Gateway Implement interface --module(emqx_sn_impl). - --behaviour(emqx_gateway_impl). +%% @doc The MQTT-SN Gateway implement interface +-module(emqx_mqttsn). -include_lib("emqx/include/logger.hrl"). +%% define a gateway named stomp +-gateway(#{ + name => mqttsn, + callback_module => ?MODULE, + config_schema_module => emqx_mqttsn_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + -import( emqx_gateway_utils, [ @@ -30,31 +45,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(mqttsn, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(mqttsn). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( @@ -64,8 +56,8 @@ on_gateway_load( }, Ctx ) -> - %% We Also need to start `emqx_sn_broadcast` & - %% `emqx_sn_registry` process + %% We Also need to start `emqx_mqttsn_broadcast` & + %% `emqx_mqttsn_registry` process case maps:get(broadcast, Config, false) of false -> ok; @@ -73,23 +65,23 @@ on_gateway_load( %% FIXME: Port = 1884, SnGwId = maps:get(gateway_id, Config, undefined), - _ = emqx_sn_broadcast:start_link(SnGwId, Port), + _ = emqx_mqttsn_broadcast:start_link(SnGwId, Port), ok end, PredefTopics = maps:get(predefined, Config, []), - {ok, RegistrySvr} = emqx_sn_registry:start_link(GwName, PredefTopics), + {ok, RegistrySvr} = emqx_mqttsn_registry:start_link(GwName, PredefTopics), NConfig = maps:without( [broadcast, predefined], - Config#{registry => emqx_sn_registry:lookup_name(RegistrySvr)} + Config#{registry => emqx_mqttsn_registry:lookup_name(RegistrySvr)} ), Listeners = emqx_gateway_utils:normalize_config(NConfig), ModCfg = #{ - frame_mod => emqx_sn_frame, - chann_mod => emqx_sn_channel + frame_mod => emqx_mqttsn_frame, + chann_mod => emqx_mqttsn_channel }, case diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_broadcast.erl similarity index 89% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl rename to apps/emqx_mqttsn/src/emqx_mqttsn_broadcast.erl index 5fc08ad7f..be0122e0e 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_broadcast.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_broadcast.erl @@ -14,17 +14,11 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_sn_broadcast). +-module(emqx_mqttsn_broadcast). -behaviour(gen_server). --ifdef(TEST). -%% make rebar3 ct happy when testing with --suite path/to/module_SUITE.erl --include_lib("emqx_gateway/src/mqttsn/include/emqx_sn.hrl"). --else. -%% make mix happy --include("src/mqttsn/include/emqx_sn.hrl"). --endif. +-include("emqx_mqttsn.hrl"). -include_lib("emqx/include/logger.hrl"). -export([ @@ -65,7 +59,7 @@ stop() -> init([GwId, Port]) -> %% FIXME: - Duration = application:get_env(emqx_sn, advertise_duration, ?DEFAULT_DURATION), + Duration = application:get_env(emqx_mqttsn, advertise_duration, ?DEFAULT_DURATION), {ok, Sock} = gen_udp:open(0, [binary, {broadcast, true}]), {ok, ensure_advertise(#state{ @@ -121,7 +115,7 @@ send_advertise(#state{ addrs = Addrs, duration = Duration }) -> - Data = emqx_sn_frame:serialize_pkt(?SN_ADVERTISE_MSG(GwId, Duration), #{}), + Data = emqx_mqttsn_frame:serialize_pkt(?SN_ADVERTISE_MSG(GwId, Duration), #{}), lists:foreach( fun(Addr) -> ?SLOG(debug, #{ diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_channel.erl similarity index 98% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl rename to apps/emqx_mqttsn/src/emqx_mqttsn_channel.erl index 23d07113c..c27c0ba3f 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_channel.erl @@ -14,11 +14,11 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_sn_channel). +-module(emqx_mqttsn_channel). -behaviour(emqx_gateway_channel). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -51,7 +51,7 @@ %% Context ctx :: emqx_gateway_ctx:context(), %% Registry - registry :: emqx_sn_registry:registry(), + registry :: emqx_mqttsn_registry:registry(), %% Gateway Id gateway_id :: integer(), %% Enable QoS3 @@ -478,7 +478,7 @@ handle_in( true -> <>; false -> - emqx_sn_registry:lookup_topic( + emqx_mqttsn_registry:lookup_topic( Registry, ?NEG_QOS_CLIENT_ID, TopicId @@ -624,7 +624,7 @@ handle_in( clientinfo = #{clientid := ClientId} } ) -> - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:register_topic(Registry, ClientId, TopicName) of TopicId when is_integer(TopicId) -> ?SLOG(debug, #{ msg => "registered_topic_name", @@ -778,7 +778,7 @@ handle_in( {ok, Channel} end; ?SN_RC_INVALID_TOPIC_ID -> - case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + case emqx_mqttsn_registry:lookup_topic(Registry, ClientId, TopicId) of undefined -> {ok, Channel}; TopicName -> @@ -1093,7 +1093,7 @@ convert_topic_id_to_name( clientinfo = #{clientid := ClientId} } ) -> - case emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId) of + case emqx_mqttsn_registry:lookup_topic(Registry, ClientId, TopicId) of undefined -> {error, ?SN_RC_INVALID_TOPIC_ID}; TopicName -> @@ -1202,7 +1202,7 @@ preproc_subs_type( %% If the gateway is able accept the subscription, %% it assigns a topic id to the received topic name %% and returns it within a SUBACK message - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:register_topic(Registry, ClientId, TopicName) of {error, too_large} -> {error, ?SN_RC2_EXCEED_LIMITATION}; {error, wildcard_topic} -> @@ -1228,7 +1228,7 @@ preproc_subs_type( } ) -> case - emqx_sn_registry:lookup_topic( + emqx_mqttsn_registry:lookup_topic( Registry, ClientId, TopicId @@ -1344,7 +1344,7 @@ preproc_unsub_type( } ) -> case - emqx_sn_registry:lookup_topic( + emqx_mqttsn_registry:lookup_topic( Registry, ClientId, TopicId @@ -1765,7 +1765,7 @@ message_to_packet( ?QOS_0 -> 0; _ -> MsgId end, - case emqx_sn_registry:lookup_topic_id(Registry, ClientId, Topic) of + case emqx_mqttsn_registry:lookup_topic_id(Registry, ClientId, Topic) of {predef, PredefTopicId} -> Flags = #mqtt_sn_flags{qos = QoS, topic_id_type = ?SN_PREDEFINED_TOPIC}, ?SN_PUBLISH_MSG(Flags, PredefTopicId, NMsgId, Payload); @@ -1932,9 +1932,9 @@ ensure_registered_topic_name( Channel = #channel{registry = Registry} ) -> ClientId = clientid(Channel), - case emqx_sn_registry:lookup_topic_id(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:lookup_topic_id(Registry, ClientId, TopicName) of undefined -> - case emqx_sn_registry:register_topic(Registry, ClientId, TopicName) of + case emqx_mqttsn_registry:register_topic(Registry, ClientId, TopicName) of {error, Reason} -> {error, Reason}; TopicId -> {ok, TopicId} end; diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl similarity index 99% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl rename to apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl index 39bd9e889..bf0fc52a4 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_frame.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl @@ -16,11 +16,11 @@ %%-------------------------------------------------------------------- %% @doc The frame parser for MQTT-SN protocol --module(emqx_sn_frame). +-module(emqx_mqttsn_frame). -behaviour(emqx_gateway_frame). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -export([ initial_parse_state/1, @@ -438,7 +438,7 @@ format(?SN_DISCONNECT_MSG(Duration)) -> format(#mqtt_sn_message{type = Type, variable = Var}) -> io_lib:format( "mqtt_sn_message(type=~s, Var=~w)", - [emqx_sn_frame:message_type(Type), Var] + [emqx_mqttsn_frame:message_type(Type), Var] ). is_message(#mqtt_sn_message{type = Type}) when diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl similarity index 91% rename from apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl rename to apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl index 689aab8ce..07da8c351 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_registry.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl @@ -15,13 +15,11 @@ %%-------------------------------------------------------------------- %% @doc The MQTT-SN Topic Registry -%% -%% XXX: --module(emqx_sn_registry). +-module(emqx_mqttsn_registry). -behaviour(gen_server). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("emqx/include/logger.hrl"). -export([start_link/2]). @@ -53,11 +51,11 @@ -export([lookup_name/1]). --define(SN_SHARD, emqx_sn_shard). +-define(SN_SHARD, emqx_mqttsn_shard). -record(state, {tabname, max_predef_topic_id = 0}). --record(emqx_sn_registry, {key, value}). +-record(emqx_mqttsn_registry, {key, value}). -type registry() :: {Tab :: atom(), RegistryPid :: pid()}. @@ -126,7 +124,7 @@ lookup_name(Pid) -> %%----------------------------------------------------------------------------- name(InstaId) -> - list_to_atom(lists:concat([emqx_sn_, InstaId, '_registry'])). + list_to_atom(lists:concat([emqx_mqttsn_, InstaId, '_registry'])). init([InstaId, PredefTopics]) -> %% {predef, TopicId} -> TopicName @@ -136,8 +134,8 @@ init([InstaId, PredefTopics]) -> Tab = name(InstaId), ok = mria:create_table(Tab, [ {storage, ram_copies}, - {record_name, emqx_sn_registry}, - {attributes, record_info(fields, emqx_sn_registry)}, + {record_name, emqx_mqttsn_registry}, + {attributes, record_info(fields, emqx_mqttsn_registry)}, {storage_properties, [{ets, [{read_concurrency, true}]}]}, {rlog_shard, ?SN_SHARD} ]), @@ -145,11 +143,11 @@ init([InstaId, PredefTopics]) -> MaxPredefId = lists:foldl( fun(#{id := TopicId, topic := TopicName0}, AccId) -> TopicName = iolist_to_binary(TopicName0), - mria:dirty_write(Tab, #emqx_sn_registry{ + mria:dirty_write(Tab, #emqx_mqttsn_registry{ key = {predef, TopicId}, value = TopicName }), - mria:dirty_write(Tab, #emqx_sn_registry{ + mria:dirty_write(Tab, #emqx_mqttsn_registry{ key = {predef, TopicName}, value = TopicId }), @@ -193,7 +191,7 @@ handle_call( handle_call({unregister, ClientId}, _From, State = #state{tabname = Tab}) -> Registry = mnesia:dirty_match_object( Tab, - {emqx_sn_registry, {ClientId, '_'}, '_'} + {emqx_mqttsn_registry, {ClientId, '_'}, '_'} ), lists:foreach( fun(R) -> @@ -234,7 +232,7 @@ code_change(_OldVsn, State, _Extra) -> do_register(Tab, ClientId, TopicId, TopicName) -> mnesia:write( Tab, - #emqx_sn_registry{ + #emqx_mqttsn_registry{ key = {ClientId, next_topic_id}, value = TopicId + 1 }, @@ -242,7 +240,7 @@ do_register(Tab, ClientId, TopicId, TopicName) -> ), mnesia:write( Tab, - #emqx_sn_registry{ + #emqx_mqttsn_registry{ key = {ClientId, TopicName}, value = TopicId }, @@ -250,7 +248,7 @@ do_register(Tab, ClientId, TopicId, TopicName) -> ), mnesia:write( Tab, - #emqx_sn_registry{ + #emqx_mqttsn_registry{ key = {ClientId, TopicId}, value = TopicName }, @@ -261,6 +259,6 @@ do_register(Tab, ClientId, TopicId, TopicName) -> next_topic_id(Tab, PredefId, ClientId) -> case mnesia:dirty_read(Tab, {ClientId, next_topic_id}) of - [#emqx_sn_registry{value = Id}] -> Id; + [#emqx_mqttsn_registry{value = Id}] -> Id; [] -> PredefId + 1 end. diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl new file mode 100644 index 000000000..cb33cbe95 --- /dev/null +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl @@ -0,0 +1,107 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_mqttsn_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(mqttsn) -> + [ + {gateway_id, + sc( + integer(), + #{ + default => 1, + required => true, + desc => ?DESC(mqttsn_gateway_id) + } + )}, + {broadcast, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(mqttsn_broadcast) + } + )}, + %% TODO: rename + {enable_qos3, + sc( + boolean(), + #{ + default => true, + desc => ?DESC(mqttsn_enable_qos3) + } + )}, + {subs_resume, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(mqttsn_subs_resume) + } + )}, + {predefined, + sc( + hoconsc:array(ref(mqttsn_predefined)), + #{ + default => [], + required => {false, recursively}, + desc => ?DESC(mqttsn_predefined) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, sc(ref(emqx_gateway_schema, udp_listeners), #{desc => ?DESC(udp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(mqttsn_predefined) -> + [ + {id, + sc(integer(), #{ + required => true, + desc => ?DESC(mqttsn_predefined_id) + })}, + + {topic, + sc(binary(), #{ + required => true, + desc => ?DESC(mqttsn_predefined_topic) + })} + ]. + +desc(mqttsn) -> + "The MQTT-SN (MQTT for Sensor Networks) protocol gateway."; +desc(mqttsn_predefined) -> + "The pre-defined topic name corresponding to the pre-defined topic\n" + "ID of N.\n\n" + "Note: the pre-defined topic ID of 0 is reserved."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% internal functions + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_stomp/.gitignore b/apps/emqx_stomp/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_stomp/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_stomp/LICENSE b/apps/emqx_stomp/LICENSE new file mode 100644 index 000000000..5a5418f0f --- /dev/null +++ b/apps/emqx_stomp/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/src/stomp/README.md b/apps/emqx_stomp/README.md similarity index 100% rename from apps/emqx_gateway/src/stomp/README.md rename to apps/emqx_stomp/README.md diff --git a/apps/emqx_stomp/i18n/emqx_stomp_schema.conf b/apps/emqx_stomp/i18n/emqx_stomp_schema.conf new file mode 100644 index 000000000..3d166abb5 --- /dev/null +++ b/apps/emqx_stomp/i18n/emqx_stomp_schema.conf @@ -0,0 +1,32 @@ +emqx_stomp_schema { + stomp { + desc { + en: """The Stomp Gateway configuration. +This gateway supports v1.2/1.1/1.0""" + zh: """Stomp 网关配置。当前实现支持 v1.2/1.1/1.0 协议版本""" + } + } + + stom_frame_max_headers { + desc { + en: """The maximum number of Header""" + zh: """允许的 Header 最大数量""" + } + } + + stomp_frame_max_headers_length { + desc { + en: """The maximum string length of the Header Value""" + zh: """允许的 Header 字符串的最大长度""" + } + } + + stom_frame_max_body_length { + desc { + en: """Maximum number of bytes of Body allowed per Stomp packet""" + zh: """允许的 Stomp 报文 Body 的最大字节数""" + } + } + + +} diff --git a/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl b/apps/emqx_stomp/include/emqx_stomp.hrl similarity index 100% rename from apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl rename to apps/emqx_stomp/include/emqx_stomp.hrl diff --git a/apps/emqx_stomp/rebar.config b/apps/emqx_stomp/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_stomp/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src new file mode 100644 index 000000000..dffb9b7a4 --- /dev/null +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -0,0 +1,10 @@ +{application, emqx_stomp, [ + {description, "Stomp gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_stomp/src/emqx_stomp.erl similarity index 83% rename from apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl rename to apps/emqx_stomp/src/emqx_stomp.erl index c2907c262..6c14e222c 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl +++ b/apps/emqx_stomp/src/emqx_stomp.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -14,13 +14,29 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_stomp_impl). - --behaviour(emqx_gateway_impl). +%% @doc The Stomp Gateway implement interface +-module(emqx_stomp). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). +%% define a gateway named stomp +-gateway(#{ + name => stomp, + callback_module => ?MODULE, + config_schema_module => emqx_stomp_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + -import( emqx_gateway_utils, [ @@ -30,33 +46,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --spec reg() -> ok | {error, any()}. -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(stomp, RegistryOptions). - --spec unreg() -> ok | {error, any()}. -unreg() -> - emqx_gateway_registry:unreg(stomp). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_stomp/src/emqx_stomp_channel.erl similarity index 99% rename from apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl rename to apps/emqx_stomp/src/emqx_stomp_channel.erl index b95bb827c..13b70348a 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl +++ b/apps/emqx_stomp/src/emqx_stomp_channel.erl @@ -18,7 +18,7 @@ -behaviour(emqx_gateway_channel). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl b/apps/emqx_stomp/src/emqx_stomp_frame.erl similarity index 99% rename from apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl rename to apps/emqx_stomp/src/emqx_stomp_frame.erl index 47e045412..4913d6b2a 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl +++ b/apps/emqx_stomp/src/emqx_stomp_frame.erl @@ -70,7 +70,7 @@ -behaviour(emqx_gateway_frame). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -export([ initial_parse_state/1, diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl b/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl similarity index 98% rename from apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl rename to apps/emqx_stomp/src/emqx_stomp_heartbeat.erl index 88720c513..f5ed99623 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl +++ b/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl @@ -17,7 +17,7 @@ %% @doc Stomp heartbeat. -module(emqx_stomp_heartbeat). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -export([ init/1, diff --git a/apps/emqx_stomp/src/emqx_stomp_schema.erl b/apps/emqx_stomp/src/emqx_stomp_schema.erl new file mode 100644 index 000000000..b1c6a92e2 --- /dev/null +++ b/apps/emqx_stomp/src/emqx_stomp_schema.erl @@ -0,0 +1,80 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_stomp_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(stomp) -> + [ + {frame, sc(ref(stomp_frame))}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, sc(ref(emqx_gateway_schema, tcp_listeners), #{desc => ?DESC(tcp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(stomp_frame) -> + [ + {max_headers, + sc( + non_neg_integer(), + #{ + default => 10, + desc => ?DESC(stom_frame_max_headers) + } + )}, + {max_headers_length, + sc( + non_neg_integer(), + #{ + default => 1024, + desc => ?DESC(stomp_frame_max_headers_length) + } + )}, + {max_body_length, + sc( + integer(), + #{ + default => 65536, + desc => ?DESC(stom_frame_max_body_length) + } + )} + ]. + +desc(stomp) -> + "The STOMP protocol gateway provides EMQX with the ability to access STOMP\n" + "(Simple (or Streaming) Text Orientated Messaging Protocol) protocol."; +desc(stomp_frame) -> + "Size limits for the STOMP frames."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% internal functions + +sc(Type) -> + sc(Type, #{}). + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/rebar.config.erl b/rebar.config.erl index 98cd30570..47bf925d4 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -389,6 +389,8 @@ relx_apps(ReleaseType, Edition) -> emqx_authz, emqx_auto_subscribe, emqx_gateway, + emqx_stomp, + emqx_mqttsn, emqx_exhook, emqx_bridge, emqx_rule_engine, From 786f03095820a606198ac30cb4c60268e6d20c71 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 11:50:41 +0800 Subject: [PATCH 004/279] test: move tests into splited gateway dirs --- apps/{emqx_gateway => emqx_mqttsn}/test/broadcast_test.py | 0 .../test/emqx_sn_frame_SUITE.erl | 6 +++--- .../test/emqx_sn_protocol_SUITE.erl | 5 +++-- .../test/emqx_sn_registry_SUITE.erl | 4 ++-- .../test/intergration_test/Makefile | 0 .../test/intergration_test/README.md | 0 .../test/intergration_test/add_emqx_sn_to_project.py | 0 .../test/intergration_test/client/case1_qos0pub.c | 0 .../test/intergration_test/client/case1_qos0sub.c | 0 .../test/intergration_test/client/case2_qos0pub.c | 0 .../test/intergration_test/client/case2_qos0sub.c | 0 .../test/intergration_test/client/case3_qos0pub.c | 0 .../test/intergration_test/client/case3_qos0sub.c | 0 .../test/intergration_test/client/case4_qos3pub.c | 0 .../test/intergration_test/client/case4_qos3sub.c | 0 .../test/intergration_test/client/case5_qos3pub.c | 0 .../test/intergration_test/client/case5_qos3sub.c | 0 .../test/intergration_test/client/case6_sleep.c | 0 .../test/intergration_test/client/case7_double_connect.c | 0 .../test/intergration_test/client/int_test_result.c | 0 .../test/intergration_test/client/int_test_result.h | 0 .../test/intergration_test/disable_qos3.py | 0 .../test/intergration_test/enable_qos3.py | 0 .../test/props/emqx_sn_proper_types.erl | 2 +- .../test/props/prop_emqx_sn_frame.erl | 6 +++--- apps/{emqx_gateway => emqx_stomp}/test/emqx_stomp_SUITE.erl | 3 ++- .../test/emqx_stomp_heartbeat_SUITE.erl | 0 27 files changed, 14 insertions(+), 12 deletions(-) rename apps/{emqx_gateway => emqx_mqttsn}/test/broadcast_test.py (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/emqx_sn_frame_SUITE.erl (97%) rename apps/{emqx_gateway => emqx_mqttsn}/test/emqx_sn_protocol_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_mqttsn}/test/emqx_sn_registry_SUITE.erl (98%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/Makefile (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/README.md (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/add_emqx_sn_to_project.py (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case1_qos0pub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case1_qos0sub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case2_qos0pub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case2_qos0sub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case3_qos0pub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case3_qos0sub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case4_qos3pub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case4_qos3sub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case5_qos3pub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case5_qos3sub.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case6_sleep.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/case7_double_connect.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/int_test_result.c (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/client/int_test_result.h (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/disable_qos3.py (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/intergration_test/enable_qos3.py (100%) rename apps/{emqx_gateway => emqx_mqttsn}/test/props/emqx_sn_proper_types.erl (99%) rename apps/{emqx_gateway => emqx_mqttsn}/test/props/prop_emqx_sn_frame.erl (94%) rename apps/{emqx_gateway => emqx_stomp}/test/emqx_stomp_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_stomp}/test/emqx_stomp_heartbeat_SUITE.erl (100%) diff --git a/apps/emqx_gateway/test/broadcast_test.py b/apps/emqx_mqttsn/test/broadcast_test.py similarity index 100% rename from apps/emqx_gateway/test/broadcast_test.py rename to apps/emqx_mqttsn/test/broadcast_test.py diff --git a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl b/apps/emqx_mqttsn/test/emqx_sn_frame_SUITE.erl similarity index 97% rename from apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl rename to apps/emqx_mqttsn/test/emqx_sn_frame_SUITE.erl index aa3fed707..86cc0cf7e 100644 --- a/apps/emqx_gateway/test/emqx_sn_frame_SUITE.erl +++ b/apps/emqx_mqttsn/test/emqx_sn_frame_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- @@ -30,11 +30,11 @@ all() -> emqx_common_test_helpers:all(?MODULE). parse(D) -> - {ok, P, _Rest, _State} = emqx_sn_frame:parse(D, #{}), + {ok, P, _Rest, _State} = emqx_mqttsn_frame:parse(D, #{}), P. serialize_pkt(P) -> - emqx_sn_frame:serialize_pkt(P, #{}). + emqx_mqttsn_frame:serialize_pkt(P, #{}). %%-------------------------------------------------------------------- %% Test cases diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_mqttsn/test/emqx_sn_protocol_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl rename to apps/emqx_mqttsn/test/emqx_sn_protocol_SUITE.erl index adc1e7382..0e04ec67a 100644 --- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_mqttsn/test/emqx_sn_protocol_SUITE.erl @@ -27,7 +27,7 @@ ] ). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -97,6 +97,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + application:load(emqx_mqttsn), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), Config. @@ -270,7 +271,7 @@ t_subscribe_case03(_) -> %% In this case We use predefined topic name to register and subscribe, %% and expect to receive the corresponding predefined topic id but not a new %% generated topic id from broker. We design this case to illustrate -%% emqx_sn_gateway's compatibility of dealing with predefined and normal +%% MQTT-SN Gateway's compatibility of dealing with predefined and normal %% topics. %% %% Once we give more restrictions to different topic id type, this case diff --git a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl b/apps/emqx_mqttsn/test/emqx_sn_registry_SUITE.erl similarity index 98% rename from apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl rename to apps/emqx_mqttsn/test/emqx_sn_registry_SUITE.erl index 739255e71..4d89a802d 100644 --- a/apps/emqx_gateway/test/emqx_sn_registry_SUITE.erl +++ b/apps/emqx_mqttsn/test/emqx_sn_registry_SUITE.erl @@ -21,7 +21,7 @@ -include_lib("eunit/include/eunit.hrl"). --define(REGISTRY, emqx_sn_registry). +-define(REGISTRY, emqx_mqttsn_registry). -define(MAX_PREDEF_ID, 2). -define(PREDEF_TOPICS, [ #{id => 1, topic => <<"/predefined/topic/name/hello">>}, @@ -66,7 +66,7 @@ t_register(Config) -> ?assertEqual(<<"Topic2">>, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID + 2)), ?assertEqual(?MAX_PREDEF_ID + 1, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic1">>)), ?assertEqual(?MAX_PREDEF_ID + 2, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic2">>)), - emqx_sn_registry:unregister_topic(Reg, <<"ClientId">>), + emqx_mqttsn_registry:unregister_topic(Reg, <<"ClientId">>), ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID + 1)), ?assertEqual(undefined, ?REGISTRY:lookup_topic(Reg, <<"ClientId">>, ?MAX_PREDEF_ID + 2)), ?assertEqual(undefined, ?REGISTRY:lookup_topic_id(Reg, <<"ClientId">>, <<"Topic1">>)), diff --git a/apps/emqx_gateway/test/intergration_test/Makefile b/apps/emqx_mqttsn/test/intergration_test/Makefile similarity index 100% rename from apps/emqx_gateway/test/intergration_test/Makefile rename to apps/emqx_mqttsn/test/intergration_test/Makefile diff --git a/apps/emqx_gateway/test/intergration_test/README.md b/apps/emqx_mqttsn/test/intergration_test/README.md similarity index 100% rename from apps/emqx_gateway/test/intergration_test/README.md rename to apps/emqx_mqttsn/test/intergration_test/README.md diff --git a/apps/emqx_gateway/test/intergration_test/add_emqx_sn_to_project.py b/apps/emqx_mqttsn/test/intergration_test/add_emqx_sn_to_project.py similarity index 100% rename from apps/emqx_gateway/test/intergration_test/add_emqx_sn_to_project.py rename to apps/emqx_mqttsn/test/intergration_test/add_emqx_sn_to_project.py diff --git a/apps/emqx_gateway/test/intergration_test/client/case1_qos0pub.c b/apps/emqx_mqttsn/test/intergration_test/client/case1_qos0pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case1_qos0pub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case1_qos0pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case1_qos0sub.c b/apps/emqx_mqttsn/test/intergration_test/client/case1_qos0sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case1_qos0sub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case1_qos0sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case2_qos0pub.c b/apps/emqx_mqttsn/test/intergration_test/client/case2_qos0pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case2_qos0pub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case2_qos0pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case2_qos0sub.c b/apps/emqx_mqttsn/test/intergration_test/client/case2_qos0sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case2_qos0sub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case2_qos0sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case3_qos0pub.c b/apps/emqx_mqttsn/test/intergration_test/client/case3_qos0pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case3_qos0pub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case3_qos0pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case3_qos0sub.c b/apps/emqx_mqttsn/test/intergration_test/client/case3_qos0sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case3_qos0sub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case3_qos0sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case4_qos3pub.c b/apps/emqx_mqttsn/test/intergration_test/client/case4_qos3pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case4_qos3pub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case4_qos3pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case4_qos3sub.c b/apps/emqx_mqttsn/test/intergration_test/client/case4_qos3sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case4_qos3sub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case4_qos3sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case5_qos3pub.c b/apps/emqx_mqttsn/test/intergration_test/client/case5_qos3pub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case5_qos3pub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case5_qos3pub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case5_qos3sub.c b/apps/emqx_mqttsn/test/intergration_test/client/case5_qos3sub.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case5_qos3sub.c rename to apps/emqx_mqttsn/test/intergration_test/client/case5_qos3sub.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case6_sleep.c b/apps/emqx_mqttsn/test/intergration_test/client/case6_sleep.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case6_sleep.c rename to apps/emqx_mqttsn/test/intergration_test/client/case6_sleep.c diff --git a/apps/emqx_gateway/test/intergration_test/client/case7_double_connect.c b/apps/emqx_mqttsn/test/intergration_test/client/case7_double_connect.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/case7_double_connect.c rename to apps/emqx_mqttsn/test/intergration_test/client/case7_double_connect.c diff --git a/apps/emqx_gateway/test/intergration_test/client/int_test_result.c b/apps/emqx_mqttsn/test/intergration_test/client/int_test_result.c similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/int_test_result.c rename to apps/emqx_mqttsn/test/intergration_test/client/int_test_result.c diff --git a/apps/emqx_gateway/test/intergration_test/client/int_test_result.h b/apps/emqx_mqttsn/test/intergration_test/client/int_test_result.h similarity index 100% rename from apps/emqx_gateway/test/intergration_test/client/int_test_result.h rename to apps/emqx_mqttsn/test/intergration_test/client/int_test_result.h diff --git a/apps/emqx_gateway/test/intergration_test/disable_qos3.py b/apps/emqx_mqttsn/test/intergration_test/disable_qos3.py similarity index 100% rename from apps/emqx_gateway/test/intergration_test/disable_qos3.py rename to apps/emqx_mqttsn/test/intergration_test/disable_qos3.py diff --git a/apps/emqx_gateway/test/intergration_test/enable_qos3.py b/apps/emqx_mqttsn/test/intergration_test/enable_qos3.py similarity index 100% rename from apps/emqx_gateway/test/intergration_test/enable_qos3.py rename to apps/emqx_mqttsn/test/intergration_test/enable_qos3.py diff --git a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl b/apps/emqx_mqttsn/test/props/emqx_sn_proper_types.erl similarity index 99% rename from apps/emqx_gateway/test/props/emqx_sn_proper_types.erl rename to apps/emqx_mqttsn/test/props/emqx_sn_proper_types.erl index 2869a8958..70b13ef8f 100644 --- a/apps/emqx_gateway/test/props/emqx_sn_proper_types.erl +++ b/apps/emqx_mqttsn/test/props/emqx_sn_proper_types.erl @@ -16,7 +16,7 @@ -module(emqx_sn_proper_types). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). diff --git a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl b/apps/emqx_mqttsn/test/props/prop_emqx_sn_frame.erl similarity index 94% rename from apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl rename to apps/emqx_mqttsn/test/props/prop_emqx_sn_frame.erl index f2dfbb8e9..0abe2485c 100644 --- a/apps/emqx_gateway/test/props/prop_emqx_sn_frame.erl +++ b/apps/emqx_mqttsn/test/props/prop_emqx_sn_frame.erl @@ -16,7 +16,7 @@ -module(prop_emqx_sn_frame). --include("src/mqttsn/include/emqx_sn.hrl"). +-include("emqx_mqttsn.hrl"). -include_lib("proper/include/proper.hrl"). -compile({no_auto_import, [register/1]}). @@ -32,11 +32,11 @@ ). parse(D) -> - {ok, P, _Rest, _State} = emqx_sn_frame:parse(D, #{}), + {ok, P, _Rest, _State} = emqx_mqttsn_frame:parse(D, #{}), P. serialize(P) -> - emqx_sn_frame:serialize_pkt(P, #{}). + emqx_mqttsn_frame:serialize_pkt(P, #{}). %%-------------------------------------------------------------------- %% Properties diff --git a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl b/apps/emqx_stomp/test/emqx_stomp_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_stomp_SUITE.erl rename to apps/emqx_stomp/test/emqx_stomp_SUITE.erl index 2cf245ce2..fed7f5163 100644 --- a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_stomp/test/emqx_stomp_SUITE.erl @@ -17,7 +17,7 @@ -module(emqx_stomp_SUITE). -include_lib("eunit/include/eunit.hrl"). --include("src/stomp/include/emqx_stomp.hrl"). +-include("emqx_stomp.hrl"). -compile(export_all). -compile(nowarn_export_all). @@ -53,6 +53,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Cfg) -> + application:load(emqx_stomp), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), Cfg. diff --git a/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl b/apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl similarity index 100% rename from apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl rename to apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl From 0b6c5c4c91b49b8bb7af5e2e8809219b8527114f Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 15:21:42 +0800 Subject: [PATCH 005/279] refactor: split out emqx_coap application --- apps/emqx_coap/.gitignore | 19 ++ apps/emqx_coap/LICENSE | 191 ++++++++++++++++++ .../src/coap => emqx_coap}/README.md | 0 .../src/coap => emqx_coap}/doc/flow.png | Bin .../coap => emqx_coap}/doc/shared_state.png | Bin .../src/coap => emqx_coap}/doc/transport.png | Bin .../i18n/emqx_coap_api_i18n.conf | 0 apps/emqx_coap/i18n/emqx_coap_schema.conf | 80 ++++++++ .../coap => emqx_coap}/include/emqx_coap.hrl | 0 apps/emqx_coap/rebar.config | 2 + apps/emqx_coap/src/emqx_coap.app.src | 10 + .../src/emqx_coap.erl} | 47 ++--- .../coap => emqx_coap/src}/emqx_coap_api.erl | 2 +- .../src}/emqx_coap_channel.erl | 2 +- .../src}/emqx_coap_frame.erl | 2 +- .../src}/emqx_coap_medium.erl | 2 +- .../src}/emqx_coap_message.erl | 2 +- .../src}/emqx_coap_mqtt_handler.erl | 2 +- .../src}/emqx_coap_observe_res.erl | 0 .../src}/emqx_coap_pubsub_handler.erl | 2 +- apps/emqx_coap/src/emqx_coap_schema.erl | 95 +++++++++ .../src}/emqx_coap_session.erl | 2 +- .../coap => emqx_coap/src}/emqx_coap_tm.erl | 2 +- .../src}/emqx_coap_transport.erl | 2 +- .../test/emqx_coap_SUITE.erl | 1 + .../test/emqx_coap_api_SUITE.erl | 3 +- .../i18n/emqx_gateway_schema_i18n.conf | 77 ------- apps/emqx_gateway/src/emqx_gateway_app.erl | 5 - apps/emqx_gateway/src/emqx_gateway_schema.erl | 65 +----- apps/emqx_mqttsn/src/emqx_mqttsn.app.src | 20 +- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.erl | 2 +- mix.exs | 3 + rebar.config.erl | 1 + 34 files changed, 448 insertions(+), 195 deletions(-) create mode 100644 apps/emqx_coap/.gitignore create mode 100644 apps/emqx_coap/LICENSE rename apps/{emqx_gateway/src/coap => emqx_coap}/README.md (100%) rename apps/{emqx_gateway/src/coap => emqx_coap}/doc/flow.png (100%) rename apps/{emqx_gateway/src/coap => emqx_coap}/doc/shared_state.png (100%) rename apps/{emqx_gateway/src/coap => emqx_coap}/doc/transport.png (100%) rename apps/{emqx_gateway => emqx_coap}/i18n/emqx_coap_api_i18n.conf (100%) create mode 100644 apps/emqx_coap/i18n/emqx_coap_schema.conf rename apps/{emqx_gateway/src/coap => emqx_coap}/include/emqx_coap.hrl (100%) create mode 100644 apps/emqx_coap/rebar.config create mode 100644 apps/emqx_coap/src/emqx_coap.app.src rename apps/{emqx_gateway/src/coap/emqx_coap_impl.erl => emqx_coap/src/emqx_coap.erl} (86%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_api.erl (99%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_channel.erl (99%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_frame.erl (99%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_medium.erl (98%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_message.erl (99%) rename apps/{emqx_gateway/src/coap/handler => emqx_coap/src}/emqx_coap_mqtt_handler.erl (96%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_observe_res.erl (100%) rename apps/{emqx_gateway/src/coap/handler => emqx_coap/src}/emqx_coap_pubsub_handler.erl (99%) create mode 100644 apps/emqx_coap/src/emqx_coap_schema.erl rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_session.erl (99%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_tm.erl (99%) rename apps/{emqx_gateway/src/coap => emqx_coap/src}/emqx_coap_transport.erl (99%) rename apps/{emqx_gateway => emqx_coap}/test/emqx_coap_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_coap}/test/emqx_coap_api_SUITE.erl (99%) diff --git a/apps/emqx_coap/.gitignore b/apps/emqx_coap/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_coap/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_coap/LICENSE b/apps/emqx_coap/LICENSE new file mode 100644 index 000000000..5a5418f0f --- /dev/null +++ b/apps/emqx_coap/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/src/coap/README.md b/apps/emqx_coap/README.md similarity index 100% rename from apps/emqx_gateway/src/coap/README.md rename to apps/emqx_coap/README.md diff --git a/apps/emqx_gateway/src/coap/doc/flow.png b/apps/emqx_coap/doc/flow.png similarity index 100% rename from apps/emqx_gateway/src/coap/doc/flow.png rename to apps/emqx_coap/doc/flow.png diff --git a/apps/emqx_gateway/src/coap/doc/shared_state.png b/apps/emqx_coap/doc/shared_state.png similarity index 100% rename from apps/emqx_gateway/src/coap/doc/shared_state.png rename to apps/emqx_coap/doc/shared_state.png diff --git a/apps/emqx_gateway/src/coap/doc/transport.png b/apps/emqx_coap/doc/transport.png similarity index 100% rename from apps/emqx_gateway/src/coap/doc/transport.png rename to apps/emqx_coap/doc/transport.png diff --git a/apps/emqx_gateway/i18n/emqx_coap_api_i18n.conf b/apps/emqx_coap/i18n/emqx_coap_api_i18n.conf similarity index 100% rename from apps/emqx_gateway/i18n/emqx_coap_api_i18n.conf rename to apps/emqx_coap/i18n/emqx_coap_api_i18n.conf diff --git a/apps/emqx_coap/i18n/emqx_coap_schema.conf b/apps/emqx_coap/i18n/emqx_coap_schema.conf new file mode 100644 index 000000000..1e6452e49 --- /dev/null +++ b/apps/emqx_coap/i18n/emqx_coap_schema.conf @@ -0,0 +1,80 @@ +emqx_coap_schema { + coap { + desc { + en: """The CoAP Gateway configuration. +This gateway is implemented based on RFC-7252 and https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" + zh: """CoAP 网关配置。 +该网关的实现基于 RFC-7252 和 https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" + } + } + + coap_heartbeat { + desc { + en: """The gateway server required minimum heartbeat interval. +When connection mode is enabled, this parameter is used to set the minimum heartbeat interval for the connection to be alive""" + zh: """CoAP 网关要求客户端的最小心跳间隔时间。 +当 connection_required 开启后，该参数用于检查客户端连接是否存活""" + } + } + + coap_connection_required { + desc { + en: """Enable or disable connection mode. +Connection mode is a feature of non-standard protocols. When connection mode is enabled, it is necessary to maintain the creation, authentication and alive of connection resources""" + zh: """是否开启连接模式。 +连接模式是非标准协议的功能。它维护 CoAP 客户端上线、认证、和连接状态的保持""" + } + } + + coap_notify_type { + desc { + en: """The Notification Message will be delivered to the CoAP client if a new message received on an observed topic. +The type of delivered coap message can be set to:
+ - non: Non-confirmable;
+ - con: Confirmable;
+ - qos: Mapping from QoS type of received message, QoS0 -> non, QoS1,2 -> con""" + zh: """投递给 CoAP 客户端的通知消息类型。当客户端 Observe 一个资源（或订阅某个主题）时，网关会向客户端推送新产生的消息。其消息类型可设置为：
+ - non: 不需要客户端返回确认消息;
+ - con: 需要客户端返回一个确认消息;
+ - qos: 取决于消息的 QoS 等级; QoS 0 会以 `non` 类型下发，QoS 1/2 会以 `con` 类型下发""" + } + } + + coap_subscribe_qos { + desc { + en: """The Default QoS Level indicator for subscribe request. +This option specifies the QoS level for the CoAP Client when establishing a subscription membership, if the subscribe request is not carried `qos` option. The indicator can be set to:
+ - qos0, qos1, qos2: Fixed default QoS level
+ - coap: Dynamic QoS level by the message type of subscribe request
+ * qos0: If the subscribe request is non-confirmable
+ * qos1: If the subscribe request is confirmable""" + + zh: """客户端订阅请求的默认 QoS 等级。 +当 CoAP 客户端发起订阅请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
+ - qos0、 qos1、qos2: 设置为固定的 QoS 等级
+ - coap: 依据订阅操作的 CoAP 报文类型来动态决定
+ * 当订阅请求为 `non-confirmable` 类型时，取值为 qos0
+ * 当订阅请求为 `confirmable` 类型时，取值为 qos1""" + } + } + + coap_publish_qos { + desc { + en: """The Default QoS Level indicator for publish request. +This option specifies the QoS level for the CoAP Client when publishing a message to EMQX PUB/SUB system, if the publish request is not carried `qos` option. The indicator can be set to:
+ - qos0, qos1, qos2: Fixed default QoS level
+ - coap: Dynamic QoS level by the message type of publish request
+ * qos0: If the publish request is non-confirmable
+ * qos1: If the publish request is confirmable""" + + zh: """客户端发布请求的默认 QoS 等级。 +当 CoAP 客户端发起发布请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
+ - qos0、qos1、qos2: 设置为固定的 QoS 等级
+ - coap: 依据发布操作的 CoAP 报文类型来动态决定
+ * 当发布请求为 `non-confirmable` 类型时，取值为 qos0
+ * 当发布请求为 `confirmable` 类型时，取值为 qos1""" + } + } + + +} diff --git a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl b/apps/emqx_coap/include/emqx_coap.hrl similarity index 100% rename from apps/emqx_gateway/src/coap/include/emqx_coap.hrl rename to apps/emqx_coap/include/emqx_coap.hrl diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_coap/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_coap/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src new file mode 100644 index 000000000..55c9de59d --- /dev/null +++ b/apps/emqx_coap/src/emqx_coap.app.src @@ -0,0 +1,10 @@ +{application, emqx_coap, [ + {description, "CoAP Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_coap/src/emqx_coap.erl similarity index 86% rename from apps/emqx_gateway/src/coap/emqx_coap_impl.erl rename to apps/emqx_coap/src/emqx_coap.erl index bebcef237..d553349a4 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl +++ b/apps/emqx_coap/src/emqx_coap.erl @@ -14,13 +14,29 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_coap_impl). - --behaviour(emqx_gateway_impl). +%% @doc The CoAP Gateway implement +-module(emqx_coap). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). +%% define a gateway named stomp +-gateway(#{ + name => coap, + callback_module => ?MODULE, + config_schema_module => emqx_coap_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + -import( emqx_gateway_utils, [ @@ -30,31 +46,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(coap, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(coap). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_coap/src/emqx_coap_api.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_api.erl rename to apps/emqx_coap/src/emqx_coap_api.erl index 0f4c7a053..50ea9829a 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl +++ b/apps/emqx_coap/src/emqx_coap_api.erl @@ -18,10 +18,10 @@ -behaviour(minirest_api). +-include("emqx_coap.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). %% API -export([api_spec/0, paths/0, schema/1, namespace/0]). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_coap/src/emqx_coap_channel.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_channel.erl rename to apps/emqx_coap/src/emqx_coap_channel.erl index d6b8594b1..4cf362d9d 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_coap/src/emqx_coap_channel.erl @@ -45,8 +45,8 @@ -export_type([channel/0]). +-include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). -include_lib("emqx/include/emqx_authentication.hrl"). -define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_coap/src/emqx_coap_frame.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_frame.erl rename to apps/emqx_coap/src/emqx_coap_frame.erl index 4d2479d75..a05116b14 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl +++ b/apps/emqx_coap/src/emqx_coap_frame.erl @@ -29,7 +29,7 @@ is_message/1 ]). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -include_lib("emqx/include/types.hrl"). -define(VERSION, 1). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl b/apps/emqx_coap/src/emqx_coap_medium.erl similarity index 98% rename from apps/emqx_gateway/src/coap/emqx_coap_medium.erl rename to apps/emqx_coap/src/emqx_coap_medium.erl index 8f5028f25..b6bd8e764 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl +++ b/apps/emqx_coap/src/emqx_coap_medium.erl @@ -20,7 +20,7 @@ -module(emqx_coap_medium). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). %% API -export([ diff --git a/apps/emqx_gateway/src/coap/emqx_coap_message.erl b/apps/emqx_coap/src/emqx_coap_message.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_message.erl rename to apps/emqx_coap/src/emqx_coap_message.erl index 99c9e0840..ee17231a7 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_message.erl +++ b/apps/emqx_coap/src/emqx_coap_message.erl @@ -43,7 +43,7 @@ set_payload_block/3, set_payload_block/4 ]). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). request(Type, Method) -> request(Type, Method, <<>>, []). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl b/apps/emqx_coap/src/emqx_coap_mqtt_handler.erl similarity index 96% rename from apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl rename to apps/emqx_coap/src/emqx_coap_mqtt_handler.erl index 59825a745..4bcf71b1a 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl +++ b/apps/emqx_coap/src/emqx_coap_mqtt_handler.erl @@ -16,7 +16,7 @@ -module(emqx_coap_mqtt_handler). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -export([handle_request/4]). -import(emqx_coap_message, [response/2, response/3]). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl b/apps/emqx_coap/src/emqx_coap_observe_res.erl similarity index 100% rename from apps/emqx_gateway/src/coap/emqx_coap_observe_res.erl rename to apps/emqx_coap/src/emqx_coap_observe_res.erl diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_coap/src/emqx_coap_pubsub_handler.erl similarity index 99% rename from apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl rename to apps/emqx_coap/src/emqx_coap_pubsub_handler.erl index 5e14ba9e4..da1f5e0ef 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_coap/src/emqx_coap_pubsub_handler.erl @@ -18,7 +18,7 @@ -module(emqx_coap_pubsub_handler). -include_lib("emqx/include/emqx_mqtt.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -export([handle_request/4]). diff --git a/apps/emqx_coap/src/emqx_coap_schema.erl b/apps/emqx_coap/src/emqx_coap_schema.erl new file mode 100644 index 000000000..b7ce88451 --- /dev/null +++ b/apps/emqx_coap/src/emqx_coap_schema.erl @@ -0,0 +1,95 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-type duration() :: non_neg_integer(). + +-typerefl_from_string({duration/0, emqx_schema, to_duration}). + +-reflect_type([duration/0]). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(coap) -> + [ + {heartbeat, + sc( + duration(), + #{ + default => <<"30s">>, + desc => ?DESC(coap_heartbeat) + } + )}, + {connection_required, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(coap_connection_required) + } + )}, + {notify_type, + sc( + hoconsc:enum([non, con, qos]), + #{ + default => qos, + desc => ?DESC(coap_notify_type) + } + )}, + {subscribe_qos, + sc( + hoconsc:enum([qos0, qos1, qos2, coap]), + #{ + default => coap, + desc => ?DESC(coap_subscribe_qos) + } + )}, + {publish_qos, + sc( + hoconsc:enum([qos0, qos1, qos2, coap]), + #{ + default => coap, + desc => ?DESC(coap_publish_qos) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, + sc( + ref(emqx_gateway_schema, udp_listeners), + #{desc => ?DESC(udp_listeners)} + )} + ] ++ emqx_gateway_schema:gateway_common_options(). + +desc(coap) -> + "The CoAP protocol gateway provides EMQX with the access capability of the CoAP protocol.\n" + "It allows publishing, subscribing, and receiving messages to EMQX in accordance\n" + "with a certain defined CoAP message format."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% helpers + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_coap/src/emqx_coap_session.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_session.erl rename to apps/emqx_coap/src/emqx_coap_session.erl index 253f34d4d..688defcbb 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_session.erl +++ b/apps/emqx_coap/src/emqx_coap_session.erl @@ -15,10 +15,10 @@ %%-------------------------------------------------------------------- -module(emqx_coap_session). +-include("emqx_coap.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). %% API -export([ diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_coap/src/emqx_coap_tm.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_tm.erl rename to apps/emqx_coap/src/emqx_coap_tm.erl index 1a0004f8c..297cbca6b 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl +++ b/apps/emqx_coap/src/emqx_coap_tm.erl @@ -29,8 +29,8 @@ -export_type([manager/0, event_result/1]). +-include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). -type direction() :: in | out. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_coap/src/emqx_coap_transport.erl similarity index 99% rename from apps/emqx_gateway/src/coap/emqx_coap_transport.erl rename to apps/emqx_coap/src/emqx_coap_transport.erl index 1e6c5238a..c58a8abbd 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl +++ b/apps/emqx_coap/src/emqx_coap_transport.erl @@ -16,8 +16,8 @@ -module(emqx_coap_transport). +-include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). -define(ACK_TIMEOUT, 2000). -define(ACK_RANDOM_FACTOR, 1000). diff --git a/apps/emqx_gateway/test/emqx_coap_SUITE.erl b/apps/emqx_coap/test/emqx_coap_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_coap_SUITE.erl rename to apps/emqx_coap/test/emqx_coap_SUITE.erl index db99c3df1..1d33e042a 100644 --- a/apps/emqx_gateway/test/emqx_coap_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_SUITE.erl @@ -56,6 +56,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + application:load(emqx_coap), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_coap/test/emqx_coap_api_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_coap_api_SUITE.erl rename to apps/emqx_coap/test/emqx_coap_api_SUITE.erl index 6c1354bc0..9c418ab57 100644 --- a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_coap/test/emqx_coap_api_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -56,6 +56,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + application:load(emqx_coap), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), Config. diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf index aaa5007ee..9ef5f3d5d 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf @@ -1,82 +1,5 @@ emqx_gateway_schema { - coap { - desc { - en: """The CoAP Gateway configuration. -This gateway is implemented based on RFC-7252 and https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" - zh: """CoAP 网关配置。 -该网关的实现基于 RFC-7252 和 https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" - } - } - - coap_heartbeat { - desc { - en: """The gateway server required minimum heartbeat interval. -When connection mode is enabled, this parameter is used to set the minimum heartbeat interval for the connection to be alive""" - zh: """CoAP 网关要求客户端的最小心跳间隔时间。 -当 connection_required 开启后，该参数用于检查客户端连接是否存活""" - } - } - - coap_connection_required { - desc { - en: """Enable or disable connection mode. -Connection mode is a feature of non-standard protocols. When connection mode is enabled, it is necessary to maintain the creation, authentication and alive of connection resources""" - zh: """是否开启连接模式。 -连接模式是非标准协议的功能。它维护 CoAP 客户端上线、认证、和连接状态的保持""" - } - } - - coap_notify_type { - desc { - en: """The Notification Message will be delivered to the CoAP client if a new message received on an observed topic. -The type of delivered coap message can be set to:
- - non: Non-confirmable;
- - con: Confirmable;
- - qos: Mapping from QoS type of received message, QoS0 -> non, QoS1,2 -> con""" - zh: """投递给 CoAP 客户端的通知消息类型。当客户端 Observe 一个资源（或订阅某个主题）时，网关会向客户端推送新产生的消息。其消息类型可设置为：
- - non: 不需要客户端返回确认消息;
- - con: 需要客户端返回一个确认消息;
- - qos: 取决于消息的 QoS 等级; QoS 0 会以 `non` 类型下发，QoS 1/2 会以 `con` 类型下发""" - } - } - - coap_subscribe_qos { - desc { - en: """The Default QoS Level indicator for subscribe request. -This option specifies the QoS level for the CoAP Client when establishing a subscription membership, if the subscribe request is not carried `qos` option. The indicator can be set to:
- - qos0, qos1, qos2: Fixed default QoS level
- - coap: Dynamic QoS level by the message type of subscribe request
- * qos0: If the subscribe request is non-confirmable
- * qos1: If the subscribe request is confirmable""" - - zh: """客户端订阅请求的默认 QoS 等级。 -当 CoAP 客户端发起订阅请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
- - qos0、 qos1、qos2: 设置为固定的 QoS 等级
- - coap: 依据订阅操作的 CoAP 报文类型来动态决定
- * 当订阅请求为 `non-confirmable` 类型时，取值为 qos0
- * 当订阅请求为 `confirmable` 类型时，取值为 qos1""" - } - } - - coap_publish_qos { - desc { - en: """The Default QoS Level indicator for publish request. -This option specifies the QoS level for the CoAP Client when publishing a message to EMQX PUB/SUB system, if the publish request is not carried `qos` option. The indicator can be set to:
- - qos0, qos1, qos2: Fixed default QoS level
- - coap: Dynamic QoS level by the message type of publish request
- * qos0: If the publish request is non-confirmable
- * qos1: If the publish request is confirmable""" - - zh: """客户端发布请求的默认 QoS 等级。 -当 CoAP 客户端发起发布请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
- - qos0、qos1、qos2: 设置为固定的 QoS 等级
- - coap: 依据发布操作的 CoAP 报文类型来动态决定
- * 当发布请求为 `non-confirmable` 类型时，取值为 qos0
- * 当发布请求为 `confirmable` 类型时，取值为 qos1""" - } - } - lwm2m { desc { en: """The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index a805a0ceb..0c78341e1 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -47,11 +47,6 @@ load_default_gateway_applications() -> callback_module => emqx_lwm2m_impl, config_schema_module => emqx_lwm2m_schema }, - #{ - name => coap, - callback_module => emqx_coap_impl, - config_schema_module => emqx_gateway_schema - }, #{ name => exproto, callback_module => emqx_exproto_impl, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index f7ce3d05c..f9cdcfe26 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -64,14 +64,6 @@ roots() -> [gateway]. fields(gateway) -> [ - {coap, - sc( - ref(coap), - #{ - required => {false, recursively}, - desc => ?DESC(coap) - } - )}, {lwm2m, sc( ref(lwm2m), @@ -89,55 +81,6 @@ fields(gateway) -> } )} ] ++ gateway_schemas(); -fields(coap) -> - [ - {heartbeat, - sc( - duration(), - #{ - default => <<"30s">>, - desc => ?DESC(coap_heartbeat) - } - )}, - {connection_required, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(coap_connection_required) - } - )}, - {notify_type, - sc( - hoconsc:enum([non, con, qos]), - #{ - default => qos, - desc => ?DESC(coap_notify_type) - } - )}, - {subscribe_qos, - sc( - hoconsc:enum([qos0, qos1, qos2, coap]), - #{ - default => coap, - desc => ?DESC(coap_subscribe_qos) - } - )}, - {publish_qos, - sc( - hoconsc:enum([qos0, qos1, qos2, coap]), - #{ - default => coap, - desc => ?DESC(coap_publish_qos) - } - )}, - {mountpoint, mountpoint()}, - {listeners, - sc( - ref(udp_listeners), - #{desc => ?DESC(udp_listeners)} - )} - ] ++ gateway_common_options(); fields(lwm2m) -> [ {xml_dir, @@ -413,10 +356,6 @@ fields(dtls_opts) -> desc(gateway) -> "EMQX Gateway configuration root."; -desc(coap) -> - "The CoAP protocol gateway provides EMQX with the access capability of the CoAP protocol.\n" - "It allows publishing, subscribing, and receiving messages to EMQX in accordance\n" - "with a certain defined CoAP message format."; desc(lwm2m) -> "The LwM2M protocol gateway."; desc(exproto) -> @@ -597,11 +536,11 @@ proxy_protocol_opts() -> %% dynamic schemas %% FIXME: don't hardcode the gateway names -gateway_schema(coap) -> fields(coap); gateway_schema(lwm2m) -> fields(lwm2m); gateway_schema(exproto) -> fields(exproto); gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); -gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn). +gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn); +gateway_schema(coap) -> emqx_coap_schema:fields(coap). gateway_schemas() -> lists:map( diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src index e58cf5147..36e4342d1 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src +++ b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src @@ -1,10 +1,10 @@ -{application, emqx_mqttsn, - [{description, "MQTT-SN Gateway"}, - {vsn, "0.1.0"}, - {registered, []}, - {applications, [kernel, stdlib]}, - {env,[]}, - {modules, []}, - {licenses, ["Apache 2.0"]}, - {links, []} - ]}. +{application, emqx_mqttsn, [ + {description, "MQTT-SN Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index dffb9b7a4..e2d1f997b 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,5 +1,5 @@ {application, emqx_stomp, [ - {description, "Stomp gateway"}, + {description, "Stomp Gateway"}, {vsn, "0.1.0"}, {registered, []}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.erl b/apps/emqx_stomp/src/emqx_stomp.erl index 6c14e222c..dbfdfdce5 100644 --- a/apps/emqx_stomp/src/emqx_stomp.erl +++ b/apps/emqx_stomp/src/emqx_stomp.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The Stomp Gateway implement interface +%% @doc The Stomp Gateway implement -module(emqx_stomp). -include_lib("emqx/include/logger.hrl"). diff --git a/mix.exs b/mix.exs index 514c9139d..229e40824 100644 --- a/mix.exs +++ b/mix.exs @@ -281,6 +281,9 @@ defmodule EMQXUmbrella.MixProject do emqx_authz: :permanent, emqx_auto_subscribe: :permanent, emqx_gateway: :permanent, + emqx_stomp: :permanent, + emqx_mqttsn: :permanent, + emqx_coap: :permanent, emqx_exhook: :permanent, emqx_bridge: :permanent, emqx_rule_engine: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index 47bf925d4..ee6532de9 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -391,6 +391,7 @@ relx_apps(ReleaseType, Edition) -> emqx_gateway, emqx_stomp, emqx_mqttsn, + emqx_coap, emqx_exhook, emqx_bridge, emqx_rule_engine, From 40c413ac055ed8cb81a33cf197d678b5a91868a1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 15:48:42 +0800 Subject: [PATCH 006/279] chore: fix dialyzer warnings --- apps/emqx_coap/src/emqx_coap.app.src | 2 +- apps/emqx_gateway/include/emqx_gateway.hrl | 7 +++++++ apps/emqx_gateway/src/emqx_gateway_app.erl | 6 +++++- apps/emqx_gateway/src/emqx_gateway_utils.erl | 1 + apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl | 2 +- apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl | 2 +- apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl | 2 +- apps/emqx_mqttsn/src/emqx_mqttsn.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- 9 files changed, 19 insertions(+), 7 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src index 55c9de59d..50b593ac7 100644 --- a/apps/emqx_coap/src/emqx_coap.app.src +++ b/apps/emqx_coap/src/emqx_coap.app.src @@ -2,7 +2,7 @@ {description, "CoAP Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_gateway/include/emqx_gateway.hrl b/apps/emqx_gateway/include/emqx_gateway.hrl index 3466ecd98..51a519589 100644 --- a/apps/emqx_gateway/include/emqx_gateway.hrl +++ b/apps/emqx_gateway/include/emqx_gateway.hrl @@ -37,4 +37,11 @@ config => emqx_config:config() }. +-type gateway_def() :: + #{ + name := gateway_name(), + callback_module := module(), + config_schema_module := module() + }. + -endif. diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 0c78341e1..0f9ef87e1 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -80,7 +80,11 @@ load_gateway_application( name => Name, callback_module => CbMod }) - end. + end; +load_gateway_application(_) -> + ?SLOG(error, #{ + msg => "invalid_gateway_defination" + }). load_gateway_by_default() -> load_gateway_by_default(confs()). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 94c7490cc..9d71263f8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -564,6 +564,7 @@ make_compatible_schema2(Path, SchemaFun) -> Schema ). +-spec find_gateway_definations() -> list(gateway_def()). find_gateway_definations() -> lists:flatten( lists:map( diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl index 16d0f9630..12fd07d93 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -17,7 +17,7 @@ -module(emqx_lwm2m_channel). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include_lib("emqx_coap/include/emqx_coap.hrl"). -include("src/lwm2m/include/emqx_lwm2m.hrl"). %% API diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl index 090af3e87..470cab8b7 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl @@ -17,7 +17,7 @@ -module(emqx_lwm2m_cmd). -include_lib("emqx/include/logger.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include_lib("emqx_coap/include/emqx_coap.hrl"). -include("src/lwm2m/include/emqx_lwm2m.hrl"). -export([ diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl index 8634280e3..36244847a 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl @@ -15,12 +15,12 @@ %%-------------------------------------------------------------------- -module(emqx_lwm2m_session). --include("src/coap/include/emqx_coap.hrl"). -include("src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx_coap/include/emqx_coap.hrl"). %% API -export([ diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src index 36e4342d1..76acc648e 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src +++ b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src @@ -2,7 +2,7 @@ {description, "MQTT-SN Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index e2d1f997b..cd9670056 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -2,7 +2,7 @@ {description, "Stomp Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, From b58ce0965877d3ece5c6b39ac3f66bc61c3ee4a6 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 16:32:22 +0800 Subject: [PATCH 007/279] chore: split out lwm2m --- .../i18n/emqx_gateway_schema_i18n.conf | 125 ------------ apps/emqx_gateway/include/emqx_gateway.hrl | 2 +- apps/emqx_gateway/src/emqx_gateway_app.erl | 5 - apps/emqx_gateway/src/emqx_gateway_schema.erl | 149 +------------- apps/emqx_gateway/src/lwm2m/.gitignore | 25 --- apps/emqx_lwm2m/.gitignore | 19 ++ apps/emqx_lwm2m/LICENSE | 191 ++++++++++++++++++ .../src/lwm2m => emqx_lwm2m}/README.md | 1 - apps/emqx_lwm2m/i18n/emqx_lwm2m_schema.conf | 127 ++++++++++++ .../include/emqx_lwm2m.hrl | 0 .../lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml | 0 .../LWM2M_Connectivity_Statistics-v1_0_1.xml | 0 .../lwm2m_xml/LWM2M_Device-v1_0_1.xml | 0 .../LWM2M_Firmware_Update-v1_0_1.xml | 0 .../lwm2m_xml/LWM2M_Location-v1_0.xml | 0 .../lwm2m_xml/LWM2M_Security-v1_0.xml | 0 .../lwm2m_xml/LWM2M_Server-v1_0.xml | 0 apps/emqx_lwm2m/rebar.config | 2 + .../lwm2m => emqx_lwm2m/src}/binary_util.erl | 0 apps/emqx_lwm2m/src/emqx_lwm2m.app.src | 10 + .../src/emqx_lwm2m.erl} | 40 ++-- .../src}/emqx_lwm2m_api.erl | 0 .../src}/emqx_lwm2m_channel.erl | 2 +- .../src}/emqx_lwm2m_cmd.erl | 2 +- .../src}/emqx_lwm2m_message.erl | 2 +- apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl | 184 +++++++++++++++++ .../src}/emqx_lwm2m_session.erl | 2 +- .../src}/emqx_lwm2m_tlv.erl | 2 +- .../src}/emqx_lwm2m_xml_object.erl | 2 +- .../src}/emqx_lwm2m_xml_object_db.erl | 2 +- .../test/emqx_lwm2m_SUITE.erl | 5 +- .../test/emqx_lwm2m_api_SUITE.erl | 7 +- .../test/emqx_tlv_SUITE.erl | 4 +- mix.exs | 1 + rebar.config.erl | 3 +- 35 files changed, 576 insertions(+), 338 deletions(-) delete mode 100644 apps/emqx_gateway/src/lwm2m/.gitignore create mode 100644 apps/emqx_lwm2m/.gitignore create mode 100644 apps/emqx_lwm2m/LICENSE rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/README.md (99%) create mode 100644 apps/emqx_lwm2m/i18n/emqx_lwm2m_schema.conf rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/include/emqx_lwm2m.hrl (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Device-v1_0_1.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Location-v1_0.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Security-v1_0.xml (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m}/lwm2m_xml/LWM2M_Server-v1_0.xml (100%) create mode 100644 apps/emqx_lwm2m/rebar.config rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/binary_util.erl (100%) create mode 100644 apps/emqx_lwm2m/src/emqx_lwm2m.app.src rename apps/{emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl => emqx_lwm2m/src/emqx_lwm2m.erl} (87%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_api.erl (100%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_channel.erl (99%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_cmd.erl (99%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_message.erl (99%) create mode 100644 apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_session.erl (99%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_tlv.erl (99%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_xml_object.erl (98%) rename apps/{emqx_gateway/src/lwm2m => emqx_lwm2m/src}/emqx_lwm2m_xml_object_db.erl (99%) rename apps/{emqx_gateway => emqx_lwm2m}/test/emqx_lwm2m_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_lwm2m}/test/emqx_lwm2m_api_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_lwm2m}/test/emqx_tlv_SUITE.erl (99%) diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf index 9ef5f3d5d..561627241 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf @@ -1,130 +1,5 @@ emqx_gateway_schema { - lwm2m { - desc { - en: """The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" - zh: """LwM2M 网关配置。仅支持 v1.0.1 协议。""" - } - } - - lwm2m_xml_dir { - desc { - en: """The Directory for LwM2M Resource definition.""" - zh: """LwM2M Resource 定义的 XML 文件目录路径。""" - } - } - - lwm2m_lifetime_min { - desc { - en: """Minimum value of lifetime allowed to be set by the LwM2M client.""" - zh: """允许 LwM2M 客户端允许设置的心跳最小值。""" - } - } - - lwm2m_lifetime_max { - desc { - en: """Maximum value of lifetime allowed to be set by the LwM2M client.""" - zh: """允许 LwM2M 客户端允许设置的心跳最大值。""" - } - } - - lwm2m_qmode_time_window { - desc { - en: """The value of the time window during which the network link is considered valid by the LwM2M Gateway in QMode mode. -For example, after receiving an update message from a client, any messages within this time window are sent directly to the LwM2M client, and all messages beyond this time window are temporarily stored in memory.""" - - zh: """在QMode模式下，LwM2M网关认为网络链接有效的时间窗口的值。 -例如，在收到客户端的更新信息后，在这个时间窗口内的任何信息都会直接发送到LwM2M客户端，而超过这个时间窗口的所有信息都会暂时储存在内存中。""" - } - } - - lwm2m_auto_observe { - desc { - en: """Automatically observe the object list of REGISTER packet.""" - zh: """自动 Observe REGISTER 数据包的 Object 列表。""" - } - } - - lwm2m_update_msg_publish_condition { - desc { - en: """Policy for publishing UPDATE event message.
- - always: send update events as long as the UPDATE request is received.
- - contains_object_list: send update events only if the UPDATE request carries any Object List""" - zh: """发布UPDATE事件消息的策略。
- - always: 只要收到 UPDATE 请求，就发送更新事件。
- - contains_object_list: 仅当 UPDATE 请求携带 Object 列表时才发送更新事件。""" - } - } - - lwm2m_translators { - desc { - en: """Topic configuration for LwM2M's gateway publishing and subscription.""" - zh: """LwM2M 网关订阅/发布消息的主题映射配置。""" - } - } - - lwm2m_translators_command { - desc { - en: """The topic for receiving downstream commands. -For each new LwM2M client that succeeds in going online, the gateway creates a subscription relationship to receive downstream commands and send it to the LwM2M client""" - - zh: """下行命令主题。 -对于每个成功上线的新 LwM2M 客户端，网关会创建一个订阅关系来接收下行消息并将其发送给客户端。""" - } - } - - lwm2m_translators_response { - desc { - en: """The topic for gateway to publish the acknowledge events from LwM2M client""" - zh: """用于网关发布来自 LwM2M 客户端的确认事件的主题。""" - } - } - - lwm2m_translators_notify { - desc { - en: """The topic for gateway to publish the notify events from LwM2M client. -After succeed observe a resource of LwM2M client, Gateway will send the notify events via this topic, if the client reports any resource changes""" - - zh: """用于发布来自 LwM2M 客户端的通知事件的主题。 -在成功 Observe 到 LwM2M 客户端的资源后，如果客户端报告任何资源状态的变化，网关将通过该主题发送通知事件。""" - } - } - - lwm2m_translators_register { - desc { - en: """The topic for gateway to publish the register events from LwM2M client.""" - zh: """用于发布来自 LwM2M 客户端的注册事件的主题。""" - } - } - - lwm2m_translators_update { - desc { - en: """The topic for gateway to publish the update events from LwM2M client""" - zh: """用于发布来自LwM2M客户端的更新事件的主题。""" - } - } - - translator { - desc { - en: """MQTT topic that corresponds to a particular type of event.""" - zh: """配置某网关客户端对于发布消息或订阅的主题和 QoS 等级。""" - } - } - - translator_topic { - desc { - en: """Topic Name""" - zh: """主题名称""" - } - } - - translator_qos { - desc { - en: """QoS Level""" - zh: """QoS 等级""" - } - } - exproto { desc { en: """The Extension Protocol configuration""" diff --git a/apps/emqx_gateway/include/emqx_gateway.hrl b/apps/emqx_gateway/include/emqx_gateway.hrl index 51a519589..c880aca26 100644 --- a/apps/emqx_gateway/include/emqx_gateway.hrl +++ b/apps/emqx_gateway/include/emqx_gateway.hrl @@ -42,6 +42,6 @@ name := gateway_name(), callback_module := module(), config_schema_module := module() - }. + }. -endif. diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 0f9ef87e1..5999e85c9 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -42,11 +42,6 @@ stop(_State) -> load_default_gateway_applications() -> BuiltInGateways = [ - #{ - name => lwm2m, - callback_module => emqx_lwm2m_impl, - config_schema_module => emqx_lwm2m_schema - }, #{ name => exproto, callback_module => emqx_exproto_impl, diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index f9cdcfe26..6a4811b94 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -64,14 +64,6 @@ roots() -> [gateway]. fields(gateway) -> [ - {lwm2m, - sc( - ref(lwm2m), - #{ - required => {false, recursively}, - desc => ?DESC(lwm2m) - } - )}, {exproto, sc( ref(exproto), @@ -81,75 +73,6 @@ fields(gateway) -> } )} ] ++ gateway_schemas(); -fields(lwm2m) -> - [ - {xml_dir, - sc( - binary(), - #{ - %% since this is not packaged with emqx, nor - %% present in the packages, we must let the user - %% specify it rather than creating a dynamic - %% default (especially difficult to handle when - %% generating docs). - example => <<"/etc/emqx/lwm2m_xml">>, - required => true, - desc => ?DESC(lwm2m_xml_dir) - } - )}, - {lifetime_min, - sc( - duration(), - #{ - default => <<"15s">>, - desc => ?DESC(lwm2m_lifetime_min) - } - )}, - {lifetime_max, - sc( - duration(), - #{ - default => <<"86400s">>, - desc => ?DESC(lwm2m_lifetime_max) - } - )}, - {qmode_time_window, - sc( - duration_s(), - #{ - default => <<"22s">>, - desc => ?DESC(lwm2m_qmode_time_window) - } - )}, - %% TODO: Support config resource path - {auto_observe, - sc( - boolean(), - #{ - default => false, - desc => ?DESC(lwm2m_auto_observe) - } - )}, - %% FIXME: not working now - {update_msg_publish_condition, - sc( - hoconsc:enum([always, contains_object_list]), - #{ - default => contains_object_list, - desc => ?DESC(lwm2m_update_msg_publish_condition) - } - )}, - {translators, - sc( - ref(lwm2m_translators), - #{ - required => true, - desc => ?DESC(lwm2m_translators) - } - )}, - {mountpoint, mountpoint("lwm2m/${endpoint_name}/")}, - {listeners, sc(ref(udp_listeners), #{desc => ?DESC(udp_listeners)})} - ] ++ gateway_common_options(); fields(exproto) -> [ {server, @@ -223,68 +146,6 @@ fields(clientinfo_override) -> })}, {clientid, sc(binary(), #{desc => ?DESC(gateway_common_clientinfo_override_clientid)})} ]; -fields(lwm2m_translators) -> - [ - {command, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_command), - required => true - } - )}, - {response, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_response), - required => true - } - )}, - {notify, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_notify), - required => true - } - )}, - {register, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_register), - required => true - } - )}, - {update, - sc( - ref(translator), - #{ - desc => ?DESC(lwm2m_translators_update), - required => true - } - )} - ]; -fields(translator) -> - [ - {topic, - sc( - binary(), - #{ - required => true, - desc => ?DESC(translator_topic) - } - )}, - {qos, - sc( - emqx_schema:qos(), - #{ - default => 0, - desc => ?DESC(translator_qos) - } - )} - ]; fields(udp_listeners) -> [ {udp, sc(map(name, ref(udp_listener)), #{desc => ?DESC(udp_listener)})}, @@ -356,8 +217,6 @@ fields(dtls_opts) -> desc(gateway) -> "EMQX Gateway configuration root."; -desc(lwm2m) -> - "The LwM2M protocol gateway."; desc(exproto) -> "Settings for EMQX extension protocol (exproto)."; desc(exproto_grpc_server) -> @@ -368,10 +227,6 @@ desc(ssl_server_opts) -> "SSL configuration for the server."; desc(clientinfo_override) -> "ClientInfo override."; -desc(lwm2m_translators) -> - "MQTT topics that correspond to LwM2M events."; -desc(translator) -> - "MQTT topic that corresponds to a particular type of event."; desc(udp_listeners) -> "Settings for the UDP listeners."; desc(tcp_listeners) -> @@ -536,11 +391,11 @@ proxy_protocol_opts() -> %% dynamic schemas %% FIXME: don't hardcode the gateway names -gateway_schema(lwm2m) -> fields(lwm2m); gateway_schema(exproto) -> fields(exproto); gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn); -gateway_schema(coap) -> emqx_coap_schema:fields(coap). +gateway_schema(coap) -> emqx_coap_schema:fields(coap); +gateway_schema(lwm2m) -> emqx_lwm2m_schema:fields(lwm2m). gateway_schemas() -> lists:map( diff --git a/apps/emqx_gateway/src/lwm2m/.gitignore b/apps/emqx_gateway/src/lwm2m/.gitignore deleted file mode 100644 index be6914be3..000000000 --- a/apps/emqx_gateway/src/lwm2m/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -deps/ -ebin/ -_rel/ -.erlang.mk/ -*.d -*.o -*.exe -data/ -*.iml -.idea/ -logs/ -*.beam -emqx_coap.d -erlang.mk -integration_test/emqx-rel/ -integration_test/build_wakaama/ -integration_test/case*.txt -integration_test/paho/ -integration_test/wakaama/ -_build/ -rebar.lock -rebar3.crashdump -*.conf.rendered -.rebar3/ -*.swp diff --git a/apps/emqx_lwm2m/.gitignore b/apps/emqx_lwm2m/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_lwm2m/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_lwm2m/LICENSE b/apps/emqx_lwm2m/LICENSE new file mode 100644 index 000000000..5a5418f0f --- /dev/null +++ b/apps/emqx_lwm2m/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/src/lwm2m/README.md b/apps/emqx_lwm2m/README.md similarity index 99% rename from apps/emqx_gateway/src/lwm2m/README.md rename to apps/emqx_lwm2m/README.md index bf7626c6f..faca6dad3 100644 --- a/apps/emqx_gateway/src/lwm2m/README.md +++ b/apps/emqx_lwm2m/README.md @@ -1,4 +1,3 @@ - # LwM2M Gateway [The LwM2M Specifications](http://www.openmobilealliance.org/release/LightweightM2M) is a Lightweight Machine to Machine protocol. diff --git a/apps/emqx_lwm2m/i18n/emqx_lwm2m_schema.conf b/apps/emqx_lwm2m/i18n/emqx_lwm2m_schema.conf new file mode 100644 index 000000000..822570f1d --- /dev/null +++ b/apps/emqx_lwm2m/i18n/emqx_lwm2m_schema.conf @@ -0,0 +1,127 @@ +emqx_lwm2m_schema { + + lwm2m { + desc { + en: """The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" + zh: """LwM2M 网关配置。仅支持 v1.0.1 协议。""" + } + } + + lwm2m_xml_dir { + desc { + en: """The Directory for LwM2M Resource definition.""" + zh: """LwM2M Resource 定义的 XML 文件目录路径。""" + } + } + + lwm2m_lifetime_min { + desc { + en: """Minimum value of lifetime allowed to be set by the LwM2M client.""" + zh: """允许 LwM2M 客户端允许设置的心跳最小值。""" + } + } + + lwm2m_lifetime_max { + desc { + en: """Maximum value of lifetime allowed to be set by the LwM2M client.""" + zh: """允许 LwM2M 客户端允许设置的心跳最大值。""" + } + } + + lwm2m_qmode_time_window { + desc { + en: """The value of the time window during which the network link is considered valid by the LwM2M Gateway in QMode mode. +For example, after receiving an update message from a client, any messages within this time window are sent directly to the LwM2M client, and all messages beyond this time window are temporarily stored in memory.""" + + zh: """在QMode模式下，LwM2M网关认为网络链接有效的时间窗口的值。 +例如，在收到客户端的更新信息后，在这个时间窗口内的任何信息都会直接发送到LwM2M客户端，而超过这个时间窗口的所有信息都会暂时储存在内存中。""" + } + } + + lwm2m_auto_observe { + desc { + en: """Automatically observe the object list of REGISTER packet.""" + zh: """自动 Observe REGISTER 数据包的 Object 列表。""" + } + } + + lwm2m_update_msg_publish_condition { + desc { + en: """Policy for publishing UPDATE event message.
+ - always: send update events as long as the UPDATE request is received.
+ - contains_object_list: send update events only if the UPDATE request carries any Object List""" + zh: """发布UPDATE事件消息的策略。
+ - always: 只要收到 UPDATE 请求，就发送更新事件。
+ - contains_object_list: 仅当 UPDATE 请求携带 Object 列表时才发送更新事件。""" + } + } + + lwm2m_translators { + desc { + en: """Topic configuration for LwM2M's gateway publishing and subscription.""" + zh: """LwM2M 网关订阅/发布消息的主题映射配置。""" + } + } + + lwm2m_translators_command { + desc { + en: """The topic for receiving downstream commands. +For each new LwM2M client that succeeds in going online, the gateway creates a subscription relationship to receive downstream commands and send it to the LwM2M client""" + + zh: """下行命令主题。 +对于每个成功上线的新 LwM2M 客户端，网关会创建一个订阅关系来接收下行消息并将其发送给客户端。""" + } + } + + lwm2m_translators_response { + desc { + en: """The topic for gateway to publish the acknowledge events from LwM2M client""" + zh: """用于网关发布来自 LwM2M 客户端的确认事件的主题。""" + } + } + + lwm2m_translators_notify { + desc { + en: """The topic for gateway to publish the notify events from LwM2M client. +After succeed observe a resource of LwM2M client, Gateway will send the notify events via this topic, if the client reports any resource changes""" + + zh: """用于发布来自 LwM2M 客户端的通知事件的主题。 +在成功 Observe 到 LwM2M 客户端的资源后，如果客户端报告任何资源状态的变化，网关将通过该主题发送通知事件。""" + } + } + + lwm2m_translators_register { + desc { + en: """The topic for gateway to publish the register events from LwM2M client.""" + zh: """用于发布来自 LwM2M 客户端的注册事件的主题。""" + } + } + + lwm2m_translators_update { + desc { + en: """The topic for gateway to publish the update events from LwM2M client""" + zh: """用于发布来自LwM2M客户端的更新事件的主题。""" + } + } + + translator { + desc { + en: """MQTT topic that corresponds to a particular type of event.""" + zh: """配置某网关客户端对于发布消息或订阅的主题和 QoS 等级。""" + } + } + + translator_topic { + desc { + en: """Topic Name""" + zh: """主题名称""" + } + } + + translator_qos { + desc { + en: """QoS Level""" + zh: """QoS 等级""" + } + } +} diff --git a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_lwm2m/include/emqx_lwm2m.hrl similarity index 100% rename from apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl rename to apps/emqx_lwm2m/include/emqx_lwm2m.hrl diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml b/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml similarity index 100% rename from apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml rename to apps/emqx_lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_lwm2m/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_gateway/src/lwm2m/binary_util.erl b/apps/emqx_lwm2m/src/binary_util.erl similarity index 100% rename from apps/emqx_gateway/src/lwm2m/binary_util.erl rename to apps/emqx_lwm2m/src/binary_util.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src new file mode 100644 index 000000000..08c3dbe3f --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -0,0 +1,10 @@ +{application, emqx_lwm2m, [ + {description, "LwM2M Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, emqx_gateway, emqx_coap]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_lwm2m/src/emqx_lwm2m.erl similarity index 87% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m.erl index fa4537315..222d1076e 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.erl @@ -14,35 +14,37 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The LwM2M Gateway Implement interface --module(emqx_lwm2m_impl). - --behaviour(emqx_gateway_impl). +%% @doc The LwM2M Gateway implement +-module(emqx_lwm2m). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). -%% APIs --export([ - reg/0, - unreg/0 -]). +%% define a gateway named stomp +-gateway(#{ + name => lwm2m, + callback_module => ?MODULE, + config_schema_module => emqx_lwm2m_schema +}). +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl -export([ on_gateway_load/2, on_gateway_update/3, on_gateway_unload/2 ]). -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(lwm2m, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(lwm2m). +-import( + emqx_gateway_utils, + [ + normalize_config/1, + start_listeners/4, + stop_listeners/2 + ] +). %%-------------------------------------------------------------------- %% emqx_gateway_registry callbacks diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl similarity index 100% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_api.erl diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl index 12fd07d93..54e5723cd 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl @@ -16,9 +16,9 @@ -module(emqx_lwm2m_channel). +-include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_coap/include/emqx_coap.hrl"). --include("src/lwm2m/include/emqx_lwm2m.hrl"). %% API -export([ diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl index 470cab8b7..9b1f6b65d 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl @@ -16,9 +16,9 @@ -module(emqx_lwm2m_cmd). +-include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_coap/include/emqx_coap.hrl"). --include("src/lwm2m/include/emqx_lwm2m.hrl"). -export([ mqtt_to_coap/2, diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_message.erl index f09a8ea3d..e541e83f1 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_message.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl @@ -24,7 +24,7 @@ translate_json/1 ]). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). tlv_to_json(BaseName, TlvData) -> DecodedTlv = emqx_lwm2m_tlv:parse(TlvData), diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl new file mode 100644 index 000000000..b674c3260 --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl @@ -0,0 +1,184 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-type duration() :: non_neg_integer(). +-type duration_s() :: non_neg_integer(). + +-typerefl_from_string({duration/0, emqx_schema, to_duration}). +-typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). + +-reflect_type([duration/0, duration_s/0]). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(lwm2m) -> + [ + {xml_dir, + sc( + binary(), + #{ + %% since this is not packaged with emqx, nor + %% present in the packages, we must let the user + %% specify it rather than creating a dynamic + %% default (especially difficult to handle when + %% generating docs). + example => <<"/etc/emqx/lwm2m_xml">>, + required => true, + desc => ?DESC(lwm2m_xml_dir) + } + )}, + {lifetime_min, + sc( + duration(), + #{ + default => <<"15s">>, + desc => ?DESC(lwm2m_lifetime_min) + } + )}, + {lifetime_max, + sc( + duration(), + #{ + default => <<"86400s">>, + desc => ?DESC(lwm2m_lifetime_max) + } + )}, + {qmode_time_window, + sc( + duration_s(), + #{ + default => <<"22s">>, + desc => ?DESC(lwm2m_qmode_time_window) + } + )}, + %% TODO: Support config resource path + {auto_observe, + sc( + boolean(), + #{ + default => false, + desc => ?DESC(lwm2m_auto_observe) + } + )}, + %% FIXME: not working now + {update_msg_publish_condition, + sc( + hoconsc:enum([always, contains_object_list]), + #{ + default => contains_object_list, + desc => ?DESC(lwm2m_update_msg_publish_condition) + } + )}, + {translators, + sc( + ref(lwm2m_translators), + #{ + required => true, + desc => ?DESC(lwm2m_translators) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint("lwm2m/${endpoint_name}/")}, + {listeners, sc(ref(emqx_gateway_schema, udp_listeners), #{desc => ?DESC(udp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(lwm2m_translators) -> + [ + {command, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_command), + required => true + } + )}, + {response, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_response), + required => true + } + )}, + {notify, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_notify), + required => true + } + )}, + {register, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_register), + required => true + } + )}, + {update, + sc( + ref(translator), + #{ + desc => ?DESC(lwm2m_translators_update), + required => true + } + )} + ]; +fields(translator) -> + [ + {topic, + sc( + binary(), + #{ + required => true, + desc => ?DESC(translator_topic) + } + )}, + {qos, + sc( + emqx_schema:qos(), + #{ + default => 0, + desc => ?DESC(translator_qos) + } + )} + ]. + +desc(lwm2m) -> + "The LwM2M protocol gateway."; +desc(lwm2m_translators) -> + "MQTT topics that correspond to LwM2M events."; +desc(translator) -> + "MQTT topic that corresponds to a particular type of event."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% helpers + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_session.erl index 36244847a..67543a910 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_lwm2m_session). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl index 782bbec5e..2f53573c4 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_tlv.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl @@ -25,7 +25,7 @@ -export([binary_to_hex_string/1]). -endif. --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -define(TLV_TYPE_OBJECT_INSTANCE, 0). -define(TLV_TYPE_RESOURCE_INSTANCE, 1). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl similarity index 98% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl index a4dc44f2c..3525f72aa 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -export([ diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl similarity index 99% rename from apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl rename to apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl index 58373e114..04c4c1af2 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object_db). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include("emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl rename to apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl index fc852709c..9abe16a35 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -31,8 +31,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include("src/lwm2m/include/emqx_lwm2m.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include_lib("emqx_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -134,6 +134,7 @@ groups() -> init_per_suite(Config) -> %% load application first for minirest api searching application:load(emqx_gateway), + application:load(emqx_lwm2m), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]), Config. diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl rename to apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl index c40d1af55..bfeeb2c9b 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl @@ -23,8 +23,8 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include("src/lwm2m/include/emqx_lwm2m.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -81,8 +81,9 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), application:load(emqx_gateway), + application:load(emqx_lwm2m), + ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]), Config. diff --git a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl b/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_tlv_SUITE.erl rename to apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl index 5dcef7e72..da1e3a9c4 100644 --- a/apps/emqx_gateway/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl @@ -21,8 +21,8 @@ -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). --include("src/lwm2m/include/emqx_lwm2m.hrl"). --include("src/coap/include/emqx_coap.hrl"). +-include("emqx_lwm2m.hrl"). +-include("emqx_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- diff --git a/mix.exs b/mix.exs index 229e40824..264d4f87e 100644 --- a/mix.exs +++ b/mix.exs @@ -284,6 +284,7 @@ defmodule EMQXUmbrella.MixProject do emqx_stomp: :permanent, emqx_mqttsn: :permanent, emqx_coap: :permanent, + emqx_lwm2m: :permanent, emqx_exhook: :permanent, emqx_bridge: :permanent, emqx_rule_engine: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index ee6532de9..978f5ec87 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -392,6 +392,7 @@ relx_apps(ReleaseType, Edition) -> emqx_stomp, emqx_mqttsn, emqx_coap, + emqx_lwm2m, emqx_exhook, emqx_bridge, emqx_rule_engine, @@ -452,7 +453,7 @@ relx_overlay(ReleaseType, Edition) -> {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"}, %% for relup {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"}, - {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"}, + {copy, "apps/emqx_lwm2m/lwm2m_xml", "etc/lwm2m_xml"}, {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"}, {template, "bin/emqx.cmd", "bin/emqx.cmd"}, {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}, From a70545b64a1e497fbcd6badef66bdcb83815c5bb Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 30 Mar 2023 18:06:43 +0800 Subject: [PATCH 008/279] chore: split out exproto gateway --- apps/emqx_exproto/.gitignore | 24 +++ apps/emqx_exproto/LICENSE | 191 ++++++++++++++++++ .../src/exproto => emqx_exproto}/README.md | 0 .../i18n/emqx_exproto_schema.conf | 52 +++++ .../include/emqx_exproto.hrl | 0 .../priv}/protos/exproto.proto | 0 apps/emqx_exproto/rebar.config | 31 +++ apps/emqx_exproto/src/emqx_exproto.app.src | 10 + .../src/emqx_exproto.erl} | 49 ++--- .../src}/emqx_exproto_channel.erl | 3 +- .../src}/emqx_exproto_frame.erl | 0 .../src}/emqx_exproto_gcli.erl | 0 .../src}/emqx_exproto_gsvr.erl | 2 +- apps/emqx_exproto/src/emqx_exproto_schema.erl | 117 +++++++++++ .../test/emqx_exproto_SUITE.erl | 1 + .../test/emqx_exproto_echo_svr.erl | 0 apps/emqx_gateway/.gitignore | 6 +- apps/emqx_gateway/Makefile | 28 --- .../i18n/emqx_gateway_schema_i18n.conf | 51 ----- apps/emqx_gateway/rebar.config | 33 --- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway_app.erl | 9 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 112 ++-------- mix.exs | 1 + rebar.config.erl | 1 + 25 files changed, 471 insertions(+), 252 deletions(-) create mode 100644 apps/emqx_exproto/.gitignore create mode 100644 apps/emqx_exproto/LICENSE rename apps/{emqx_gateway/src/exproto => emqx_exproto}/README.md (100%) create mode 100644 apps/emqx_exproto/i18n/emqx_exproto_schema.conf rename apps/{emqx_gateway/src/exproto => emqx_exproto}/include/emqx_exproto.hrl (100%) rename apps/{emqx_gateway/src/exproto => emqx_exproto/priv}/protos/exproto.proto (100%) create mode 100644 apps/emqx_exproto/rebar.config create mode 100644 apps/emqx_exproto/src/emqx_exproto.app.src rename apps/{emqx_gateway/src/exproto/emqx_exproto_impl.erl => emqx_exproto/src/emqx_exproto.erl} (93%) rename apps/{emqx_gateway/src/exproto => emqx_exproto/src}/emqx_exproto_channel.erl (99%) rename apps/{emqx_gateway/src/exproto => emqx_exproto/src}/emqx_exproto_frame.erl (100%) rename apps/{emqx_gateway/src/exproto => emqx_exproto/src}/emqx_exproto_gcli.erl (100%) rename apps/{emqx_gateway/src/exproto => emqx_exproto/src}/emqx_exproto_gsvr.erl (99%) create mode 100644 apps/emqx_exproto/src/emqx_exproto_schema.erl rename apps/{emqx_gateway => emqx_exproto}/test/emqx_exproto_SUITE.erl (99%) rename apps/{emqx_gateway => emqx_exproto}/test/emqx_exproto_echo_svr.erl (100%) delete mode 100644 apps/emqx_gateway/Makefile diff --git a/apps/emqx_exproto/.gitignore b/apps/emqx_exproto/.gitignore new file mode 100644 index 000000000..922b0f989 --- /dev/null +++ b/apps/emqx_exproto/.gitignore @@ -0,0 +1,24 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ +src/emqx_exproto_pb.erl +src/emqx_exproto_v_1_connection_adapter_bhvr.erl +src/emqx_exproto_v_1_connection_adapter_client.erl +src/emqx_exproto_v_1_connection_handler_bhvr.erl +src/emqx_exproto_v_1_connection_handler_client.erl diff --git a/apps/emqx_exproto/LICENSE b/apps/emqx_exproto/LICENSE new file mode 100644 index 000000000..5a5418f0f --- /dev/null +++ b/apps/emqx_exproto/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/src/exproto/README.md b/apps/emqx_exproto/README.md similarity index 100% rename from apps/emqx_gateway/src/exproto/README.md rename to apps/emqx_exproto/README.md diff --git a/apps/emqx_exproto/i18n/emqx_exproto_schema.conf b/apps/emqx_exproto/i18n/emqx_exproto_schema.conf new file mode 100644 index 000000000..0c6fd2286 --- /dev/null +++ b/apps/emqx_exproto/i18n/emqx_exproto_schema.conf @@ -0,0 +1,52 @@ +emqx_exproto_schema { + exproto { + desc { + en: """The Extension Protocol configuration""" + zh: """ExProto 网关""" + } + } + + exproto_server { + desc { + en: """Configurations for starting the ConnectionAdapter service""" + zh: """配置 ExProto 网关需要启动的 ConnectionAdapter 服务。 +该服务用于提供客户端的认证、发布、订阅和数据下行等功能。""" + } + } + + exproto_grpc_server_bind { + desc { + en: """Listening address and port for the gRPC server.""" + zh: """服务监听地址和端口。""" + } + } + + exproto_grpc_server_ssl { + desc { + en: """SSL configuration for the gRPC server.""" + zh: """服务 SSL 配置。""" + } + } + + exproto_handler { + desc { + en: """Configurations for request to ConnectionHandler service""" + zh: """配置 ExProto 网关需要请求的 ConnectionHandler 服务地址。 +该服务用于给 ExProto 提供客户端的 Socket 事件处理、字节解码、订阅消息接收等功能。""" + } + } + + exproto_grpc_handler_address { + desc { + en: """gRPC server address.""" + zh: """对端 gRPC 服务器地址。""" + } + } + + exproto_grpc_handler_ssl { + desc { + en: """SSL configuration for the gRPC client.""" + zh: """gRPC 客户端的 SSL 配置。""" + } + } +} diff --git a/apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl b/apps/emqx_exproto/include/emqx_exproto.hrl similarity index 100% rename from apps/emqx_gateway/src/exproto/include/emqx_exproto.hrl rename to apps/emqx_exproto/include/emqx_exproto.hrl diff --git a/apps/emqx_gateway/src/exproto/protos/exproto.proto b/apps/emqx_exproto/priv/protos/exproto.proto similarity index 100% rename from apps/emqx_gateway/src/exproto/protos/exproto.proto rename to apps/emqx_exproto/priv/protos/exproto.proto diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config new file mode 100644 index 000000000..556404166 --- /dev/null +++ b/apps/emqx_exproto/rebar.config @@ -0,0 +1,31 @@ +{erl_opts, [debug_info]}. + +{plugins, [ + {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} +]}. + +{grpc, [ + {protos, ["priv/protos"]}, + {out_dir, "src"}, + {gpb_opts, [ + {module_name_prefix, "emqx_"}, + {module_name_suffix, "_pb"} + ]} +]}. + +{provider_hooks, [ + {pre, [ + {compile, {grpc, gen}}, + {clean, {grpc, clean}} + ]} +]}. + +{xref_ignores, [emqx_exproto_pb]}. + +{cover_excl_mods, [ + emqx_exproto_pb, + emqx_exproto_v_1_connection_adapter_client, + emqx_exproto_v_1_connection_adapter_bhvr, + emqx_exproto_v_1_connection_handler_client, + emqx_exproto_v_1_connection_handler_bhvr +]}. diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src new file mode 100644 index 000000000..0b4ac3966 --- /dev/null +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -0,0 +1,10 @@ +{application, emqx_exproto, [ + {description, "ExProto Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, emqx_gateway, grpc]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_exproto/src/emqx_exproto.erl similarity index 93% rename from apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl rename to apps/emqx_exproto/src/emqx_exproto.erl index 0c25e5e08..1e6e0e6de 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_exproto/src/emqx_exproto.erl @@ -14,12 +14,28 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% @doc The ExProto Gateway Implement interface --module(emqx_exproto_impl). - --behaviour(emqx_gateway_impl). +%% @doc The ExProto Gateway implement +-module(emqx_exproto). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). + +%% define a gateway named stomp +-gateway(#{ + name => exproto, + callback_module => ?MODULE, + config_schema_module => emqx_exproto_schema +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). -import( emqx_gateway_utils, @@ -30,31 +46,8 @@ ] ). -%% APIs --export([ - reg/0, - unreg/0 -]). - --export([ - on_gateway_load/2, - on_gateway_update/3, - on_gateway_unload/2 -]). - %%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -reg() -> - RegistryOptions = [{cbkmod, ?MODULE}], - emqx_gateway_registry:reg(exproto, RegistryOptions). - -unreg() -> - emqx_gateway_registry:unreg(exproto). - -%%-------------------------------------------------------------------- -%% emqx_gateway_registry callbacks +%% emqx_gateway_impl callbacks %%-------------------------------------------------------------------- on_gateway_load( diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl similarity index 99% rename from apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl rename to apps/emqx_exproto/src/emqx_exproto_channel.erl index 301154df0..7234e7a2f 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -15,7 +15,8 @@ %%-------------------------------------------------------------------- -module(emqx_exproto_channel). --include("src/exproto/include/emqx_exproto.hrl"). + +-include("emqx_exproto.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/types.hrl"). diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl b/apps/emqx_exproto/src/emqx_exproto_frame.erl similarity index 100% rename from apps/emqx_gateway/src/exproto/emqx_exproto_frame.erl rename to apps/emqx_exproto/src/emqx_exproto_frame.erl diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl b/apps/emqx_exproto/src/emqx_exproto_gcli.erl similarity index 100% rename from apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl rename to apps/emqx_exproto/src/emqx_exproto_gcli.erl diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl similarity index 99% rename from apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl rename to apps/emqx_exproto/src/emqx_exproto_gsvr.erl index 13bd49e55..5bbe7bf37 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl +++ b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl @@ -19,7 +19,7 @@ % -behaviour(emqx_exproto_v_1_connection_adapter_bhvr). --include("src/exproto/include/emqx_exproto.hrl"). +-include("emqx_exproto.hrl"). -include_lib("emqx/include/logger.hrl"). -define(IS_QOS(X), (X =:= 0 orelse X =:= 1 orelse X =:= 2)). diff --git a/apps/emqx_exproto/src/emqx_exproto_schema.erl b/apps/emqx_exproto/src/emqx_exproto_schema.erl new file mode 100644 index 000000000..eb44c030b --- /dev/null +++ b/apps/emqx_exproto/src/emqx_exproto_schema.erl @@ -0,0 +1,117 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_exproto_schema). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-type ip_port() :: tuple() | integer(). + +-typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). + +-reflect_type([ + ip_port/0 +]). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(exproto) -> + [ + {server, + sc( + ref(exproto_grpc_server), + #{ + required => true, + desc => ?DESC(exproto_server) + } + )}, + {handler, + sc( + ref(exproto_grpc_handler), + #{ + required => true, + desc => ?DESC(exproto_handler) + } + )}, + {mountpoint, emqx_gateway_schema:mountpoint()}, + {listeners, + sc(ref(emqx_gateway_schema, tcp_udp_listeners), #{desc => ?DESC(tcp_udp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(); +fields(exproto_grpc_server) -> + [ + {bind, + sc( + hoconsc:union([ip_port(), integer()]), + #{ + required => true, + desc => ?DESC(exproto_grpc_server_bind) + } + )}, + {ssl_options, + sc( + ref(ssl_server_opts), + #{ + required => {false, recursively}, + desc => ?DESC(exproto_grpc_server_ssl) + } + )} + ]; +fields(exproto_grpc_handler) -> + [ + {address, sc(binary(), #{required => true, desc => ?DESC(exproto_grpc_handler_address)})}, + {ssl_options, + sc( + ref(emqx_schema, "ssl_client_opts"), + #{ + required => {false, recursively}, + desc => ?DESC(exproto_grpc_handler_ssl) + } + )} + ]; +fields(ssl_server_opts) -> + emqx_schema:server_ssl_opts_schema( + #{ + depth => 10, + reuse_sessions => true, + versions => tls_all_available + }, + true + ). + +desc(exproto) -> + "Settings for EMQX extension protocol (exproto)."; +desc(exproto_grpc_server) -> + "Settings for the exproto gRPC server."; +desc(exproto_grpc_handler) -> + "Settings for the exproto gRPC connection handler."; +desc(ssl_server_opts) -> + "SSL configuration for the server."; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% helpers + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl similarity index 99% rename from apps/emqx_gateway/test/emqx_exproto_SUITE.erl rename to apps/emqx_exproto/test/emqx_exproto_SUITE.erl index b476a40cb..a8ce41f44 100644 --- a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl @@ -76,6 +76,7 @@ metrics() -> [tcp, ssl, udp, dtls]. init_per_group(GrpName, Cfg) -> + application:load(emqx_exproto), put(grpname, GrpName), Svrs = emqx_exproto_echo_svr:start(), emqx_common_test_helpers:start_apps([emqx_authn, emqx_gateway], fun set_special_cfg/1), diff --git a/apps/emqx_gateway/test/emqx_exproto_echo_svr.erl b/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl similarity index 100% rename from apps/emqx_gateway/test/emqx_exproto_echo_svr.erl rename to apps/emqx_exproto/test/emqx_exproto_echo_svr.erl diff --git a/apps/emqx_gateway/.gitignore b/apps/emqx_gateway/.gitignore index 5bff8a84d..a81bb07da 100644 --- a/apps/emqx_gateway/.gitignore +++ b/apps/emqx_gateway/.gitignore @@ -18,8 +18,4 @@ _build rebar3.crashdump *~ rebar.lock -src/exproto/emqx_exproto_pb.erl -src/exproto/emqx_exproto_v_1_connection_adapter_bhvr.erl -src/exproto/emqx_exproto_v_1_connection_adapter_client.erl -src/exproto/emqx_exproto_v_1_connection_handler_bhvr.erl -src/exproto/emqx_exproto_v_1_connection_handler_client.erl + diff --git a/apps/emqx_gateway/Makefile b/apps/emqx_gateway/Makefile deleted file mode 100644 index b2a54f7dd..000000000 --- a/apps/emqx_gateway/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -## shallow clone for speed - -REBAR_GIT_CLONE_OPTIONS += --depth 1 -export REBAR_GIT_CLONE_OPTIONS - -REBAR = rebar3 -all: compile - -compile: - $(REBAR) compile - -clean: distclean - -ct: - $(REBAR) as test ct -v - -eunit: - $(REBAR) as test eunit - -xref: - $(REBAR) xref - -cover: - $(REBAR) cover - -distclean: - @rm -rf _build - @rm -f data/app.*.config data/vm.*.args rebar.lock diff --git a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf index 561627241..1ffc5c6c1 100644 --- a/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf +++ b/apps/emqx_gateway/i18n/emqx_gateway_schema_i18n.conf @@ -1,56 +1,5 @@ emqx_gateway_schema { - exproto { - desc { - en: """The Extension Protocol configuration""" - zh: """ExProto 网关""" - } - } - - exproto_server { - desc { - en: """Configurations for starting the ConnectionAdapter service""" - zh: """配置 ExProto 网关需要启动的 ConnectionAdapter 服务。 -该服务用于提供客户端的认证、发布、订阅和数据下行等功能。""" - } - } - - exproto_grpc_server_bind { - desc { - en: """Listening address and port for the gRPC server.""" - zh: """服务监听地址和端口。""" - } - } - - exproto_grpc_server_ssl { - desc { - en: """SSL configuration for the gRPC server.""" - zh: """服务 SSL 配置。""" - } - } - - exproto_handler { - desc { - en: """Configurations for request to ConnectionHandler service""" - zh: """配置 ExProto 网关需要请求的 ConnectionHandler 服务地址。 -该服务用于给 ExProto 提供客户端的 Socket 事件处理、字节解码、订阅消息接收等功能。""" - } - } - - exproto_grpc_handler_address { - desc { - en: """gRPC server address.""" - zh: """对端 gRPC 服务器地址。""" - } - } - - exproto_grpc_handler_ssl { - desc { - en: """SSL configuration for the gRPC client.""" - zh: """gRPC 客户端的 SSL 配置。""" - } - } - gateway_common_enable { desc { en: """Whether to enable this gateway""" diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 272783758..7e5228a9e 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -1,38 +1,5 @@ %% -*- mode: erlang -*- - {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../emqx"}} ]}. - -{plugins, [ - {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} -]}. - -{grpc, [ - {protos, ["src/exproto/protos"]}, - {out_dir, "src/exproto/"}, - {gpb_opts, [ - {module_name_prefix, "emqx_"}, - {module_name_suffix, "_pb"} - ]} -]}. - -{provider_hooks, [ - {pre, [ - {compile, {grpc, gen}}, - {clean, {grpc, clean}} - ]} -]}. - -{xref_ignores, [emqx_exproto_pb]}. - -{cover_excl_mods, [ - emqx_exproto_pb, - emqx_exproto_v_1_connection_adapter_client, - emqx_exproto_v_1_connection_adapter_bhvr, - emqx_exproto_v_1_connection_handler_client, - emqx_exproto_v_1_connection_handler_bhvr -]}. - -{project_plugins, [erlfmt]}. diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index ced013497..850d38cdd 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -4,7 +4,7 @@ {vsn, "0.1.14"}, {registered, []}, {mod, {emqx_gateway_app, []}}, - {applications, [kernel, stdlib, grpc, emqx, emqx_authn, emqx_ctl]}, + {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 5999e85c9..01a1aaddd 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -41,18 +41,11 @@ stop(_State) -> %% Internal funcs load_default_gateway_applications() -> - BuiltInGateways = [ - #{ - name => exproto, - callback_module => emqx_exproto_impl, - config_schema_module => emqx_gateway_schema - } - ], lists:foreach( fun(Def) -> load_gateway_application(Def) end, - emqx_gateway_utils:find_gateway_definations() ++ BuiltInGateways + emqx_gateway_utils:find_gateway_definations() ). load_gateway_application( diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 6a4811b94..84e9ee7e4 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -60,79 +60,22 @@ namespace() -> gateway. tags() -> [<<"Gateway">>]. -roots() -> [gateway]. +roots() -> + [{gateway, sc(ref(?MODULE, gateway), #{importance => ?IMPORTANCE_HIDDEN})}]. fields(gateway) -> - [ - {exproto, - sc( - ref(exproto), - #{ - required => {false, recursively}, - desc => ?DESC(exproto) - } - )} - ] ++ gateway_schemas(); -fields(exproto) -> - [ - {server, - sc( - ref(exproto_grpc_server), - #{ - required => true, - desc => ?DESC(exproto_server) - } - )}, - {handler, - sc( - ref(exproto_grpc_handler), - #{ - required => true, - desc => ?DESC(exproto_handler) - } - )}, - {mountpoint, mountpoint()}, - {listeners, sc(ref(tcp_udp_listeners), #{desc => ?DESC(tcp_udp_listeners)})} - ] ++ gateway_common_options(); -fields(exproto_grpc_server) -> - [ - {bind, - sc( - hoconsc:union([ip_port(), integer()]), - #{ - required => true, - desc => ?DESC(exproto_grpc_server_bind) - } - )}, - {ssl_options, - sc( - ref(ssl_server_opts), - #{ - required => {false, recursively}, - desc => ?DESC(exproto_grpc_server_ssl) - } - )} - ]; -fields(exproto_grpc_handler) -> - [ - {address, sc(binary(), #{required => true, desc => ?DESC(exproto_grpc_handler_address)})}, - {ssl_options, - sc( - ref(emqx_schema, "ssl_client_opts"), - #{ - required => {false, recursively}, - desc => ?DESC(exproto_grpc_handler_ssl) - } - )} - ]; -fields(ssl_server_opts) -> - emqx_schema:server_ssl_opts_schema( - #{ - depth => 10, - reuse_sessions => true, - versions => tls_all_available - }, - true + lists:map( + fun(#{name := Name, config_schema_module := Mod}) -> + {Name, + sc( + ref(Mod, Name), + #{ + required => {false, recursively}, + desc => ?DESC(Name) + } + )} + end, + emqx_gateway_utils:find_gateway_definations() ); fields(clientinfo_override) -> [ @@ -217,14 +160,6 @@ fields(dtls_opts) -> desc(gateway) -> "EMQX Gateway configuration root."; -desc(exproto) -> - "Settings for EMQX extension protocol (exproto)."; -desc(exproto_grpc_server) -> - "Settings for the exproto gRPC server."; -desc(exproto_grpc_handler) -> - "Settings for the exproto gRPC connection handler."; -desc(ssl_server_opts) -> - "SSL configuration for the server."; desc(clientinfo_override) -> "ClientInfo override."; desc(udp_listeners) -> @@ -391,26 +326,11 @@ proxy_protocol_opts() -> %% dynamic schemas %% FIXME: don't hardcode the gateway names -gateway_schema(exproto) -> fields(exproto); gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn); gateway_schema(coap) -> emqx_coap_schema:fields(coap); -gateway_schema(lwm2m) -> emqx_lwm2m_schema:fields(lwm2m). - -gateway_schemas() -> - lists:map( - fun(#{name := Name, config_schema_module := Mod}) -> - {Name, - sc( - ref(Mod, Name), - #{ - required => {false, recursively}, - desc => ?DESC(Name) - } - )} - end, - emqx_gateway_utils:find_gateway_definations() - ). +gateway_schema(lwm2m) -> emqx_lwm2m_schema:fields(lwm2m); +gateway_schema(exproto) -> emqx_exproto_schema:fields(exproto). %%-------------------------------------------------------------------- %% helpers diff --git a/mix.exs b/mix.exs index 264d4f87e..15b6b997b 100644 --- a/mix.exs +++ b/mix.exs @@ -285,6 +285,7 @@ defmodule EMQXUmbrella.MixProject do emqx_mqttsn: :permanent, emqx_coap: :permanent, emqx_lwm2m: :permanent, + emqx_exproto: :permanent, emqx_exhook: :permanent, emqx_bridge: :permanent, emqx_rule_engine: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index 978f5ec87..d1f26fda2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -393,6 +393,7 @@ relx_apps(ReleaseType, Edition) -> emqx_mqttsn, emqx_coap, emqx_lwm2m, + emqx_exproto, emqx_exhook, emqx_bridge, emqx_rule_engine, From b24ff9bc6e76bdbbe4518aae39b81c956515ae57 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 1 Apr 2023 10:30:37 +0800 Subject: [PATCH 009/279] test(gateway): refine all test cases --- apps/emqx_coap/src/emqx_coap_api.erl | 3 ++ apps/emqx_coap/src/emqx_coap_frame.erl | 2 ++ apps/emqx_coap/src/emqx_coap_tm.erl | 8 +++-- apps/emqx_coap/src/emqx_coap_transport.erl | 6 ++++ apps/emqx_gateway/src/emqx_gateway_utils.erl | 2 +- apps/emqx_gateway/test/emqx_gateway_SUITE.erl | 20 +++++++++---- .../test/emqx_gateway_api_SUITE.erl | 10 ++++++- .../test/emqx_gateway_authz_SUITE.erl | 3 +- .../test/emqx_gateway_cli_SUITE.erl | 11 +++---- .../test/emqx_gateway_conf_SUITE.erl | 1 + .../test/emqx_gateway_test_utils.erl | 6 ++++ apps/emqx_lwm2m/src/emqx_lwm2m_api.erl | 2 ++ apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl | 6 ++-- apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl | 4 +-- apps/emqx_lwm2m/src/emqx_lwm2m_message.erl | 8 +++-- apps/emqx_lwm2m/src/emqx_lwm2m_session.erl | 10 +++---- apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl | 18 ++++++++---- .../src/emqx_lwm2m_xml_object_db.erl | 8 +++-- apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl | 11 ++++++- apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl | 29 +++---------------- apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl | 16 +++++----- apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl | 6 ++-- apps/emqx_stomp/src/emqx_stomp_heartbeat.erl | 2 ++ 23 files changed, 117 insertions(+), 75 deletions(-) diff --git a/apps/emqx_coap/src/emqx_coap_api.erl b/apps/emqx_coap/src/emqx_coap_api.erl index 50ea9829a..b4fce5473 100644 --- a/apps/emqx_coap/src/emqx_coap_api.erl +++ b/apps/emqx_coap/src/emqx_coap_api.erl @@ -34,9 +34,12 @@ -import(hoconsc, [mk/2, enum/1]). -import(emqx_dashboard_swagger, [error_codes/2]). +-elvis([{elvis_style, atom_naming_convention, disable}]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- + namespace() -> "gateway_coap". api_spec() -> diff --git a/apps/emqx_coap/src/emqx_coap_frame.erl b/apps/emqx_coap/src/emqx_coap_frame.erl index a05116b14..535d07a94 100644 --- a/apps/emqx_coap/src/emqx_coap_frame.erl +++ b/apps/emqx_coap/src/emqx_coap_frame.erl @@ -55,6 +55,8 @@ -define(OPTION_PROXY_SCHEME, 39). -define(OPTION_SIZE1, 60). +-elvis([{elvis_style, no_if_expression, disable}]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- diff --git a/apps/emqx_coap/src/emqx_coap_tm.erl b/apps/emqx_coap/src/emqx_coap_tm.erl index 297cbca6b..82f616b25 100644 --- a/apps/emqx_coap/src/emqx_coap_tm.erl +++ b/apps/emqx_coap/src/emqx_coap_tm.erl @@ -80,6 +80,8 @@ -import(emqx_coap_medium, [empty/0, iter/4, reset/1, proto_out/2]). +-elvis([{elvis_style, no_if_expression, disable}]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -401,9 +403,9 @@ alloc_message_id(MsgId, TM) -> next_message_id(MsgId) -> Next = MsgId + 1, - if - Next >= ?MAX_MESSAGE_ID -> - 1; + case Next >= ?MAX_MESSAGE_ID of true -> + 1; + false -> Next end. diff --git a/apps/emqx_coap/src/emqx_coap_transport.erl b/apps/emqx_coap/src/emqx_coap_transport.erl index c58a8abbd..1948c969d 100644 --- a/apps/emqx_coap/src/emqx_coap_transport.erl +++ b/apps/emqx_coap/src/emqx_coap_transport.erl @@ -60,6 +60,12 @@ reply/2 ]). +-elvis([{elvis_style, atom_naming_convention, disable}]). +-elvis([{elvis_style, no_if_expression, disable}]). + +%%-------------------------------------------------------------------- +%% APIs + -spec new() -> transport(). new() -> new(undefined). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 9d71263f8..9d80de00e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -601,7 +601,7 @@ find_attrs(App, Def) -> module_attributes(Module) -> try - Module:module_info(attributes) + apply(Module, module_info, [attributes]) catch error:undef -> [] end. diff --git a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl index f611988a0..5120e096e 100644 --- a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl @@ -33,6 +33,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([emqx_authn, emqx_gateway]), Conf. @@ -67,11 +68,11 @@ end_per_testcase(_TestCase, _Config) -> t_registered_gateway(_) -> [ - {coap, #{cbkmod := emqx_coap_impl}}, - {exproto, #{cbkmod := emqx_exproto_impl}}, - {lwm2m, #{cbkmod := emqx_lwm2m_impl}}, - {mqttsn, #{cbkmod := emqx_sn_impl}}, - {stomp, #{cbkmod := emqx_stomp_impl}} + {coap, #{cbkmod := emqx_coap}}, + {exproto, #{cbkmod := emqx_exproto}}, + {lwm2m, #{cbkmod := emqx_lwm2m}}, + {mqttsn, #{cbkmod := emqx_mqttsn}}, + {stomp, #{cbkmod := emqx_stomp}} ] = emqx_gateway:registered_gateway(). t_load_unload_list_lookup(_) -> @@ -187,7 +188,14 @@ read_lwm2m_conf(DataDir) -> Conf. setup_fake_usage_data(Lwm2mDataDir) -> - XmlDir = emqx_common_test_helpers:deps_path(emqx_gateway, "src/lwm2m/lwm2m_xml"), + XmlDir = filename:join( + [ + emqx_common_test_helpers:proj_root(), + "apps", + "emqx_lwm2m", + "lwm2m_xml" + ] + ), Lwm2mConf = read_lwm2m_conf(Lwm2mDataDir), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, Lwm2mConf), emqx_config:put([gateway, lwm2m, xml_dir], XmlDir), diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 7aac45d61..bfcebd772 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -214,9 +214,17 @@ t_gateway_coap(_) -> t_gateway_lwm2m(_) -> {200, Gw} = request(get, "/gateways/lwm2m"), assert_gw_unloaded(Gw), + XmlDir = filename:join( + [ + emqx_common_test_helpers:proj_root(), + "apps", + "emqx_lwm2m", + "lwm2m_xml" + ] + ), GwConf = #{ name => <<"lwm2m">>, - xml_dir => <<"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml">>, + xml_dir => list_to_binary(XmlDir), lifetime_min => <<"1s">>, lifetime_max => <<"1000s">>, qmode_time_window => <<"30s">>, diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index 2e44415aa..9bbcf2711 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -66,6 +66,7 @@ end_per_group(AuthName, Conf) -> init_per_suite(Config) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), init_gateway_conf(), meck:new(emqx_authz_file, [non_strict, passthrough, no_history, no_link]), meck:expect(emqx_authz_file, create, fun(S) -> S end), @@ -225,7 +226,7 @@ t_case_sn_subscribe(_) -> ) end, Sub(<<"/subscribe">>, fun(Data) -> - {ok, Msg, _, _} = emqx_sn_frame:parse(Data, undefined), + {ok, Msg, _, _} = emqx_mqttsn_frame:parse(Data, undefined), ?assertMatch({mqtt_sn_message, _, {_, 3, 0, Payload}}, Msg) end), Sub(<<"/badsubscribe">>, fun(Data) -> diff --git a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl index c66785e00..a234dd126 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl @@ -62,6 +62,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), Conf. @@ -116,11 +117,11 @@ t_gateway_registry_usage(_) -> t_gateway_registry_list(_) -> emqx_gateway_cli:'gateway-registry'(["list"]), ?assertEqual( - "Registered Name: coap, Callback Module: emqx_coap_impl\n" - "Registered Name: exproto, Callback Module: emqx_exproto_impl\n" - "Registered Name: lwm2m, Callback Module: emqx_lwm2m_impl\n" - "Registered Name: mqttsn, Callback Module: emqx_sn_impl\n" - "Registered Name: stomp, Callback Module: emqx_stomp_impl\n", + "Registered Name: coap, Callback Module: emqx_coap\n" + "Registered Name: exproto, Callback Module: emqx_exproto\n" + "Registered Name: lwm2m, Callback Module: emqx_lwm2m\n" + "Registered Name: mqttsn, Callback Module: emqx_mqttsn\n" + "Registered Name: stomp, Callback Module: emqx_stomp\n", acc_print() ). diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index 6f6c2c45a..33c307770 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -37,6 +37,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, <<"gateway {}">>), emqx_common_test_helpers:start_apps([emqx_conf, emqx_authn, emqx_gateway]), Conf. diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index deb602bc7..56a2fe7f9 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -101,6 +101,12 @@ assert_fields_exist(Ks, Map) -> end, Ks ). +load_all_gateway_apps() -> + application:load(emqx_stomp), + application:load(emqx_mqttsn), + application:load(emqx_coap), + application:load(emqx_lwm2m), + application:load(emqx_exproto). %%-------------------------------------------------------------------- %% http diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl index 2cd53d6eb..80afadb8e 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl @@ -32,6 +32,8 @@ -import(hoconsc, [mk/2, ref/1, ref/2]). -import(emqx_dashboard_swagger, [error_codes/2]). +-elvis([{elvis_style, atom_naming_convention, disable}]). + namespace() -> "lwm2m". api_spec() -> diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl index 54e5723cd..276b4f19d 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl @@ -464,14 +464,14 @@ check_lwm2m_version( _ -> false end, - if - IsValid -> + case IsValid of + true -> NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond), proto_ver => Ver }, {ok, Channel#channel{conninfo = NConnInfo}}; - true -> + _ -> ?SLOG(error, #{ msg => "reject_REGISTRE_request", reason => {unsupported_version, Ver} diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl index 9b1f6b65d..9ef3fb10d 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl @@ -292,9 +292,9 @@ make_response(Code, Ref = #{}) -> BaseRsp = make_base_response(Ref), make_data_response(BaseRsp, Code). -make_response(Code, Ref = #{}, _Format, Result) -> +make_response(Code, Ref = #{}, Format, Result) -> BaseRsp = make_base_response(Ref), - make_data_response(BaseRsp, Code, _Format, Result). + make_data_response(BaseRsp, Code, Format, Result). %% The base response format is what included in the request: %% diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl index e541e83f1..90a0306b7 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl @@ -412,9 +412,11 @@ byte_size_of_signed(UInt) -> byte_size_of_signed(UInt, N) -> BitSize = (8 * N - 1), Max = (1 bsl BitSize), - if - UInt =< Max -> N; - UInt > Max -> byte_size_of_signed(UInt, N + 1) + case UInt =< Max of + true -> + N; + false -> + byte_size_of_signed(UInt, N + 1) end. binary_to_number(NumStr) -> diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl index 67543a910..6c8b419ee 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl @@ -379,8 +379,8 @@ is_alternate_path(LinkAttrs) -> true; [AttrKey, _] when AttrKey =/= <<>> -> false; - _BadAttr -> - throw({bad_attr, _BadAttr}) + BadAttr -> + throw({bad_attr, BadAttr}) end end, LinkAttrs @@ -679,10 +679,10 @@ send_to_coap(#session{queue = Queue} = Session) -> case queue:out(Queue) of {{value, {Timestamp, Ctx, Req}}, Q2} -> Now = ?NOW, - if - Timestamp =:= 0 orelse Timestamp > Now -> - send_to_coap(Ctx, Req, Session#session{queue = Q2}); + case Timestamp =:= 0 orelse Timestamp > Now of true -> + send_to_coap(Ctx, Req, Session#session{queue = Q2}); + false -> send_to_coap(Session#session{queue = Q2}) end; {empty, _} -> diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl index 2f53573c4..314666638 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl @@ -37,13 +37,18 @@ -define(TLV_LEGNTH_16_BIT, 2). -define(TLV_LEGNTH_24_BIT, 3). -%---------------------------------------------------------------------------------------------------------------------------------------- -% [#{tlv_object_instance := Id11, value := Value11}, #{tlv_object_instance := Id12, value := Value12}, ...] +-elvis([{elvis_style, no_if_expression, disable}]). + +%%-------------------------------------------------------------------- +% [#{tlv_object_instance := Id11, value := Value11}, +% #{tlv_object_instance := Id12, value := Value12}, ...] % where Value11 and Value12 is a list: -% [#{tlv_resource_with_value => Id21, value => Value21}, #{tlv_multiple_resource => Id22, value = Value22}, ...] +% [#{tlv_resource_with_value => Id21, value => Value21}, +% #{tlv_multiple_resource => Id22, value = Value22}, ...] % where Value21 is a binary % Value22 is a list: -% [#{tlv_resource_instance => Id31, value => Value31}, #{tlv_resource_instance => Id32, value => Value32}, ...] +% [#{tlv_resource_instance => Id31, value => Value31}, +% #{tlv_resource_instance => Id32, value => Value32}, ...] % where Value31 and Value32 is a binary % % correspond to three levels: @@ -51,8 +56,9 @@ % 2) Resource Level % 3) Resource Instance Level % -% NOTE: TLV does not has object level, only has object instance level. It implies TLV can not represent multiple objects -%---------------------------------------------------------------------------------------------------------------------------------------- +% NOTE: TLV does not has object level, only has object instance level. +% It implies TLV can not represent multiple objects +%%-------------------------------------------------------------------- parse(Data) -> parse_loop(Data, []). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl index 04c4c1af2..2908a65e0 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl @@ -45,6 +45,8 @@ -record(state, {}). +-elvis([{elvis_style, atom_naming_convention, disable}]). + %% ------------------------------------------------------------------ %% API Function Definitions %% ------------------------------------------------------------------ @@ -124,10 +126,10 @@ code_change(_OldVsn, State, _Extra) -> load(BaseDir) -> Wild = filename:join(BaseDir, "*.xml"), Wild2 = - if - is_binary(Wild) -> - erlang:binary_to_list(Wild); + case is_binary(Wild) of true -> + erlang:binary_to_list(Wild); + false -> Wild end, case filelib:wildcard(Wild2) of diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl index 9abe16a35..dd2e3bbfd 100644 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -177,11 +177,19 @@ default_config() -> default_config(#{}). default_config(Overrides) -> + XmlDir = filename:join( + [ + emqx_common_test_helpers:proj_root(), + "apps", + "emqx_lwm2m", + "lwm2m_xml" + ] + ), iolist_to_binary( io_lib:format( "\n" "gateway.lwm2m {\n" - " xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"\n" + " xml_dir = \"~s\"\n" " lifetime_min = 1s\n" " lifetime_max = 86400s\n" " qmode_time_window = 22\n" @@ -200,6 +208,7 @@ default_config(Overrides) -> " }\n" "}\n", [ + XmlDir, maps:get(auto_observe, Overrides, false), maps:get(bind, Overrides, ?PORT) ] diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl index bfeeb2c9b..a1d048d76 100644 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl @@ -28,29 +28,6 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --define(CONF_DEFAULT, << - "\n" - "gateway.lwm2m {\n" - " xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\"\n" - " lifetime_min = 100s\n" - " lifetime_max = 86400s\n" - " qmode_time_window = 200\n" - " auto_observe = false\n" - " mountpoint = \"lwm2m/${username}\"\n" - " update_msg_publish_condition = contains_object_list\n" - " translators {\n" - " command = {topic = \"/dn/#\", qos = 0}\n" - " response = {topic = \"/up/resp\", qos = 0}\n" - " notify = {topic = \"/up/notify\", qos = 0}\n" - " register = {topic = \"/up/resp\", qos = 0}\n" - " update = {topic = \"/up/resp\", qos = 0}\n" - " }\n" - " listeners.udp.default {\n" - " bind = 5783\n" - " }\n" - "}\n" ->>). - -define(assertExists(Map, Key), ?assertNotEqual(maps:get(Key, Map, undefined), undefined) ). @@ -83,7 +60,8 @@ all() -> init_per_suite(Config) -> application:load(emqx_gateway), application:load(emqx_lwm2m), - ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), + DefaultConfig = emqx_lwm2m_SUITE:default_config(), + ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, DefaultConfig), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]), Config. @@ -94,7 +72,8 @@ end_per_suite(Config) -> Config. init_per_testcase(_AllTestCase, Config) -> - ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), + DefaultConfig = emqx_lwm2m_SUITE:default_config(), + ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, DefaultConfig), {ok, _} = application:ensure_all_started(emqx_gateway), {ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]), diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl index bf0fc52a4..3be2f1dc2 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl @@ -58,10 +58,10 @@ serialize_opts() -> %% Parse MQTT-SN Message %%-------------------------------------------------------------------- -parse(<<16#01:?byte, Len:?short, Type:?byte, Var/binary>>, _State) -> - {ok, parse(Type, Len - 4, Var), <<>>, _State}; -parse(<>, _State) -> - {ok, parse(Type, Len - 2, Var), <<>>, _State}. +parse(<<16#01:?byte, Len:?short, Type:?byte, Var/binary>>, State) -> + {ok, parse(Type, Len - 4, Var), <<>>, State}; +parse(<>, State) -> + {ok, parse(Type, Len - 2, Var), <<>>, State}. parse(Type, Len, Var) when Len =:= size(Var) -> #mqtt_sn_message{type = Type, variable = parse_var(Type, Var)}; @@ -160,9 +160,11 @@ parse_topic(2#11, Topic) -> Topic. serialize_pkt(#mqtt_sn_message{type = Type, variable = Var}, Opts) -> VarBin = serialize(Type, Var, Opts), VarLen = size(VarBin), - if - VarLen < 254 -> <<(VarLen + 2), Type, VarBin/binary>>; - true -> <<16#01, (VarLen + 4):?short, Type, VarBin/binary>> + case VarLen < 254 of + true -> + <<(VarLen + 2), Type, VarBin/binary>>; + false -> + <<16#01, (VarLen + 4):?short, Type, VarBin/binary>> end. serialize(?SN_ADVERTISE, {GwId, Duration}, _Opts) -> diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl b/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl index 07da8c351..9db355a9b 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl +++ b/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl @@ -151,9 +151,9 @@ init([InstaId, PredefTopics]) -> key = {predef, TopicName}, value = TopicId }), - if - TopicId > AccId -> TopicId; - true -> AccId + case TopicId > AccId of + true -> TopicId; + false -> AccId end end, 0, diff --git a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl b/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl index f5ed99623..2e4239bdc 100644 --- a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl +++ b/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl @@ -36,6 +36,8 @@ outgoing => #heartbeater{} }. +-elvis([{elvis_style, no_if_expression, disable}]). + %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- From 9577beaa4eeabef347119bdd7a55d1e42c72c619 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 1 Apr 2023 10:34:09 +0800 Subject: [PATCH 010/279] chore: update rebar.conf in emqx_exproto --- apps/emqx_exproto/rebar.config | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index 556404166..d21a7ece3 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -1,4 +1,5 @@ {erl_opts, [debug_info]}. +{deps, []}. {plugins, [ {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} From 3a3879f99f43b2dfada7512876382a3231f554a5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 1 Apr 2023 11:22:01 +0800 Subject: [PATCH 011/279] chore: update gateways deps --- apps/emqx_coap/rebar.config | 4 +++- apps/emqx_coap/src/emqx_coap.app.src | 2 +- apps/emqx_exproto/rebar.config | 4 +++- apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- apps/emqx_lwm2m/rebar.config | 4 +++- apps/emqx_lwm2m/src/emqx_lwm2m.app.src | 2 +- apps/emqx_mqttsn/rebar.config | 4 +++- apps/emqx_mqttsn/src/emqx_mqttsn.app.src | 2 +- apps/emqx_stomp/rebar.config | 4 +++- apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- mix.exs | 5 +++++ 11 files changed, 25 insertions(+), 10 deletions(-) diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_coap/rebar.config index 2656fd554..c8675c3ba 100644 --- a/apps/emqx_coap/rebar.config +++ b/apps/emqx_coap/rebar.config @@ -1,2 +1,4 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_coap/src/emqx_coap.app.src index 50b593ac7..c0f3f23da 100644 --- a/apps/emqx_coap/src/emqx_coap.app.src +++ b/apps/emqx_coap/src/emqx_coap.app.src @@ -2,7 +2,7 @@ {description, "CoAP Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx_gateway]}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index d21a7ece3..928949c69 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -1,5 +1,7 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. {plugins, [ {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 0b4ac3966..aa586a4fd 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -2,7 +2,7 @@ {description, "ExProto Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx_gateway, grpc]}, + {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config index 2656fd554..c8675c3ba 100644 --- a/apps/emqx_lwm2m/rebar.config +++ b/apps/emqx_lwm2m/rebar.config @@ -1,2 +1,4 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src index 08c3dbe3f..6338fa9d3 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -2,7 +2,7 @@ {description, "LwM2M Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx_gateway, emqx_coap]}, + {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_coap]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_mqttsn/rebar.config b/apps/emqx_mqttsn/rebar.config index 2656fd554..c8675c3ba 100644 --- a/apps/emqx_mqttsn/rebar.config +++ b/apps/emqx_mqttsn/rebar.config @@ -1,2 +1,4 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src index 76acc648e..55e18e800 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src +++ b/apps/emqx_mqttsn/src/emqx_mqttsn.app.src @@ -2,7 +2,7 @@ {description, "MQTT-SN Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx_gateway]}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_stomp/rebar.config b/apps/emqx_stomp/rebar.config index 2656fd554..c8675c3ba 100644 --- a/apps/emqx_stomp/rebar.config +++ b/apps/emqx_stomp/rebar.config @@ -1,2 +1,4 @@ {erl_opts, [debug_info]}. -{deps, []}. +{deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} + ]}. diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index cd9670056..e118f8370 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -2,7 +2,7 @@ {description, "Stomp Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx_gateway]}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/mix.exs b/mix.exs index 15b6b997b..981755b01 100644 --- a/mix.exs +++ b/mix.exs @@ -221,6 +221,11 @@ defmodule EMQXUmbrella.MixProject do applications: applications(edition_type), skip_mode_validation_for: [ :emqx_gateway, + :emqx_stomp, + :emqx_mqttsn, + :emqx_coap, + :emqx_lwm2m, + :emqx_exproto, :emqx_dashboard, :emqx_resource, :emqx_connector, From b2d018f2490125b959459111e026a57f8e90afda Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 1 Apr 2023 18:51:39 +0800 Subject: [PATCH 012/279] chore: fix test cases --- apps/emqx_modules/test/emqx_telemetry_SUITE.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl index cee255e77..a61781e13 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -45,6 +45,7 @@ init_per_suite(Config) -> ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF, #{ raw_with_default => true }), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:start_apps( [emqx_conf, emqx_authn, emqx_authz, emqx_modules], fun set_special_configs/1 From 5138e6371c2979c6d79a9eb46d358367b24e3b3e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 3 Apr 2023 09:47:05 +0800 Subject: [PATCH 013/279] chore: update changes --- changes/ce/feat-10278.en.md | 1 + changes/ce/feat-10278.zh.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/ce/feat-10278.en.md create mode 100644 changes/ce/feat-10278.zh.md diff --git a/changes/ce/feat-10278.en.md b/changes/ce/feat-10278.en.md new file mode 100644 index 000000000..d029c1420 --- /dev/null +++ b/changes/ce/feat-10278.en.md @@ -0,0 +1 @@ +Refactor the directory structure of all gateways. diff --git a/changes/ce/feat-10278.zh.md b/changes/ce/feat-10278.zh.md new file mode 100644 index 000000000..d2e738ec1 --- /dev/null +++ b/changes/ce/feat-10278.zh.md @@ -0,0 +1 @@ +重构所有网关的源码目录结构。 From 205e97fdca07b0d9446b1cd9342425c4f6e0aa78 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 3 Apr 2023 14:30:41 +0800 Subject: [PATCH 014/279] chore(gw): update README files --- apps/emqx_coap/README.md | 452 ++---------------------------------- apps/emqx_gateway/README.md | 346 +++------------------------ apps/emqx_mqttsn/README.md | 118 ++-------- apps/emqx_stomp/README.md | 84 ++----- 4 files changed, 98 insertions(+), 902 deletions(-) diff --git a/apps/emqx_coap/README.md b/apps/emqx_coap/README.md index 045db529d..405366e89 100644 --- a/apps/emqx_coap/README.md +++ b/apps/emqx_coap/README.md @@ -1,443 +1,31 @@ +# emqx_coap -# Table of Contents +The CoAP gateway implements publish, subscribe, and receive messages as standard +with [Publish-Subscribe Broker for the CoAP](https://datatracker.ietf.org/doc/html/draft-ietf-core-coap-pubsub-09). -1. [EMQX 5.0 CoAP Gateway](#org61e5bb8) - 1. [Features](#orgeddbc94) - 1. [PubSub Handler](#orgfc7be2d) - 2. [MQTT Handler](#org55be508) - 3. [Heartbeat](#org3d1a32e) - 4. [Query String](#org9a6b996) - 2. [Implementation](#org9985dfe) - 1. [Request/Response flow](#orge94210c) - 3. [Example](#ref_example) +## Quick Start +In EMQX 5.0, CoAP gateways can be configured and enabled through the Dashboard. +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: - +```properties +gateway.coap { -# EMQX 5.0 CoAP Gateway + mountpoint = "coap/" -emqx-coap is a CoAP Gateway for EMQX. It translates CoAP messages into MQTT messages and make it possible to communiate between CoAP clients and MQTT clients. + connection_required = false - - - -## Features - -- Partially achieves [Publish-Subscribe Broker for the Constrained Application Protocol (CoAP)](https://datatracker.ietf.org/doc/html/draft-ietf-core-coap-pubsub-09) - we called this as ps handler, include following functions: - - Publish - - Subscribe - - UnSubscribe -- Long connection and authorization verification called as MQTT handler - - - - -### PubSub Handler - -1. Publish - - Method: POST\ - URI Schema: ps/{+topic}{?q\*}\ - q\*: [Shared Options](#orgc50043b)\ - Response: - - - 2.04 "Changed" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" when with wrong auth uri query - -2. Subscribe - - Method: GET - Options: - - - Observer = 0 - - URI Schema: ps/{+topic}{?q\*}\ - q\*: see [Shared Options](#orgc50043b)\ - Response: - - - 2.05 "Content" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" when with wrong auth uri query - -``` - Client1 Client2 Broker - | | Subscribe | - | | ----- GET /ps/topic1 Observe:0 Token:XX ----> | - | | | - | | <---------- 2.05 Content Observe:10---------- | - | | | - | | | - | | Publish | - | ---------|----------- PUT /ps/topic1 "1033.3" --------> | - | | Notify | - | | <---------- 2.05 Content Observe:11 --------- | - | | | + listeners.udp.default { + bind = "5683" + max_connections = 1024000 + max_conn_rate = 1000 + } +} ``` -3. UnSubscribe - - Method : GET - Options: - - - Observe = 1 - - URI Schema: ps/{+topic}{?q\*}\ - q\*: see [Shared Options](#orgc50043b)\ - Response: - - - 2.07 "No Content" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" when with wrong auth uri query - - - - -### MQTT Handler - - Establishing a connection is optional. If the CoAP client needs to use connection-based operations, it must first establish a connection. -At the same time, the connectionless mode and the connected mode cannot be mixed. -In connection mode, the Publish/Subscribe/UnSubscribe sent by the client must be has Token and ClientId in query string. -If the Token and Clientid is wrong/miss, EMQX will reset the request. -The communication token is the data carried in the response payload after the client successfully establishes a connection. -After obtaining the token, the client's subsequent request must attach "token=Token" to the Query String -ClientId is necessary when there is a connection, and is a unique identifier defined by the client. -The server manages the client through the ClientId. If the ClientId is wrong, EMQX will reset the request. - -1. Create a Connection - - Method: POST - URI Schema: mqtt/connection{?q\*} - q\*: - - - clientid := client uid - - username - - password - - Response: - - - 2.01 "Created" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" wrong username or password - - Payload: Token if success - -2. Close a Connection - - Method : DELETE - URI Schema: mqtt/connection{?q\*} - q\*: - - - clientid := client uid - - token - - Response: - - - 2.01 "Deleted" when success - - 4.00 "Bad Request" when error - - 4.01 "Unauthorized" wrong clientid or token - - - - -### Heartbeat - -The Coap client can maintain the "connection" with the server through the heartbeat, -regardless of whether it is authenticated or not, -so that the server will not release related resources -Method : PUT -URI Schema: mqtt/connection{?q\*} -q\*: - -- clientid if authenticated -- token if authenticated - -Response: - -- 2.01 "Changed" when success -- 4.00 "Bad Request" when error -- 4.01 "Unauthorized" wrong clientid or token - - - - -### Query String - -CoAP gateway uses some options in query string to conversion between MQTT CoAP. - -1. Shared Options - - - clientid - - token - -2. Connect Options - - - username - - password - -3. Publish - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionTypeDefault
retainbooleanfalse
qosMQTT QosSee here
expiryMessage Expiry Interval0(Never expiry)
- -4. Subscribe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionTypeDefault
qosMQTT QosSee here
nlMQTT Subscribe No Local0
rhMQTT Subscribe Retain Handing0
- -5. MQTT Qos <=> CoAP non/con - - 1.notif_type - Control the type of notify messages when the observed object has changed.Can be: - - - non - - con - - qos - in this value, MQTT Qos0 -> non, Qos1/Qos2 -> con - - 2.subscribe_qos - Control the qos of subscribe.Can be: - - - qos0 - - qos1 - - qos2 - - coap - in this value, CoAP non -> qos0, con -> qos1 - - 3.publish_qos - like subscribe_qos, but control the qos of the publish MQTT message - - - - -## Implementation - - - - -### Request/Response flow - -![img](./doc/flow.png) - -1. Authorization check - - Check whether the clientid and token in the query string match the current connection - -2. Session - - Manager the "Transport Manager" "Observe Resources Manager" and next message id - -3. Transport Mnager - - Manager "Transport" create/close/dispatch - -4. Observe resources Mnager - - Mnager observe topic and token - -5. Transport - - ![img](./doc/transport.png) - - 1. Shared State - - ![img](./doc/shared_state.png) - -6. Handler - - 1. pubsub - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MethodObserveAction
GET0subscribe and reply result
GET1unsubscribe and reply result
POSTXpublish and reply result
- - 2. mqtt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MethodAction
PUTreply result
POSTreturn create connection action
DELETEreturn close connection action
- - - -## Example -1. Create Connection -``` -coap-client -m post -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&username=admin&password=public" -``` -Server will return token **X** in payload - -2. Update Connection -``` -coap-client -m put -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&token=X" -``` - -3. Publish -``` -coap-client -m post -e "Hellow" "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&username=admin&password=public" -``` -if you want to publish with auth, you must first establish a connection, and then post publish request on the same socket, so libcoap client can't simulation publish with a token - -``` -coap-client -m post -e "Hellow" "coap://127.0.0.1/ps/coap/test?clientid=123&token=X" -``` - -4. Subscribe -``` -coap-client -m get -s 60 -O 6,0x00 -o - -T "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&username=admin&password=public" -``` -**Or** - -``` -coap-client -m get -s 60 -O 6,0x00 -o - -T "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&token=X" -``` -5. Close Connection -``` -coap-client -m delete -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&token=X -``` +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. +More documentations: [CoAP Gateway](https://www.emqx.io/docs/en/v5.0/gateway/coap.html) diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index be8f6cb35..57e8febab 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -1,332 +1,58 @@ # emqx_gateway -EMQX Gateway +EMQX Gateway is an application that managing all gateways in EMQX. -## Concept +It provides a set of standards to define how to implement a certain type of +protocol access on EMQX. For example: - EMQX Gateway Management - - Gateway-Registry (or Gateway Type) - - *Load - - *UnLoad - - *List +- Frame parsing +- Access authentication +- Publish and subscribe +- Configuration & Schema +- HTTP/CLI management interfaces - - Gateway - - *Create - - *Delete - - *Update - - *Stop-And-Start - - *Hot-Upgrade - - *Satrt/Enable - - *Stop/Disable - - Listener +There are some standard implementations available, such as [Stomp](../emqx_stomp/README.md), +[MQTT-SN](../emqx_mqttsn/README.md), [CoAP](../emqx_coap/README.md), +and [LwM2M](../emqx_lwm2m/README.md) gateway. -## ROADMAP +The emqx_gateway application depends on `emqx`, `emqx_authn`, `emqx_ctl` that +provide the foundation for protocol access. -Gateway v0.1: "Basic Functionals" - - Management support - - Conn/Frame/Protocol Template - - Support Stomp/MQTT-SN/CoAP/LwM2M/ExProto +## Three ways to create your gateway -Gateway v0.2: "Integration & Friendly Management" - - Hooks & Metrics & Statistic - - HTTP APIs - - Management in the cluster - - Integrate with AuthN - - Integrate with `emqx_config` - - Improve hocon config - - Mountpoint & ClientInfo's Metadata - - The Concept Review +## Raw Erlang Application -Gateway v0.3: "Fault tolerance and high availability" - - A common session modoule for message delivery policy - - The restart mechanism for gateway-instance - - Consistency of cluster state - - Configuration hot update +This approach is the same as in EMQX 4.x. You need to implement an Erlang application, +which is packaged in EMQX as a [Plugin](todo) or as a source code dependency. +In this approach, you do not need to respect any specifications of emqx_gateway, +and you can freely implement the features you need. -Gateway v1.0: "Best practices for each type of protocol" - - CoAP - - Stomp - - MQTT-SN - - LwM2M -### Compatible with EMQX +Steps guide: [Implement Gateway via Raw Application](doc/implement_gateway_via_raw_appliction.md) -> Why we need to compatible +## Respect emqx_gateway framework -1. Authentication -2. Hooks/Event system -3. Messages Mode & Rule Engine -4. Cluster registration -5. Metrics & Statistic +Similar to the first approach, you still need to implement an application using Erlang +and package it into EMQX. +The only difference is that you need to follow the standard behaviors(callbacks) provided +by emqx_gateway. -> How to do it +This is the approach we recommend. In this approach, your implementation can be managed +by the emqx_gateway framework, even if it may require you to understand more details about it. -> -### User Interface +Steps guide: [Implement Gateway via Gateway framework](doc/implement_gateway_via_gateway_framekwork.md) -#### Configurations +## Use ExProto Gateway (Non-Erlang developers) -```hocon -gateway { +If you want to implement your gateway using other programming languages such as +Java, Python, Go, etc. - ## ... some confs for top scope - .. - ## End. +You need to implement a gRPC service in the other programming language to parse +your device protocol and integrate it with EMQX. - ## Gateway Instances +Refer to: [ExProto Gateway](../emqx_exproto/README.md) - lwm2m[.name] { +## Cookbook for emqx_gateway framework - ## variable support - mountpoint: lwm2m/%e/ - - lifetime_min: 1s - lifetime_max: 86400s - #qmode_time_window: 22 - #auto_observe: off - - #update_msg_publish_condition: contains_object_list - - xml_dir: {{ platform_etc_dir }}/lwm2m_xml - - clientinfo_override: { - username: ${register.opts.uname} - password: ${register.opts.passwd} - clientid: ${epn} - } - - #authenticator: allow_anonymous - authenticator: [ - { - type: auth-http - method: post - //?? how to generate clientinfo ?? - params: $client.credential - } - ] - - translator: { - downlink: "dn/#" - uplink: { - notify: "up/notify" - response: "up/resp" - register: "up/resp" - update: "up/reps" - } - } - - %% ?? listener.$type.name ?? - listener.udp[.name] { - listen_on: 0.0.0.0:5683 - max_connections: 1024000 - max_conn_rate: 1000 - ## ?? udp keepalive in socket level ??? - #keepalive: - ## ?? udp proxy-protocol in socket level ??? - #proxy_protocol: on - #proxy_timeout: 30s - recbuf: 2KB - sndbuf: 2KB - buffer: 2KB - tune_buffer: off - #access: allow all - read_packets: 20 - } - - listener.dtls[.name] { - listen_on: 0.0.0.0:5684 - ... - } - } - - ## The CoAP Gateway - coap[.name] { - - #enable_stats: on - - authenticator: [ - ... - ] - - listener.udp[.name] { - ... - } - - listener.dtls[.name] { - ... - } -} - - ## The Stomp Gateway - stomp[.name] { - - allow_anonymous: true - - default_user.login: guest - default_user.passcode: guest - - frame.max_headers: 10 - frame.max_header_length: 1024 - frame.max_body_length: 8192 - - listener.tcp[.name] { - ... - } - - listener.ssl[.name] { - ... - } - } - - exproto[.name] { - - proto_name: DL-648 - - authenticators: [...] - - adapter: { - type: grpc - options: { - listen_on: 9100 - } - } - - handler: { - type: grpc - options: { - url: - } - } - - listener.tcp[.name] { - ... - } - } - - ## ============================ Enterpise gateways - - ## The JT/T 808 Gateway - jtt808[.name] { - - idle_timeout: 30s - enable_stats: on - max_packet_size: 8192 - - clientinfo_override: { - clientid: $phone - username: xxx - password: xxx - } - - authenticator: [ - { - type: auth-http - method: post - params: $clientinfo.credential - } - ] - - translator: { - subscribe: [jt808/%c/dn] - publish: [jt808/%c/up] - } - - listener.tcp[.name] { - ... - } - - listener.ssl[.name] { - ... - } - } - - gbt32960[.name] { - - frame.max_length: 8192 - retx_interval: 8s - retx_max_times: 3 - message_queue_len: 10 - - authenticators: [...] - - translator: { - ## upstream - login: gbt32960/${vin}/upstream/vlogin - logout: gbt32960/${vin}/upstream/vlogout - informing: gbt32960/${vin}/upstream/info - reinforming: gbt32960/${vin}/upstream/reinfo - ## downstream - downstream: gbt32960/${vin}/dnstream - response: gbt32960/${vin}/upstream/response - } - - listener.tcp[.name] { - ... - } - - listener.ssl[.name] { - ... - } - } - - privtcp[.name] { - - max_packet_size: 65535 - idle_timeout: 15s - - enable_stats: on - - force_gc_policy: 1000|1MB - force_shutdown_policy: 8000|800MB - - translator: { - up_topic: tcp/%c/up - dn_topic: tcp/%c/dn - } - - listener.tcp[.name]: { - ... - } - } -} -``` - -#### CLI - -##### Gateway - -```bash -## List all started gateway and gateway-instance -emqx_ctl gateway list -emqx_ctl gateway lookup -emqx_ctl gateway stop -emqx_ctl gateway start - -emqx_ctl gateway-registry re-searching -emqx_ctl gateway-registry list - -emqx_ctl gateway-clients list -emqx_ctl gateway-clients show -emqx_ctl gateway-clients kick - -## Banned ?? -emqx_ctl gateway-banned - -## Metrics -emqx_ctl gateway-metrics [] -``` - -#### Management by HTTP-API/Dashboard/ - -#### How to integrate a protocol to your platform - -### Develop your protocol gateway - -There are 3 way to create your protocol gateway for EMQX 5.0: - -1. Use Erlang to create a new emqx plugin to handle all of protocol packets (same as v5.0 before) - -2. Based on the emqx-gateway-impl-bhvr and emqx-gateway - -3. Use the gRPC Gateway +*WIP* diff --git a/apps/emqx_mqttsn/README.md b/apps/emqx_mqttsn/README.md index 67938b748..dd72a86a5 100644 --- a/apps/emqx_mqttsn/README.md +++ b/apps/emqx_mqttsn/README.md @@ -1,110 +1,34 @@ -# MQTT-SN Gateway +# emqx_mqttsn -EMQX MQTT-SN Gateway. +The MQTT-SN gateway is based on the +[MQTT-SN v1.2](https://www.oasis-open.org/committees/download.php/66091/MQTT-SN_spec_v1.2.pdf). -## Configure Plugin +## Quick Start +In EMQX 5.0, MQTT-SN gateway can be configured and enabled through the Dashboard. -File: etc/emqx_sn.conf +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: -``` -## The UDP port which emq-sn is listening on. -## -## Value: IP:Port | Port -## -## Examples: 1884, 127.0.0.1:1884, ::1:1884 -mqtt.sn.port = 1884 +```properties +gateway.mqttsn { -## The duration(seconds) that emq-sn broadcast ADVERTISE message through. -## -## Value: Second -mqtt.sn.advertise_duration = 900 + mountpoint = "mqtt/sn" -## The MQTT-SN Gateway id in ADVERTISE message. -## -## Value: Number -mqtt.sn.gateway_id = 1 + gateway_id = 1 -## To control whether write statistics data into ETS table for dashboard to read. -## -## Value: on | off -mqtt.sn.enable_stats = off + broadcast = true -## To control whether accept and process the received publish message with qos=-1. -## -## Value: on | off -mqtt.sn.enable_qos3 = off + enable_qos3 = true -## The pre-defined topic name corresponding to the pre-defined topic id of N. -## Note that the pre-defined topic id of 0 is reserved. -mqtt.sn.predefined.topic.0 = reserved -mqtt.sn.predefined.topic.1 = /predefined/topic/name/hello -mqtt.sn.predefined.topic.2 = /predefined/topic/name/nice - -## Default username for MQTT-SN. This parameter is optional. If specified, -## emq-sn will connect EMQ core with this username. It is useful if any auth -## plug-in is enabled. -## -## Value: String -mqtt.sn.username = mqtt_sn_user - -## This parameter is optional. Pair with username above. -## -## Value: String -mqtt.sn.password = abc + listeners.udp.default { + bind = 1884 + max_connections = 10240000 max_conn_rate = 1000 + } +} ``` -- mqtt.sn.port - * The UDP port which emqx-sn is listening on. -- mqtt.sn.advertise_duration - * The duration(seconds) that emqx-sn broadcast ADVERTISE message through. -- mqtt.sn.gateway_id - * Gateway id in ADVERTISE message. -- mqtt.sn.enable_stats - * To control whether write statistics data into ETS table for dashboard to read. -- mqtt.sn.enable_qos3 - * To control whether accept and process the received publish message with qos=-1. -- mqtt.sn.predefined.topic.N - * The pre-defined topic name corresponding to the pre-defined topic id of N. Note that the pre-defined topic id of 0 is reserved. -- mqtt.sn.username - * This parameter is optional. If specified, emqx-sn will connect EMQX core with this username. It is useful if any auth plug-in is enabled. -- mqtt.sn.password - * This parameter is optional. Pair with username above. +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. -## Load Plugin - -``` -./bin/emqx_ctl plugins load emqx_sn -``` - -## Client - -### NOTE -- Topic ID is per-client, and will be cleared if client disconnected with broker or keepalive failure is detected in broker. -- Please register your topics again each time connected with broker. -- If your udp socket(mqtt-sn client) has successfully connected to broker, don't try to send another CONNECT on this socket again, which will lead to confusing behaviour. If you want to start from beging, please do as following: - + destroy your present socket and create a new socket to connect again - + or send DISCONNECT on the same socket and connect again. - -### Library - -- https://github.com/eclipse/paho.mqtt-sn.embedded-c/ -- https://github.com/ty4tw/MQTT-SN -- https://github.com/njh/mqtt-sn-tools -- https://github.com/arobenko/mqtt-sn - -### sleeping device - -PINGREQ must have a ClientId which is identical to the one in CONNECT message. Without ClientId, emqx-sn will ignore such PINGREQ. - -### pre-defined topics - -The mapping of a pre-defined topic id and topic name should be known inadvance by both client's application and gateway. We define this mapping info in emqx_sn.conf file, and which shall be kept equivalent in all client's side. - -## License - -Apache License Version 2.0 - -## Author - -EMQX Team. +More documentations: [MQTT-SN Gateway](https://www.emqx.io/docs/en/v5.0/gateway/mqttsn.html) diff --git a/apps/emqx_stomp/README.md b/apps/emqx_stomp/README.md index d96999aec..0c41ff520 100644 --- a/apps/emqx_stomp/README.md +++ b/apps/emqx_stomp/README.md @@ -1,73 +1,31 @@ +# emqx_stomp -# emqx-stomp +The Stomp Gateway is based on the +[Stomp v1.2](https://stomp.github.io/stomp-specification-1.2.html) and is +compatible with the Stomp v1.0 and v1.1 specification. +## Quick Start -The plugin adds STOMP 1.0/1.1/1.2 protocol supports to the EMQX broker. +In EMQX 5.0, Stomp gateway can be configured and enabled through the Dashboard. -The STOMP clients could PubSub to the MQTT clients. +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: -## Configuration +```properties +gateway.stomp { -etc/emqx_stomp.conf + mountpoint = "stomp/" -``` -## The Port that stomp listener will bind. -## -## Value: Port -stomp.listener = 61613 - -## The acceptor pool for stomp listener. -## -## Value: Number -stomp.listener.acceptors = 4 - -## Maximum number of concurrent stomp connections. -## -## Value: Number -stomp.listener.max_connections = 512 - -## Default login user -## -## Value: String -stomp.default_user.login = guest - -## Default login password -## -## Value: String -stomp.default_user.passcode = guest - -## Allow anonymous authentication. -## -## Value: true | false -stomp.allow_anonymous = true - -## Maximum numbers of frame headers. -## -## Value: Number -stomp.frame.max_headers = 10 - -## Maximum length of frame header. -## -## Value: Number -stomp.frame.max_header_length = 1024 - -## Maximum body length of frame. -## -## Value: Number -stomp.frame.max_body_length = 8192 + listeners.tcp.default { + bind = 61613 + acceptors = 16 + max_connections = 1024000 + max_conn_rate = 1000 + } +} ``` -## Load the Plugin - -``` -./bin/emqx_ctl plugins load emqx_stomp -``` - -## License - -Apache License Version 2.0 - -## Author - -EMQX Team. +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. +More documentations: [Stomp Gateway](https://www.emqx.io/docs/en/v5.0/gateway/stomp.html) From 51eec32d6bcc08f0c60ff99e38afa8d9d5fcc935 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 3 Apr 2023 09:11:14 -0300 Subject: [PATCH 015/279] docs: improve descriptions Co-authored-by: LenaLenaPan <120552185+LenaLenaPan@users.noreply.github.com> --- lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf index 566d1889e..badced748 100644 --- a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_kafka.conf @@ -62,7 +62,7 @@ emqx_ee_bridge_kafka { mqtt_topic { desc { en: "MQTT topic or topic filter as data source (bridge input). If rule action is used as data source, this config should be left empty, otherwise messages will be duplicated in Kafka." - zh: "指定源 MQTT 主题，若由规则动作指定数据源，则该配置应留空，否则消息会被重复发送到 Kafka。" + zh: "MQTT 主题数据源由桥接指定，或留空由规则动作指定。" } label { en: "Source MQTT Topic" From 04626ce9cc59c95444c8fb8740a8fb176b7713bd Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 24 Mar 2023 16:42:21 +0100 Subject: [PATCH 016/279] fix: create consistent interface 'with_node' for API access --- apps/emqx/include/emqx_api_lib.hrl | 36 +++++++ apps/emqx/src/emqx_api_lib.erl | 69 ++++++++++++ apps/emqx/test/emqx_api_lib_SUITE.erl | 101 ++++++++++++++++++ apps/emqx_bridge/src/emqx_bridge_api.erl | 5 +- .../emqx_dashboard/src/emqx_dashboard.app.src | 2 +- .../src/emqx_dashboard_monitor_api.erl | 27 ++--- .../src/emqx_management.app.src | 2 +- .../src/emqx_mgmt_api_nodes.erl | 62 +++++------ .../test/emqx_mgmt_api_nodes_SUITE.erl | 6 +- changes/ce/fix-10237.en.md | 1 + 10 files changed, 249 insertions(+), 62 deletions(-) create mode 100644 apps/emqx/include/emqx_api_lib.hrl create mode 100644 apps/emqx/src/emqx_api_lib.erl create mode 100644 apps/emqx/test/emqx_api_lib_SUITE.erl create mode 100644 changes/ce/fix-10237.en.md diff --git a/apps/emqx/include/emqx_api_lib.hrl b/apps/emqx/include/emqx_api_lib.hrl new file mode 100644 index 000000000..549b0f94c --- /dev/null +++ b/apps/emqx/include/emqx_api_lib.hrl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_API_LIB_HRL). +-define(EMQX_API_LIB_HRL, true). + +-define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_misc:readable_error_msg(REASON)}). + +-define(OK(CONTENT), {200, CONTENT}). + +-define(NO_CONTENT, 204). + +-define(BAD_REQUEST(CODE, REASON), {400, ?ERROR_MSG(CODE, REASON)}). +-define(BAD_REQUEST(REASON), ?BAD_REQUEST('BAD_REQUEST', REASON)). + +-define(NOT_FOUND(REASON), {404, ?ERROR_MSG('NOT_FOUND', REASON)}). + +-define(INTERNAL_ERROR(REASON), {500, ?ERROR_MSG('INTERNAL_ERROR', REASON)}). + +-define(NOT_IMPLEMENTED, 501). + +-define(SERVICE_UNAVAILABLE(REASON), {503, ?ERROR_MSG('SERVICE_UNAVAILABLE', REASON)}). +-endif. diff --git a/apps/emqx/src/emqx_api_lib.erl b/apps/emqx/src/emqx_api_lib.erl new file mode 100644 index 000000000..8c49c57c3 --- /dev/null +++ b/apps/emqx/src/emqx_api_lib.erl @@ -0,0 +1,69 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_api_lib). + +-export([ + with_node/2, + with_node_or_cluster/2 +]). + +-include("emqx_api_lib.hrl"). + +-define(NODE_NOT_FOUND(NODE), ?NOT_FOUND(<<"Node not found: ", NODE/binary>>)). + +%%-------------------------------------------------------------------- +%% exported API +%%-------------------------------------------------------------------- +-spec with_node(binary(), fun((atom()) -> {ok, term()} | {error, term()})) -> + ?OK(term()) | ?NOT_FOUND(binary()) | ?BAD_REQUEST(term()). +with_node(BinNode, Fun) -> + case lookup_node(BinNode) of + {ok, Node} -> + handle_result(Fun(Node)); + not_found -> + ?NODE_NOT_FOUND(BinNode) + end. + +-spec with_node_or_cluster(binary(), fun((atom()) -> {ok, term()} | {error, term()})) -> + ?OK(term()) | ?NOT_FOUND(iolist()) | ?BAD_REQUEST(term()). +with_node_or_cluster(<<"all">>, Fun) -> + handle_result(Fun(all)); +with_node_or_cluster(Node, Fun) -> + with_node(Node, Fun). + +%%-------------------------------------------------------------------- +%% Internal +%%-------------------------------------------------------------------- + +-spec lookup_node(binary()) -> {ok, atom()} | not_found. +lookup_node(BinNode) -> + case emqx_misc:safe_to_existing_atom(BinNode, utf8) of + {ok, Node} -> + case lists:member(Node, mria:running_nodes()) of + true -> + {ok, Node}; + false -> + not_found + end; + _Error -> + not_found + end. + +handle_result({ok, Result}) -> + ?OK(Result); +handle_result({error, Reason}) -> + ?BAD_REQUEST(Reason). diff --git a/apps/emqx/test/emqx_api_lib_SUITE.erl b/apps/emqx/test/emqx_api_lib_SUITE.erl new file mode 100644 index 000000000..29f5c6095 --- /dev/null +++ b/apps/emqx/test/emqx_api_lib_SUITE.erl @@ -0,0 +1,101 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_api_lib_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx_api_lib.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(DUMMY, dummy_module). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:boot_modules(all), + emqx_common_test_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([]). + +init_per_testcase(_Case, Config) -> + meck:new(?DUMMY, [non_strict]), + meck:expect(?DUMMY, expect_not_called, 1, fun(Node) -> throw({blow_this_up, Node}) end), + meck:expect(?DUMMY, expect_success, 1, {ok, success}), + meck:expect(?DUMMY, expect_error, 1, {error, error}), + Config. + +end_per_testcase(_Case, _Config) -> + meck:unload(?DUMMY). + +t_with_node(_) -> + test_with(fun emqx_api_lib:with_node/2, [<<"all">>]). + +t_with_node_or_cluster(_) -> + test_with(fun emqx_api_lib:with_node_or_cluster/2, []), + meck:reset(?DUMMY), + ?assertEqual( + ?OK(success), + emqx_api_lib:with_node_or_cluster( + <<"all">>, + fun ?DUMMY:expect_success/1 + ) + ), + ?assertMatch([{_, {?DUMMY, expect_success, [all]}, {ok, success}}], meck:history(?DUMMY)). + +%% helpers +test_with(TestFun, ExtraBadNodes) -> + % make sure this is an atom + 'unknownnode@unknownnohost', + BadNodes = + [ + <<"undefined">>, + <<"this_should_not_be_an_atom">>, + <<"unknownnode@unknownnohost">> + ] ++ ExtraBadNodes, + [ensure_not_found(TestFun(N, fun ?DUMMY:expect_not_called/1)) || N <- BadNodes], + ensure_not_called(?DUMMY, expect_not_called), + ensure_not_existing_atom(<<"this_should_not_be_an_atom">>), + + GoodNode = node(), + + ?assertEqual( + ?OK(success), + TestFun(GoodNode, fun ?DUMMY:expect_success/1) + ), + + ?assertEqual( + ?BAD_REQUEST(error), + TestFun(GoodNode, fun ?DUMMY:expect_error/1) + ), + ok. + +ensure_not_found(Result) -> + ?assertMatch({404, _}, Result). + +ensure_not_called(Mod, Fun) -> + ?assert(not meck:called(Mod, Fun, '_')). + +ensure_not_existing_atom(Bin) -> + try binary_to_existing_atom(Bin) of + _ -> throw(is_atom) + catch + error:badarg -> + ok + end. diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index b9a6d4c06..ff93ac584 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -20,6 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_api_lib.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). -import(hoconsc, [mk/2, array/1, enum/1]). @@ -46,14 +47,10 @@ -export([lookup_from_local_node/2]). --define(BAD_REQUEST(Reason), {400, error_msg('BAD_REQUEST', Reason)}). - -define(BRIDGE_NOT_ENABLED, ?BAD_REQUEST(<<"Forbidden operation, bridge not enabled">>) ). --define(NOT_FOUND(Reason), {404, error_msg('NOT_FOUND', Reason)}). - -define(BRIDGE_NOT_FOUND(BridgeType, BridgeName), ?NOT_FOUND( <<"Bridge lookup failed: bridge named '", (BridgeName)/binary, "' of type ", diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 3970d76e4..8a4764c84 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.15"}, + {vsn, "5.0.16"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index 69f5bf34e..aaee92b8c 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -121,32 +121,27 @@ fields(sampler_current) -> monitor(get, #{query_string := QS, bindings := Bindings}) -> Latest = maps:get(<<"latest">>, QS, infinity), - RawNode = maps:get(node, Bindings, all), - with_node(RawNode, dashboard_samplers_fun(Latest)). + RawNode = maps:get(node, Bindings, <<"all">>), + emqx_api_lib:with_node_or_cluster(RawNode, dashboard_samplers_fun(Latest)). dashboard_samplers_fun(Latest) -> fun(NodeOrCluster) -> case emqx_dashboard_monitor:samplers(NodeOrCluster, Latest) of - {badrpc, _} = Error -> Error; + {badrpc, _} = Error -> {error, Error}; Samplers -> {ok, Samplers} end end. monitor_current(get, #{bindings := Bindings}) -> - RawNode = maps:get(node, Bindings, all), - with_node(RawNode, fun emqx_dashboard_monitor:current_rate/1). + RawNode = maps:get(node, Bindings, <<"all">>), + emqx_api_lib:with_node_or_cluster(RawNode, fun current_rate/1). -with_node(RawNode, Fun) -> - case emqx_misc:safe_to_existing_atom(RawNode, utf8) of - {ok, NodeOrCluster} -> - case Fun(NodeOrCluster) of - {badrpc, {Node, Reason}} -> - {404, 'NOT_FOUND', io_lib:format("Node not found: ~p (~p)", [Node, Reason])}; - {ok, Result} -> - {200, Result} - end; - _Error -> - {404, 'NOT_FOUND', io_lib:format("Node not found: ~p", [RawNode])} +current_rate(Node) -> + case emqx_dashboard_monitor:current_rate(Node) of + {badrpc, _} = BadRpc -> + {error, BadRpc}; + {ok, _} = OkResult -> + OkResult end. %% ------------------------------------------------------------------------------------------------- diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 966358f47..9863f5cf6 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.16"}, + {vsn, "5.0.17"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index 21d905331..a4173f5b0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -17,7 +17,6 @@ -behaviour(minirest_api). --include_lib("emqx/include/emqx.hrl"). -include_lib("typerefl/include/types.hrl"). -import(hoconsc, [mk/2, ref/1, ref/2, enum/1, array/1]). @@ -25,8 +24,6 @@ -define(NODE_METRICS_MODULE, emqx_mgmt_api_metrics). -define(NODE_STATS_MODULE, emqx_mgmt_api_stats). --define(SOURCE_ERROR, 'SOURCE_ERROR'). - %% Swagger specs from hocon schema -export([ api_spec/0, @@ -88,7 +85,7 @@ schema("/nodes/:node") -> ref(node_info), #{desc => <<"Get node info successfully">>} ), - 400 => node_error() + 404 => not_found() } } }; @@ -106,7 +103,7 @@ schema("/nodes/:node/metrics") -> ref(?NODE_METRICS_MODULE, node_metrics), #{desc => <<"Get node metrics successfully">>} ), - 400 => node_error() + 404 => not_found() } } }; @@ -124,7 +121,7 @@ schema("/nodes/:node/stats") -> ref(?NODE_STATS_MODULE, node_stats_data), #{desc => <<"Get node stats successfully">>} ), - 400 => node_error() + 404 => not_found() } } }. @@ -136,7 +133,7 @@ fields(node_name) -> [ {node, mk( - atom(), + binary(), #{ in => path, description => <<"Node name">>, @@ -250,55 +247,46 @@ nodes(get, _Params) -> list_nodes(#{}). node(get, #{bindings := #{node := NodeName}}) -> - get_node(NodeName). + emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun get_node/1)). node_metrics(get, #{bindings := #{node := NodeName}}) -> - get_metrics(NodeName). + emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_metrics/1)). node_stats(get, #{bindings := #{node := NodeName}}) -> - get_stats(NodeName). + emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_stats/1)). %%-------------------------------------------------------------------- %% api apply list_nodes(#{}) -> - NodesInfo = [format(Node, NodeInfo) || {Node, NodeInfo} <- emqx_mgmt:list_nodes()], + NodesInfo = [format(NodeInfo) || {_Node, NodeInfo} <- emqx_mgmt:list_nodes()], {200, NodesInfo}. get_node(Node) -> - case emqx_mgmt:lookup_node(Node) of - {error, _} -> - {400, #{code => 'SOURCE_ERROR', message => <<"rpc_failed">>}}; - NodeInfo -> - {200, format(Node, NodeInfo)} - end. - -get_metrics(Node) -> - case emqx_mgmt:get_metrics(Node) of - {error, _} -> - {400, #{code => 'SOURCE_ERROR', message => <<"rpc_failed">>}}; - Metrics -> - {200, Metrics} - end. - -get_stats(Node) -> - case emqx_mgmt:get_stats(Node) of - {error, _} -> - {400, #{code => 'SOURCE_ERROR', message => <<"rpc_failed">>}}; - Stats -> - {200, Stats} - end. + format(emqx_mgmt:lookup_node(Node)). %%-------------------------------------------------------------------- %% internal function -format(_Node, Info = #{memory_total := Total, memory_used := Used}) -> +format(Info = #{memory_total := Total, memory_used := Used}) -> Info#{ memory_total := emqx_mgmt_util:kmg(Total), memory_used := emqx_mgmt_util:kmg(Used) }; -format(_Node, Info) when is_map(Info) -> +format(Info) when is_map(Info) -> Info. -node_error() -> - emqx_dashboard_swagger:error_codes([?SOURCE_ERROR], <<"Node error">>). +to_ok_result({error, _} = Error) -> + Error; +to_ok_result({ok, _} = Ok) -> + Ok; +to_ok_result(Result) -> + {ok, Result}. + +to_ok_result_fun(Fun) when is_function(Fun) -> + fun(Arg) -> + to_ok_result(Fun(Arg)) + end. + +not_found() -> + emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Node not found">>). diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 03b0ea2d9..30313e555 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -68,7 +68,7 @@ t_nodes_api(_) -> BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode"]), ?assertMatch( - {error, {_, 400, _}}, + {error, {_, 404, _}}, emqx_mgmt_api_test_util:request_api(get, BadNodePath) ). @@ -94,7 +94,7 @@ t_node_stats_api(_) -> BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode", "stats"]), ?assertMatch( - {error, {_, 400, _}}, + {error, {_, 404, _}}, emqx_mgmt_api_test_util:request_api(get, BadNodePath) ). @@ -112,7 +112,7 @@ t_node_metrics_api(_) -> BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode", "metrics"]), ?assertMatch( - {error, {_, 400, _}}, + {error, {_, 404, _}}, emqx_mgmt_api_test_util:request_api(get, BadNodePath) ). diff --git a/changes/ce/fix-10237.en.md b/changes/ce/fix-10237.en.md new file mode 100644 index 000000000..cf3fc707b --- /dev/null +++ b/changes/ce/fix-10237.en.md @@ -0,0 +1 @@ +Ensure we return `404` status code for unknown node names in `/nodes/:node[/metrics|/stats]` API. From c20da5ffa6decba63b573f78d5caf14e740fe61b Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 3 Apr 2023 11:55:36 +0200 Subject: [PATCH 017/279] fix(emqx_dashboard): fix monitor_current api --- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 3970d76e4..8a4764c84 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.15"}, + {vsn, "5.0.16"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index 69f5bf34e..34a8f0231 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -132,6 +132,8 @@ dashboard_samplers_fun(Latest) -> end end. +monitor_current(get, #{bindings := []}) -> + with_node(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); monitor_current(get, #{bindings := Bindings}) -> RawNode = maps:get(node, Bindings, all), with_node(RawNode, fun emqx_dashboard_monitor:current_rate/1). From 3d7ceb01a0ba4af0d68fee5598a7bc1f70d7b972 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 3 Apr 2023 14:14:21 +0200 Subject: [PATCH 018/279] fix(mgmt): fix stats api by applying filter to running_nodes --- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt_api_stats.erl | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 966358f47..9863f5cf6 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.16"}, + {vsn, "5.0.17"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 1d3c0e21b..105f6c047 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -129,7 +129,19 @@ list(get, #{query_string := Qs}) -> _ -> Data = [ maps:from_list(emqx_mgmt:get_stats(Node) ++ [{node, Node}]) - || Node <- mria:running_nodes() + || Node <- running_nodes() ], {200, Data} end. + +%%%============================================================================================== +%% Internal + +running_nodes() -> + Nodes = erlang:nodes([visible, this]), + RpcResults = erpc:multicall(Nodes, emqx, is_running, [], 15000), + [ + Node + || {Node, IsRunning} <- lists:zip(Nodes, RpcResults), + IsRunning =:= {ok, true} + ]. From 2571da368cf9ba18f07476fec32dca2fc86947eb Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 3 Apr 2023 15:28:18 +0200 Subject: [PATCH 019/279] chore: add changelog --- changes/ce/fix-10314.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/fix-10314.en.md diff --git a/changes/ce/fix-10314.en.md b/changes/ce/fix-10314.en.md new file mode 100644 index 000000000..0557e714e --- /dev/null +++ b/changes/ce/fix-10314.en.md @@ -0,0 +1,2 @@ +Fix /monitor_current API so that it only looks at the current node. +Fix /stats API to not crash when one or more nodes in the cluster are down. From 9d1a16aae1c329a3fb4fe5e3fc1d9c20782d5bf2 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 4 Apr 2023 20:40:47 +0200 Subject: [PATCH 020/279] feat: add emqx_rpc:multicall_on_running also move emqx:is_running multicall to emqx_proto_v2:are_running --- apps/emqx/priv/bpapi.versions | 1 + apps/emqx/src/emqx_rpc.erl | 13 +++ apps/emqx/src/proto/emqx_proto_v2.erl | 86 +++++++++++++++++++ .../src/emqx_mgmt_api_stats.erl | 2 +- 4 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 apps/emqx/src/proto/emqx_proto_v2.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 6190925d2..c5619102c 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -1,5 +1,6 @@ %% This file is automatically generated by `make static_checks`, do not edit. {emqx,1}. +{emqx,2}. {emqx_authn,1}. {emqx_authz,1}. {emqx_bridge,1}. diff --git a/apps/emqx/src/emqx_rpc.erl b/apps/emqx/src/emqx_rpc.erl index e1b5122c4..062bde68b 100644 --- a/apps/emqx/src/emqx_rpc.erl +++ b/apps/emqx/src/emqx_rpc.erl @@ -27,6 +27,8 @@ cast/5, multicall/4, multicall/5, + multicall_on_running/5, + on_running/3, unwrap_erpc/1 ]). @@ -91,6 +93,17 @@ multicall(Nodes, Mod, Fun, Args) -> multicall(Key, Nodes, Mod, Fun, Args) -> gen_rpc:multicall(rpc_nodes([{Key, Node} || Node <- Nodes]), Mod, Fun, Args). +-spec multicall_on_running([node()], module(), atom(), list(), timeout()) -> [term() | {error, _}]. +multicall_on_running(Nodes, Mod, Fun, Args, Timeout) -> + unwrap_erpc(erpc:multicall(Nodes, emqx_rpc, on_running, [Mod, Fun, Args], Timeout)). + +-spec on_running(module(), atom(), list()) -> term(). +on_running(Mod, Fun, Args) -> + case emqx:is_running() of + true -> apply(Mod, Fun, Args); + false -> error(emqx_down) + end. + -spec cast(node(), module(), atom(), list()) -> cast_result(). cast(Node, Mod, Fun, Args) -> %% Note: using a non-ordered cast here, since the generated key is diff --git a/apps/emqx/src/proto/emqx_proto_v2.erl b/apps/emqx/src/proto/emqx_proto_v2.erl new file mode 100644 index 000000000..a11c8a10e --- /dev/null +++ b/apps/emqx/src/proto/emqx_proto_v2.erl @@ -0,0 +1,86 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_proto_v2). + +-behaviour(emqx_bpapi). + +-include("bpapi.hrl"). + +-export([ + introduced_in/0, + + are_running/1, + is_running/1, + + get_alarms/2, + get_stats/1, + get_metrics/1, + + deactivate_alarm/2, + delete_all_deactivated_alarms/1, + + clean_authz_cache/1, + clean_authz_cache/2, + clean_pem_cache/1 +]). + +introduced_in() -> + "5.0.22". + +-spec is_running(node()) -> boolean() | {badrpc, term()}. +is_running(Node) -> + rpc:call(Node, emqx, is_running, []). + +-spec are_running([node()]) -> emqx_rpc:erpc_multicall(boolean()). +are_running(Nodes) when is_list(Nodes) -> + erpc:multicall(Nodes, emqx, is_running, []). + +-spec get_alarms(node(), all | activated | deactivated) -> [map()]. +get_alarms(Node, Type) -> + rpc:call(Node, emqx_alarm, get_alarms, [Type]). + +-spec get_stats(node()) -> emqx_stats:stats() | {badrpc, _}. +get_stats(Node) -> + rpc:call(Node, emqx_stats, getstats, []). + +-spec get_metrics(node()) -> [{emqx_metrics:metric_name(), non_neg_integer()}] | {badrpc, _}. +get_metrics(Node) -> + rpc:call(Node, emqx_metrics, all, []). + +-spec clean_authz_cache(node(), emqx_types:clientid()) -> + ok + | {error, not_found} + | {badrpc, _}. +clean_authz_cache(Node, ClientId) -> + rpc:call(Node, emqx_authz_cache, drain_cache, [ClientId]). + +-spec clean_authz_cache(node()) -> ok | {badrpc, _}. +clean_authz_cache(Node) -> + rpc:call(Node, emqx_authz_cache, drain_cache, []). + +-spec clean_pem_cache(node()) -> ok | {badrpc, _}. +clean_pem_cache(Node) -> + rpc:call(Node, ssl_pem_cache, clear, []). + +-spec deactivate_alarm(node(), binary() | atom()) -> + ok | {error, not_found} | {badrpc, _}. +deactivate_alarm(Node, Name) -> + rpc:call(Node, emqx_alarm, deactivate, [Name]). + +-spec delete_all_deactivated_alarms(node()) -> ok | {badrpc, _}. +delete_all_deactivated_alarms(Node) -> + rpc:call(Node, emqx_alarm, delete_all_deactivated_alarms, []). diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 105f6c047..1e752aaac 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -139,7 +139,7 @@ list(get, #{query_string := Qs}) -> running_nodes() -> Nodes = erlang:nodes([visible, this]), - RpcResults = erpc:multicall(Nodes, emqx, is_running, [], 15000), + RpcResults = emqx_proto_v2:are_running(Nodes), [ Node || {Node, IsRunning} <- lists:zip(Nodes, RpcResults), From a3262486e598ab2bd34ee47e0f23deaad5770856 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 5 Apr 2023 15:25:08 +0800 Subject: [PATCH 021/279] chore: delete all LICENSE files under gateway dirs --- apps/emqx_coap/LICENSE | 191 ------------------ apps/emqx_exproto/LICENSE | 191 ------------------ apps/emqx_lwm2m/LICENSE | 191 ------------------ apps/emqx_lwm2m/README.md | 407 ++++++-------------------------------- apps/emqx_mqttsn/LICENSE | 191 ------------------ apps/emqx_stomp/LICENSE | 191 ------------------ 6 files changed, 56 insertions(+), 1306 deletions(-) delete mode 100644 apps/emqx_coap/LICENSE delete mode 100644 apps/emqx_exproto/LICENSE delete mode 100644 apps/emqx_lwm2m/LICENSE delete mode 100644 apps/emqx_mqttsn/LICENSE delete mode 100644 apps/emqx_stomp/LICENSE diff --git a/apps/emqx_coap/LICENSE b/apps/emqx_coap/LICENSE deleted file mode 100644 index 5a5418f0f..000000000 --- a/apps/emqx_coap/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, JianBo He . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_exproto/LICENSE b/apps/emqx_exproto/LICENSE deleted file mode 100644 index 5a5418f0f..000000000 --- a/apps/emqx_exproto/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, JianBo He . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_lwm2m/LICENSE b/apps/emqx_lwm2m/LICENSE deleted file mode 100644 index 5a5418f0f..000000000 --- a/apps/emqx_lwm2m/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, JianBo He . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_lwm2m/README.md b/apps/emqx_lwm2m/README.md index faca6dad3..678d74dcf 100644 --- a/apps/emqx_lwm2m/README.md +++ b/apps/emqx_lwm2m/README.md @@ -1,356 +1,61 @@ -# LwM2M Gateway +# emqx_lwm2m -[The LwM2M Specifications](http://www.openmobilealliance.org/release/LightweightM2M) is a Lightweight Machine to Machine protocol. +[LwM2M (Lightweight Machine-to-Machine)](https://lwm2m.openmobilealliance.org/) +is a protocol designed for IoT devices and machine-to-machine communication. +It is a lightweight protocol that supports devices with limited processing power and memory. -With `emqx_lwm2m`, user is able to send LwM2M commands(READ/WRITE/EXECUTE/...) and get LwM2M response in MQTT way. `emqx_lwm2m` transforms data between MQTT and LwM2M protocol. + +The **LwM2M Gateway** in EMQX can accept LwM2M clients and translate theirevents +and messages into MQTT Publish messages. + +In the current implementation, it has the following limitations: +- Based UDP/DTLS transport. +- Only supports v1.0.2. The v1.1.x and v1.2.x is not supported yet. +- Not included LwM2M Bootstrap services. + +## Quick Start + +In EMQX 5.0, LwM2M gateways can be configured and enabled through the Dashboard. + +It can also be enabled via the HTTP API, and emqx.conf e.g, In emqx.conf: + +```properties +gateway.lwm2m { + xml_dir = "etc/lwm2m_xml/" + auto_observe = true + enable_stats = true + idle_timeout = "30s" + lifetime_max = "86400s" + lifetime_min = "1s" + mountpoint = "lwm2m/${endpoint_namea}/" + qmode_time_window = "22s" + update_msg_publish_condition = "contains_object_list" + translators { + command {qos = 0, topic = "dn/#"} + notify {qos = 0, topic = "up/notify"} + register {qos = 0, topic = "up/resp"} + response {qos = 0, topic = "up/resp"} + update {qos = 0, topic = "up/update"} + } + listeners { + udp { + default { + bind = "5783" + max_conn_rate = 1000 + max_connections = 1024000 + } + } + } +} +``` + +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. +::: + +## Object definations emqx_lwm2m needs object definitions to parse data from lwm2m devices. Object definitions are declared by organizations in XML format, you could find those XMLs from [LwM2MRegistry](http://www.openmobilealliance.org/wp/OMNA/LwM2M/LwM2MRegistry.html), download and put them into the directory specified by `lwm2m.xml_dir`. If no associated object definition is found, response from device will be discarded and report an error message in log. -## Load emqx_lwm2m - -``` -./bin/emqx_ctl plugins load emqx_lwm2m -``` - -## Test emqx-lwm2m using *wakaama* - -[wakaama](https://github.com/eclipse/wakaama) is an easy-to-use lwm2m client command line tool. - -Start *lwm2mclient* using an endpoint name `ep1`: -``` -./lwm2mclient -n ep1 -h 127.0.0.1 -p 5683 -4 -``` - -To send an LwM2M DISCOVER command to *lwm2mclient*, publish an MQTT message to topic `lwm2m//dn` (where `` is the endpoint name of the client), with following payload: - -```json -{ - "reqID": "2", - "msgType": "discover", - "data": { - "path": "/3/0" - } -} -``` - -The MQTT message will be translated to an LwM2M DISCOVER command and sent to the *lwm2mclient*. Then the response of *lwm2mclient* will be in turn translated to an MQTT message, with topic `lwm2m//up/resp`, with following payload: - -```json -{ - "reqID": "2", - "msgType": "discover", - "data": { - "code":"2.05", - "codeMsg": "content", - "content": [ - ";dim=8", - "", - "", - "", - "" - ] - } -} -``` - -## LwM2M <--> MQTT Mapping - -### Register/Update (LwM2M Client Registration Interface) - -- **LwM2M Register and Update message will be converted to following MQTT message:** - - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/up/resp` (configurable) - - **Payload**: - - MsgType **register** and **update**: - ```json - { - "msgType": {?MsgType}, - "data": { - "ep": {?EndpointName}, - "lt": {?LifeTime}, - "sms": {?MSISDN}, - "lwm2m": {?Lwm2mVersion}, - "b": {?Binding}, - "alternatePath": {?AlternatePath}, - "objectList": {?ObjectList} - } - } - ``` - - {?EndpointName}: String, the endpoint name of the LwM2M client - - {?MsgType}: String, could be: - - "register": LwM2M Register - - "update": LwM2M Update - - "data" contains the query options and the object-list of the register message - - The *update* message is only published if the object-list changed. - -### Downlink Command and Uplink Response (LwM2M Device Management & Service Enablement Interface) - -- **To send a downlink command to device, publish following MQTT message:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/dn` - - **Request Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "data": {?Data} - } - ``` - - {?ReqID}: Integer, request-id, used for matching the response to the request - - {?MsgType}: String, can be one of the following: - - "read": LwM2M Read - - "discover": LwM2M Discover - - "write": LwM2M Write - - "write-attr": LwM2M Write Attributes - - "execute": LwM2M Execute - - "create": LwM2M Create - - "delete": LwM2M Delete - - {?Data}: JSON Object, its value depends on the {?MsgType}: - - **If {?MsgType} = "read" or "discover"**: - ```json - { - "path": {?ResourcePath} - } - ``` - - {?ResourcePath}: String, LwM2M full resource path. e.g. "3/0", "/3/0/0", "/3/0/6/0" - - **If {?MsgType} = "write" (single write)**: - ```json - { - "path": {?ResourcePath}, - "type": {?ValueType}, - "value": {?Value} - } - ``` - - {?ValueType}: String, can be: "Time", "String", "Integer", "Float", "Boolean", "Opaque", "Objlnk" - - {?Value}: Value of the resource, depends on "type". - - **If {?MsgType} = "write" (batch write)**: - ```json - { - "basePath": {?BasePath}, - "content": [ - { - "path": {?ResourcePath}, - "type": {?ValueType}, - "value": {?Value} - } - ] - } - ``` - - The full path is concatenation of "basePath" and "path". - - **If {?MsgType} = "write-attr"**: - ```json - { - "path": {?ResourcePath}, - "pmin": {?PeriodMin}, - "pmax": {?PeriodMax}, - "gt": {?GreaterThan}, - "lt": {?LessThan}, - "st": {?Step} - } - ``` - - {?PeriodMin}: Number, LwM2M Notification Class Attribute - Minimum Period. - - {?PeriodMax}: Number, LwM2M Notification Class Attribute - Maximum Period. - - {?GreaterThan}: Number, LwM2M Notification Class Attribute - Greater Than. - - {?LessThan}: Number, LwM2M Notification Class Attribute - Less Than. - - {?Step}: Number, LwM2M Notification Class Attribute - Step. - - - **If {?MsgType} = "execute"**: - ```json - { - "path": {?ResourcePath}, - "args": {?Arguments} - } - ``` - - {?Arguments}: String, LwM2M Execute Arguments. - - - **If {?MsgType} = "create"**: - ```json - { - "basePath": "/{?ObjectID}", - "content": [ - { - "path": {?ResourcePath}, - "type": {?ValueType}, - "value": {?Value} - } - ] - } - ``` - - {?ObjectID}: Integer, LwM2M Object ID - - - **If {?MsgType} = "delete"**: - ```json - { - "path": "{?ObjectID}/{?ObjectInstanceID}" - } - ``` - - {?ObjectInstanceID}: Integer, LwM2M Object Instance ID - -- **The response of LwM2M will be converted to following MQTT message:** - - **Method:** PUBLISH - - **Topic:** `"lwm2m/{?EndpointName}/up/resp"` - - **Response Payload:** - - ```json - { - "reqID": {?ReqID}, - "imei": {?IMEI}, - "imsi": {?IMSI}, - "msgType": {?MsgType}, - "data": {?Data} - } - ``` - - - {?MsgType}: String, can be: - - "read": LwM2M Read - - "discover": LwM2M Discover - - "write": LwM2M Write - - "write-attr": LwM2M Write Attributes - - "execute": LwM2M Execute - - "create": LwM2M Create - - "delete": LwM2M Delete - - **"ack"**: [CoAP Empty ACK](https://tools.ietf.org/html/rfc7252#section-5.2.2) - - {?Data}: JSON Object, its value depends on {?MsgType}: - - **If {?MsgType} = "write", "write-attr", "execute", "create", "delete", or "read"(when response without content)**: - ```json - { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath} - } - ``` - - {?StatusCode}: String, LwM2M status code, e.g. "2.01", "4.00", etc. - - {?CodeMsg}: String, LwM2M response message, e.g. "content", "bad_request" - - {?RequestPath}: String, the requested "path" or "basePath" - - - **If {?MsgType} = "discover"**: - ```json - { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath}, - "content": [ - {?Link}, - ... - ] - } - ``` - - {?Link}: String(LwM2M link format) e.g. `""`, `"<3/0/1>;dim=8"` - - - **If {?MsgType} = "read"(when response with content)**: - ```json - { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "content": {?Content} - } - ``` - - {?Content} - ```json - [ - { - "path": {?ResourcePath}, - "value": {?Value} - } - ] - ``` - - - **If {?MsgType} = "ack", "data" does not exists** - -### Observe (Information Reporting Interface - Observe/Cancel-Observe) - -- **To observe/cancel-observe LwM2M client, send following MQTT PUBLISH:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/dn` - - **Request Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "data": { - "path": {?ResourcePath} - } - } - ``` - - {?ResourcePath}: String, the LwM2M resource to be observed/cancel-observed. - - {?MsgType}: String, can be: - - "observe": LwM2M Observe - - "cancel-observe": LwM2M Cancel Observe - - {?ReqID}: Integer, request-id, is the {?ReqID} in the request - -- **Responses will be converted to following MQTT message:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpointName}/up/resp` - - **Response Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "data": { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath}, - "content": [ - { - "path": {?ResourcePath}, - "value": {?Value} - } - ] - } - } - ``` - - {?MsgType}: String, can be: - - "observe": LwM2M Observe - - "cancel-observe": LwM2M Cancel Observe - - **"ack"**: [CoAP Empty ACK](https://tools.ietf.org/html/rfc7252#section-5.2.2) - -### Notification (Information Reporting Interface - Notify) - -- **The notifications from LwM2M clients will be converted to MQTT PUBLISH:** - - **Method:** PUBLISH - - **Topic:** `lwm2m/{?EndpiontName}/up/notify` - - **Notification Payload**: - ```json - { - "reqID": {?ReqID}, - "msgType": {?MsgType}, - "seqNum": {?ObserveSeqNum}, - "data": { - "code": {?StatusCode}, - "codeMsg": {?CodeMsg}, - "reqPath": {?RequestPath}, - "content": [ - { - "path": {?ResourcePath}, - "value": {?Value} - } - ] - } - } - ``` - - {?MsgType}: String, must be "notify" - - {?ObserveSeqNum}: Number, value of "Observe" option in CoAP message - - "content": same to the "content" field contains in the response of "read" command - -## Feature limitations - -- emqx_lwm2m implements LwM2M gateway to EMQX, not a full-featured and independent LwM2M server. -- emqx_lwm2m does not include LwM2M bootstrap server. -- emqx_lwm2m supports UDP binding, no SMS binding yet. -- Firmware object is not fully supported now since mqtt to coap block-wise transfer is not available. -- Object Versioning is not supported now. - -## DTLS - -emqx-lwm2m support DTLS to secure UDP data. - -Please config lwm2m.certfile and lwm2m.keyfile in emqx_lwm2m.conf. If certfile or keyfile are invalid, DTLS will be turned off and you could read a error message in the log. - -## License - -Apache License Version 2.0 - -## Author - -EMQX-Men Team. +More documentations: [LwM2M Gateway](https://www.emqx.io/docs/en/v5.0/gateway/lwm2m.html) diff --git a/apps/emqx_mqttsn/LICENSE b/apps/emqx_mqttsn/LICENSE deleted file mode 100644 index 5a5418f0f..000000000 --- a/apps/emqx_mqttsn/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, JianBo He . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_stomp/LICENSE b/apps/emqx_stomp/LICENSE deleted file mode 100644 index 5a5418f0f..000000000 --- a/apps/emqx_stomp/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, JianBo He . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - From aea870f319b2f07cbcddb0f21e5f351bb158e25c Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 5 Apr 2023 14:40:18 +0200 Subject: [PATCH 022/279] feat: add `/rule_engine` API endpoint --- .../src/emqx_rule_api_schema.erl | 3 + .../src/emqx_rule_engine_api.erl | 81 ++++++++++++++++--- .../test/emqx_rule_engine_api_SUITE.erl | 13 +++ changes/ce/feat-10336.en.md | 1 + rel/i18n/emqx_rule_api_schema.hocon | 11 +++ rel/i18n/emqx_rule_engine_api.hocon | 37 ++++++--- 6 files changed, 125 insertions(+), 21 deletions(-) create mode 100644 changes/ce/feat-10336.en.md diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 23c2aab50..188dd10f9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -48,12 +48,15 @@ check_params(Params, Tag) -> roots() -> [ + {"rule_engine", sc(ref("rule_engine"), #{desc => ?DESC("root_rule_engine")})}, {"rule_creation", sc(ref("rule_creation"), #{desc => ?DESC("root_rule_creation")})}, {"rule_info", sc(ref("rule_info"), #{desc => ?DESC("root_rule_info")})}, {"rule_events", sc(ref("rule_events"), #{desc => ?DESC("root_rule_events")})}, {"rule_test", sc(ref("rule_test"), #{desc => ?DESC("root_rule_test")})} ]. +fields("rule_engine") -> + emqx_rule_engine_schema:fields("rule_engine"); fields("rule_creation") -> emqx_rule_engine_schema:fields("rules"); fields("rule_info") -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 106693a0a..251ba053d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -32,6 +32,7 @@ %% API callbacks -export([ + '/rule_engine'/2, '/rule_events'/2, '/rule_test'/2, '/rules'/2, @@ -41,7 +42,7 @@ ]). %% query callback --export([qs2ms/2, run_fuzzy_match/2, format_rule_resp/1]). +-export([qs2ms/2, run_fuzzy_match/2, format_rule_info_resp/1]). -define(ERR_BADARGS(REASON), begin R0 = err_msg(REASON), @@ -134,6 +135,7 @@ api_spec() -> paths() -> [ + "/rule_engine", "/rule_events", "/rule_test", "/rules", @@ -145,6 +147,9 @@ paths() -> error_schema(Code, Message) when is_atom(Code) -> emqx_dashboard_swagger:error_codes([Code], list_to_binary(Message)). +rule_engine_schema() -> + ref(emqx_rule_api_schema, "rule_engine"). + rule_creation_schema() -> ref(emqx_rule_api_schema, "rule_creation"). @@ -184,7 +189,7 @@ schema("/rules") -> responses => #{ 200 => [ - {data, mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})}, + {data, mk(array(rule_info_schema()), #{desc => ?DESC("api1_resp")})}, {meta, mk(ref(emqx_dashboard_swagger, meta), #{})} ], 400 => error_schema('BAD_REQUEST', "Invalid Parameters") @@ -289,6 +294,26 @@ schema("/rule_test") -> 200 => <<"Rule Test Pass">> } } + }; +schema("/rule_engine") -> + #{ + 'operationId' => '/rule_engine', + get => #{ + tags => [<<"rules">>], + description => ?DESC("api9"), + responses => #{ + 200 => rule_engine_schema() + } + }, + put => #{ + tags => [<<"rules">>], + description => ?DESC("api10"), + 'requestBody' => rule_engine_schema(), + responses => #{ + 200 => rule_engine_schema(), + 400 => error_schema('BAD_REQUEST', "Invalid request") + } + } }. param_path_id() -> @@ -309,7 +334,7 @@ param_path_id() -> QueryString, ?RULE_QS_SCHEMA, fun ?MODULE:qs2ms/2, - fun ?MODULE:format_rule_resp/1 + fun ?MODULE:format_rule_info_resp/1 ) of {error, page_limit_invalid} -> @@ -331,7 +356,7 @@ param_path_id() -> case emqx_conf:update(ConfPath, Params, #{override_to => cluster}) of {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> [Rule] = get_one_rule(AllRules, Id), - {201, format_rule_resp(Rule)}; + {201, format_rule_info_resp(Rule)}; {error, Reason} -> ?SLOG(error, #{ msg => "create_rule_failed", @@ -362,7 +387,7 @@ param_path_id() -> '/rules/:id'(get, #{bindings := #{id := Id}}) -> case emqx_rule_engine:get_rule(Id) of {ok, Rule} -> - {200, format_rule_resp(Rule)}; + {200, format_rule_info_resp(Rule)}; not_found -> {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} end; @@ -372,7 +397,7 @@ param_path_id() -> case emqx_conf:update(ConfPath, Params, #{override_to => cluster}) of {ok, #{post_config_update := #{emqx_rule_engine := AllRules}}} -> [Rule] = get_one_rule(AllRules, Id), - {200, format_rule_resp(Rule)}; + {200, format_rule_info_resp(Rule)}; {error, Reason} -> ?SLOG(error, #{ msg => "update_rule_failed", @@ -419,6 +444,20 @@ param_path_id() -> {404, #{code => 'NOT_FOUND', message => <<"Rule Id Not Found">>}} end. +'/rule_engine'(get, _Params) -> + {200, format_rule_engine_resp(emqx_conf:get([rule_engine]))}; +'/rule_engine'(put, #{body := Params}) -> + case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of + {ok, #{config := Config}} -> + {200, format_rule_engine_resp(Config)}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "update_rule_engine_failed", + reason => Reason + }), + {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} + end. + %%------------------------------------------------------------------------------ %% Internal functions %%------------------------------------------------------------------------------ @@ -440,11 +479,11 @@ encode_nested_error(RuleError, Reason) -> {RuleError, Reason} end. -format_rule_resp(Rules) when is_list(Rules) -> - [format_rule_resp(R) || R <- Rules]; -format_rule_resp({Id, Rule}) -> - format_rule_resp(Rule#{id => Id}); -format_rule_resp(#{ +format_rule_info_resp(Rules) when is_list(Rules) -> + [format_rule_info_resp(R) || R <- Rules]; +format_rule_info_resp({Id, Rule}) -> + format_rule_info_resp(Rule#{id => Id}); +format_rule_info_resp(#{ id := Id, name := Name, created_at := CreatedAt, @@ -465,6 +504,26 @@ format_rule_resp(#{ description => Descr }. +format_rule_engine_resp(#{rules := Rules} = Config) -> + Config#{rules => maps:map(fun format_rule_resp/2, Rules)}. + +format_rule_resp(_Id, #{ + name := Name, + metadata := MetaData = #{created_at := CreatedAt}, + actions := Action, + sql := SQL, + enable := Enable, + description := Descr +}) -> + #{ + name => Name, + actions => format_action(Action), + sql => SQL, + enable => Enable, + metadata => MetaData#{created_at => format_datetime(CreatedAt, millisecond)}, + description => Descr + }. + format_datetime(Timestamp, Unit) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index d89bc2651..47dce98b1 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -281,3 +281,16 @@ test_rule_params(Sql, Payload) -> <<"sql">> => Sql } }. + +t_rule_engine(_) -> + {200, _} = emqx_rule_engine_api:'/rule_engine'(get, foo), + {200, #{ + jq_function_default_timeout := 12000, + jq_implementation_module := jq_port + }} = emqx_rule_engine_api:'/rule_engine'(put, #{ + body => #{ + <<"jq_function_default_timeout">> => <<"12s">>, + <<"jq_implementation_module">> => <<"jq_port">> + } + }), + {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{body => #{<<"something">> => <<"weird">>}}). diff --git a/changes/ce/feat-10336.en.md b/changes/ce/feat-10336.en.md new file mode 100644 index 000000000..5e6039f9b --- /dev/null +++ b/changes/ce/feat-10336.en.md @@ -0,0 +1 @@ +Add `/rule_engine` API endpoint to manage configuration of rule engine. diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index f9b344666..e2c8532e7 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -638,6 +638,17 @@ emqx_rule_api_schema { } } + root_rule_engine { + desc { + en: "Rule engine configuration schema" + zh: "规则引擎配置模式" + } + label: { + en: "Configuration Schema" + zh: "配置模式" + } + } + root_rule_creation { desc { en: "Schema for creating rules" diff --git a/rel/i18n/emqx_rule_engine_api.hocon b/rel/i18n/emqx_rule_engine_api.hocon index 39fc3186c..181c1ba40 100644 --- a/rel/i18n/emqx_rule_engine_api.hocon +++ b/rel/i18n/emqx_rule_engine_api.hocon @@ -50,7 +50,16 @@ emqx_rule_engine_api { zh: "根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配" } } - + api1_resp { + desc { + en: "List of rules" + zh: "列出所有规则" + } + label: { + en: "List Rules" + zh: "列出所有规则" + } + } api2 { desc { en: "Create a new rule using given Id" @@ -116,7 +125,6 @@ emqx_rule_engine_api { zh: "删除集群规则" } } - api7 { desc { en: "Reset a rule metrics" @@ -127,7 +135,6 @@ emqx_rule_engine_api { zh: "重置规则计数" } } - api8 { desc { en: "Test a rule" @@ -138,14 +145,24 @@ emqx_rule_engine_api { zh: "测试规则" } } - desc9 { + api9 { desc { - en: "List of rules" - zh: "列出所有规则" + en: "Get rule engine configuration" + zh: "获取规则引擎配置" } - label: { - en: "List Rules" - zh: "列出所有规则" + label { + en: "Get configuration" + zh: "获取配置" + } + } + api10 { + desc { + en: "Update rule engine configuration" + zh: "更新规则引擎配置" + } + label { + en: "Update configuration" + zh: "更新配置" } - } + } } From 5cd8865a9335ead6beaf9950092be15c7482d286 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 5 Apr 2023 15:34:42 +0200 Subject: [PATCH 023/279] fix: deprecate and hide jq_implementation_module --- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 2281eea53..33dedae9a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -61,7 +61,9 @@ fields("rule_engine") -> #{ default => jq_nif, mapping => "jq.jq_implementation_module", - desc => ?DESC("rule_engine_jq_implementation_module") + desc => ?DESC("rule_engine_jq_implementation_module"), + deprecated => {since, "v5.0.22"}, + importance => ?IMPORTANCE_HIDDEN } )} ]; From 9cfe9cc709731b97df9ccb38dcee5b127110f491 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 5 Apr 2023 16:28:54 +0200 Subject: [PATCH 024/279] fix: don't allow `rules` to be set from `/rule_engine` --- .../src/emqx_rule_api_schema.erl | 2 +- .../src/emqx_rule_engine_api.erl | 51 +++++++++++-------- .../src/emqx_rule_engine_schema.erl | 12 +++-- .../test/emqx_rule_engine_api_SUITE.erl | 9 +++- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 188dd10f9..72fd085a3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -56,7 +56,7 @@ roots() -> ]. fields("rule_engine") -> - emqx_rule_engine_schema:fields("rule_engine"); + emqx_rule_engine_schema:fields("rule_engine_api"); fields("rule_creation") -> emqx_rule_engine_schema:fields("rules"); fields("rule_info") -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 251ba053d..b138d992e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -447,16 +447,20 @@ param_path_id() -> '/rule_engine'(get, _Params) -> {200, format_rule_engine_resp(emqx_conf:get([rule_engine]))}; '/rule_engine'(put, #{body := Params}) -> - case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of - {ok, #{config := Config}} -> - {200, format_rule_engine_resp(Config)}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "update_rule_engine_failed", - reason => Reason - }), - {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} - end. + ?CHECK_PARAMS( + Params, + rule_engine, + case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of + {ok, #{config := Config}} -> + {200, format_rule_engine_resp(Config)}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "update_rule_engine_failed", + reason => Reason + }), + {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} + end + ). %%------------------------------------------------------------------------------ %% Internal functions @@ -507,22 +511,29 @@ format_rule_info_resp(#{ format_rule_engine_resp(#{rules := Rules} = Config) -> Config#{rules => maps:map(fun format_rule_resp/2, Rules)}. -format_rule_resp(_Id, #{ - name := Name, - metadata := MetaData = #{created_at := CreatedAt}, - actions := Action, - sql := SQL, - enable := Enable, - description := Descr -}) -> +format_rule_resp( + _Id, #{ + name := Name, + actions := Action, + sql := SQL, + enable := Enable, + description := Descr + } = Rule +) -> + Format = #{ name => Name, actions => format_action(Action), sql => SQL, enable => Enable, - metadata => MetaData#{created_at => format_datetime(CreatedAt, millisecond)}, description => Descr - }. + }, + case Rule of + #{metadata := MetaData = #{created_at := CreatedAt}} -> + Format#{metadata => MetaData#{created_at => format_datetime(CreatedAt, millisecond)}}; + _ -> + Format + end. format_datetime(Timestamp, Unit) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 33dedae9a..57b64cb49 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -40,13 +40,17 @@ tags() -> roots() -> ["rule_engine"]. fields("rule_engine") -> + fields("rule_engine_api") ++ + [ + {rules, + ?HOCON(hoconsc:map("id", ?R_REF("rules")), #{ + desc => ?DESC("rule_engine_rules"), default => #{} + })} + ]; +fields("rule_engine_api") -> [ {ignore_sys_message, ?HOCON(boolean(), #{default => true, desc => ?DESC("rule_engine_ignore_sys_message")})}, - {rules, - ?HOCON(hoconsc:map("id", ?R_REF("rules")), #{ - desc => ?DESC("rule_engine_rules"), default => #{} - })}, {jq_function_default_timeout, ?HOCON( emqx_schema:duration_ms(), diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index 47dce98b1..e59b5c6df 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -285,12 +285,17 @@ test_rule_params(Sql, Payload) -> t_rule_engine(_) -> {200, _} = emqx_rule_engine_api:'/rule_engine'(get, foo), {200, #{ - jq_function_default_timeout := 12000, - jq_implementation_module := jq_port + %, + jq_function_default_timeout := 12000 + % hidden! jq_implementation_module := jq_port }} = emqx_rule_engine_api:'/rule_engine'(put, #{ body => #{ <<"jq_function_default_timeout">> => <<"12s">>, <<"jq_implementation_module">> => <<"jq_port">> } }), + SomeRule = #{<<"sql">> => <<"SELECT * FROM \"t/#\"">>}, + {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{ + body => #{<<"rules">> => #{<<"some_rule">> => SomeRule}} + }), {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{body => #{<<"something">> => <<"weird">>}}). From 576d1524bb411c1987b80ad58a1c2bfb51f96c93 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 5 Apr 2023 16:29:17 +0200 Subject: [PATCH 025/279] style: fix wording of i18n files --- rel/i18n/emqx_rule_api_schema.hocon | 8 ++++---- rel/i18n/emqx_rule_engine_api.hocon | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index e2c8532e7..3c2b3e7e4 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -640,12 +640,12 @@ emqx_rule_api_schema { root_rule_engine { desc { - en: "Rule engine configuration schema" - zh: "规则引擎配置模式" + en: "Rule engine configurations. This API can be used to change EMQX rule engine settings. But not for the rules. To list, create, or update rules, call the '/rules' API instead." + zh: "规则引擎配置。该 API 可用于查看和修改规则引擎相关的一些设置。但不可用于规则，请调用 '/rules' API 来对规则进行操作。" } label: { - en: "Configuration Schema" - zh: "配置模式" + en: "Rule engine configuration" + zh: "规则引擎配置" } } diff --git a/rel/i18n/emqx_rule_engine_api.hocon b/rel/i18n/emqx_rule_engine_api.hocon index 181c1ba40..7be0588c9 100644 --- a/rel/i18n/emqx_rule_engine_api.hocon +++ b/rel/i18n/emqx_rule_engine_api.hocon @@ -147,8 +147,8 @@ emqx_rule_engine_api { } api9 { desc { - en: "Get rule engine configuration" - zh: "获取规则引擎配置" + en: "Get rule engine configuration." + zh: "获取规则引擎配置。" } label { en: "Get configuration" @@ -157,8 +157,8 @@ emqx_rule_engine_api { } api10 { desc { - en: "Update rule engine configuration" - zh: "更新规则引擎配置" + en: "Update rule engine configuration." + zh: "更新规则引擎配置。" } label { en: "Update configuration" From b799af1f716f50e906d11f938dd2177144821d77 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Wed, 5 Apr 2023 16:46:19 +0200 Subject: [PATCH 026/279] fix: don't create virtual type "rule_engine_api" --- .../src/emqx_rule_api_schema.erl | 2 +- .../src/emqx_rule_engine_schema.erl | 54 ++++++++++--------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 72fd085a3..8a8822044 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -56,7 +56,7 @@ roots() -> ]. fields("rule_engine") -> - emqx_rule_engine_schema:fields("rule_engine_api"); + emqx_rule_engine_schema:rule_engine_settings(); fields("rule_creation") -> emqx_rule_engine_schema:fields("rules"); fields("rule_info") -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 57b64cb49..5b205f355 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -27,7 +27,8 @@ roots/0, fields/1, desc/1, - post_config_update/5 + post_config_update/5, + rule_engine_settings/0 ]). -export([validate_sql/1]). @@ -40,37 +41,13 @@ tags() -> roots() -> ["rule_engine"]. fields("rule_engine") -> - fields("rule_engine_api") ++ + rule_engine_settings() ++ [ {rules, ?HOCON(hoconsc:map("id", ?R_REF("rules")), #{ desc => ?DESC("rule_engine_rules"), default => #{} })} ]; -fields("rule_engine_api") -> - [ - {ignore_sys_message, - ?HOCON(boolean(), #{default => true, desc => ?DESC("rule_engine_ignore_sys_message")})}, - {jq_function_default_timeout, - ?HOCON( - emqx_schema:duration_ms(), - #{ - default => <<"10s">>, - desc => ?DESC("rule_engine_jq_function_default_timeout") - } - )}, - {jq_implementation_module, - ?HOCON( - hoconsc:enum([jq_nif, jq_port]), - #{ - default => jq_nif, - mapping => "jq.jq_implementation_module", - desc => ?DESC("rule_engine_jq_implementation_module"), - deprecated => {since, "v5.0.22"}, - importance => ?IMPORTANCE_HIDDEN - } - )} - ]; fields("rules") -> [ rule_name(), @@ -233,6 +210,31 @@ actions() -> qos() -> ?UNION([emqx_schema:qos(), binary()]). +rule_engine_settings() -> + [ + {ignore_sys_message, + ?HOCON(boolean(), #{default => true, desc => ?DESC("rule_engine_ignore_sys_message")})}, + {jq_function_default_timeout, + ?HOCON( + emqx_schema:duration_ms(), + #{ + default => <<"10s">>, + desc => ?DESC("rule_engine_jq_function_default_timeout") + } + )}, + {jq_implementation_module, + ?HOCON( + hoconsc:enum([jq_nif, jq_port]), + #{ + default => jq_nif, + mapping => "jq.jq_implementation_module", + desc => ?DESC("rule_engine_jq_implementation_module"), + deprecated => {since, "v5.0.22"}, + importance => ?IMPORTANCE_HIDDEN + } + )} + ]. + validate_sql(Sql) -> case emqx_rule_sqlparser:parse(Sql) of {ok, _Result} -> ok; From bbb3fdb49ecac3e24a45b0850f2fddad426a7893 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 6 Apr 2023 16:57:02 +0800 Subject: [PATCH 027/279] fix: make emqx_api_lib compatible --- apps/emqx/src/emqx_api_lib.erl | 32 +++++++++++-------- .../src/emqx_dashboard_monitor_api.erl | 2 +- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/apps/emqx/src/emqx_api_lib.erl b/apps/emqx/src/emqx_api_lib.erl index 8c49c57c3..f1d65f350 100644 --- a/apps/emqx/src/emqx_api_lib.erl +++ b/apps/emqx/src/emqx_api_lib.erl @@ -28,17 +28,17 @@ %%-------------------------------------------------------------------- %% exported API %%-------------------------------------------------------------------- --spec with_node(binary(), fun((atom()) -> {ok, term()} | {error, term()})) -> +-spec with_node(binary() | atom(), fun((atom()) -> {ok, term()} | {error, term()})) -> ?OK(term()) | ?NOT_FOUND(binary()) | ?BAD_REQUEST(term()). -with_node(BinNode, Fun) -> - case lookup_node(BinNode) of +with_node(Node0, Fun) -> + case lookup_node(Node0) of {ok, Node} -> handle_result(Fun(Node)); not_found -> - ?NODE_NOT_FOUND(BinNode) + ?NODE_NOT_FOUND(Node0) end. --spec with_node_or_cluster(binary(), fun((atom()) -> {ok, term()} | {error, term()})) -> +-spec with_node_or_cluster(binary() | atom(), fun((atom()) -> {ok, term()} | {error, term()})) -> ?OK(term()) | ?NOT_FOUND(iolist()) | ?BAD_REQUEST(term()). with_node_or_cluster(<<"all">>, Fun) -> handle_result(Fun(all)); @@ -49,18 +49,24 @@ with_node_or_cluster(Node, Fun) -> %% Internal %%-------------------------------------------------------------------- --spec lookup_node(binary()) -> {ok, atom()} | not_found. -lookup_node(BinNode) -> +-spec lookup_node(atom() | binary()) -> {ok, atom()} | not_found. +lookup_node(BinNode) when is_binary(BinNode) -> case emqx_misc:safe_to_existing_atom(BinNode, utf8) of {ok, Node} -> - case lists:member(Node, mria:running_nodes()) of - true -> - {ok, Node}; - false -> - not_found - end; + is_running_node(Node); _Error -> not_found + end; +lookup_node(Node) when is_atom(Node) -> + is_running_node(Node). + +-spec is_running_node(atom()) -> {ok, atom()} | not_found. +is_running_node(Node) -> + case lists:member(Node, mria:running_nodes()) of + true -> + {ok, Node}; + false -> + not_found end. handle_result({ok, Result}) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index f0b5ef574..c2dd4a9dd 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -133,7 +133,7 @@ dashboard_samplers_fun(Latest) -> end. monitor_current(get, #{bindings := []}) -> - with_node(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); + emqx_api_lib:with_node_or_cluster(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); monitor_current(get, #{bindings := Bindings}) -> RawNode = maps:get(node, Bindings, <<"all">>), emqx_api_lib:with_node_or_cluster(RawNode, fun current_rate/1). From 11a7770d163f5ade6b3739ad342f2969793f3005 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 6 Apr 2023 11:57:07 +0200 Subject: [PATCH 028/279] chore: bump version to e5.0.2-rc.5 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 76920928b..6394d940e 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.21"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.2-rc.4"). +-define(EMQX_RELEASE_EE, "5.0.2-rc.5"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 4c24b08244b27afa59f25c8184024ba734d17f28 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 6 Apr 2023 15:30:45 -0300 Subject: [PATCH 029/279] fix(rule_action): fix metrics for bridges returning `async_return` Kafka Producer, when called asynchronously, will return `{async_return, {ok, pid()}}`, which currently counts as an unknown failure. --- apps/emqx_rule_engine/src/emqx_rule_runtime.erl | 6 ++++-- .../test/emqx_bridge_impl_kafka_producer_SUITE.erl | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 153832246..51491df53 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -508,8 +508,6 @@ nested_put(Alias, Val, Columns0) -> emqx_rule_maps:nested_put(Alias, Val, Columns). -define(IS_RES_DOWN(R), R == stopped; R == not_connected; R == not_found). -inc_action_metrics(ok, RuleId) -> - emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success'); inc_action_metrics({error, {recoverable_error, _}}, RuleId) -> emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.failed.out_of_service'); inc_action_metrics(?RESOURCE_ERROR_M(R, _), RuleId) when ?IS_RES_DOWN(R) -> @@ -525,6 +523,10 @@ inc_action_metrics(R, RuleId) -> emqx_metrics_worker:inc(rule_metrics, RuleId, 'actions.success') end. +is_ok_result(ok) -> + true; +is_ok_result({async_return, R}) -> + is_ok_result(R); is_ok_result(R) when is_tuple(R) -> ok == erlang:element(1, R); is_ok_result(_) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index 9e32f818d..9b6ac05a7 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -309,7 +309,7 @@ kafka_bridge_rest_api_helper(Config) -> AtomsAfter = erlang:system_info(atom_count), ?assertEqual(AtomsBefore, AtomsAfter), %% Create a rule that uses the bridge - {ok, 201, _Rule} = http_post( + {ok, 201, Rule} = http_post( ["rules"], #{ <<"name">> => <<"kafka_bridge_rest_api_helper_rule">>, @@ -318,6 +318,7 @@ kafka_bridge_rest_api_helper(Config) -> <<"sql">> => <<"SELECT * from \"kafka_bridge_topic/#\"">> } ), + #{<<"id">> := RuleId} = emqx_json:decode(Rule, [return_maps]), %% counters should be empty before ?assertEqual(0, emqx_resource_metrics:matched_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:success_get(ResourceId)), @@ -346,6 +347,8 @@ kafka_bridge_rest_api_helper(Config) -> %% Check crucial counters and gauges ?assertEqual(1, emqx_resource_metrics:matched_get(ResourceId)), ?assertEqual(1, emqx_resource_metrics:success_get(ResourceId)), + ?assertEqual(1, emqx_metrics_worker:get(rule_metrics, RuleId, 'actions.success')), + ?assertEqual(0, emqx_metrics_worker:get(rule_metrics, RuleId, 'actions.failed')), ?assertEqual(0, emqx_resource_metrics:dropped_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:failed_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:inflight_get(ResourceId)), From 33100ecca68a9f16c669aae6d3367faa619b28bb Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 4 Apr 2023 08:55:43 -0300 Subject: [PATCH 030/279] feat: implement schema registry for 5.0 (avro) Part of https://emqx.atlassian.net/browse/EMQX-9251 This ports part of the Schema Registry app from 4.x to 5.0. Here, only support for Avro is added. Subsequent PRs will follow to add support for other formats. --- apps/emqx/src/emqx_schema.erl | 16 +- apps/emqx/test/emqx_common_test_helpers.erl | 43 +- .../src/emqx_dashboard_swagger.erl | 2 + apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 33 +- changes/ee/feat-10337.en.md | 3 + lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src | 2 +- .../emqx_ee_conf/src/emqx_ee_conf_schema.erl | 2 +- lib-ee/emqx_ee_schema_registry/.gitignore | 19 + lib-ee/emqx_ee_schema_registry/README.md | 69 +++ .../etc/emqx_ee_schema_registry.conf | 0 .../include/emqx_ee_schema_registry.hrl | 39 ++ lib-ee/emqx_ee_schema_registry/rebar.config | 12 + .../src/emqx_ee_schema_registry.app.src | 15 + .../src/emqx_ee_schema_registry.erl | 242 ++++++++++ .../src/emqx_ee_schema_registry_app.erl | 19 + .../src/emqx_ee_schema_registry_http_api.erl | 251 ++++++++++ .../src/emqx_ee_schema_registry_schema.erl | 127 +++++ .../src/emqx_ee_schema_registry_serde.erl | 70 +++ .../src/emqx_ee_schema_registry_sup.erl | 43 ++ .../test/emqx_ee_schema_registry_SUITE.erl | 433 ++++++++++++++++++ ...emqx_ee_schema_registry_http_api_SUITE.erl | 250 ++++++++++ .../emqx_ee_schema_registry_serde_SUITE.erl | 121 +++++ mix.exs | 5 +- rebar.config | 2 + rebar.config.erl | 3 +- rel/i18n/emqx_bridge_api.hocon | 2 +- .../emqx_ee_schema_registry_http_api.hocon | 69 +++ rel/i18n/emqx_ee_schema_registry_schema.hocon | 78 ++++ scripts/spellcheck/dicts/emqx.txt | 1 + 29 files changed, 1941 insertions(+), 30 deletions(-) create mode 100644 changes/ee/feat-10337.en.md create mode 100644 lib-ee/emqx_ee_schema_registry/.gitignore create mode 100644 lib-ee/emqx_ee_schema_registry/README.md create mode 100644 lib-ee/emqx_ee_schema_registry/etc/emqx_ee_schema_registry.conf create mode 100644 lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl create mode 100644 lib-ee/emqx_ee_schema_registry/rebar.config create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl create mode 100644 lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl create mode 100644 lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl create mode 100644 lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl create mode 100644 lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl create mode 100644 rel/i18n/emqx_ee_schema_registry_http_api.hocon create mode 100644 rel/i18n/emqx_ee_schema_registry_schema.hocon diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 6bfff38d3..20018b2d5 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -44,6 +44,7 @@ -type port_number() :: 1..65536. -type server_parse_option() :: #{default_port => port_number(), no_port => boolean()}. -type url() :: binary(). +-type json_binary() :: binary(). -typerefl_from_string({duration/0, emqx_schema, to_duration}). -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}). @@ -58,6 +59,7 @@ -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). -typerefl_from_string({url/0, emqx_schema, to_url}). +-typerefl_from_string({json_binary/0, emqx_schema, to_json_binary}). -export([ validate_heap_size/1, @@ -84,7 +86,8 @@ to_ip_port/1, to_erl_cipher_suite/1, to_comma_separated_atoms/1, - to_url/1 + to_url/1, + to_json_binary/1 ]). -export([ @@ -112,7 +115,8 @@ ip_port/0, cipher/0, comma_separated_atoms/0, - url/0 + url/0, + json_binary/0 ]). -export([namespace/0, roots/0, roots/1, fields/1, desc/1, tags/0]). @@ -2576,6 +2580,14 @@ to_url(Str) -> Error end. +to_json_binary(Str) -> + case emqx_json:safe_decode(Str) of + {ok, _} -> + {ok, iolist_to_binary(Str)}; + Error -> + Error + end. + to_bar_separated_list(Str) -> {ok, string:tokens(Str, "| ")}. diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 077ebe138..406183094 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -676,7 +676,8 @@ start_slave(Name, Opts) when is_map(Opts) -> ] ); slave -> - slave:start_link(host(), Name, ebin_path()) + Env = " -env HOCON_ENV_OVERRIDE_PREFIX EMQX_", + slave:start_link(host(), Name, ebin_path() ++ Env) end end, case DoStart() of @@ -749,6 +750,20 @@ setup_node(Node, Opts) when is_map(Opts) -> %% `emqx_conf' app and correctly catch up the config. StartAutocluster = maps:get(start_autocluster, Opts, false), + ct:pal( + "setting up node ~p:\n ~p", + [ + Node, + #{ + start_autocluster => StartAutocluster, + load_apps => LoadApps, + apps => Apps, + env => Env, + start_apps => StartApps + } + ] + ), + %% Load env before doing anything to avoid overriding [ok = erpc:call(Node, ?MODULE, load, [App]) || App <- [gen_rpc, ekka, mria, emqx | LoadApps]], @@ -773,10 +788,7 @@ setup_node(Node, Opts) when is_map(Opts) -> end, %% Setting env before starting any applications - [ - ok = rpc:call(Node, application, set_env, [Application, Key, Value]) - || {Application, Key, Value} <- Env - ], + set_envs(Node, Env), %% Here we start the apps EnvHandlerForRpc = @@ -794,8 +806,9 @@ setup_node(Node, Opts) when is_map(Opts) -> node(), integer_to_list(erlang:unique_integer()) ]), + Cookie = atom_to_list(erlang:get_cookie()), os:putenv("EMQX_NODE__DATA_DIR", NodeDataDir), - os:putenv("EMQX_NODE__COOKIE", atom_to_list(erlang:get_cookie())), + os:putenv("EMQX_NODE__COOKIE", Cookie), emqx_config:init_load(SchemaMod), os:unsetenv("EMQX_NODE__DATA_DIR"), os:unsetenv("EMQX_NODE__COOKIE"), @@ -826,7 +839,15 @@ setup_node(Node, Opts) when is_map(Opts) -> ok; _ -> StartAutocluster andalso - (ok = rpc:call(Node, emqx_machine_boot, start_autocluster, [])), + begin + %% Note: we need to re-set the env because + %% starting the apps apparently make some of them + %% to be lost... This is particularly useful for + %% setting extra apps to be restarted after + %% joining. + set_envs(Node, Env), + ok = erpc:call(Node, emqx_machine_boot, start_autocluster, []) + end, case rpc:call(Node, ekka, join, [JoinTo]) of ok -> ok; @@ -883,6 +904,14 @@ merge_opts(Opts1, Opts2) -> Opts2 ). +set_envs(Node, Env) -> + lists:foreach( + fun({Application, Key, Value}) -> + ok = rpc:call(Node, application, set_env, [Application, Key, Value]) + end, + Env + ). + erl_flags() -> %% One core and redirecting logs to master "+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path(). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e2872c0d7..eb7f6c741 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -755,6 +755,8 @@ typename_to_spec("initial()", _Mod) -> #{type => string, example => <<"0MB">>}; typename_to_spec("bucket_name()", _Mod) -> #{type => string, example => <<"retainer">>}; +typename_to_spec("json_binary()", _Mod) -> + #{type => string, example => <<"{\"a\": [1,true]}">>}; typename_to_spec(Name, Mod) -> Spec = range(Name), Spec1 = remote_module_type(Spec, Name, Mod), diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index b8bfeb84c..79e0406c1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -1097,26 +1097,27 @@ date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> %% Here the emqx_rule_funcs module acts as a proxy, forwarding %% the function handling to the worker module. %% @end -% '$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> -% emqx_schema_parser:decode(SchemaId, Data, MoreArgs); -% '$handle_undefined_function'(schema_decode, Args) -> -% error({args_count_error, {schema_decode, Args}}); - -% '$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> -% emqx_schema_parser:encode(SchemaId, Term, MoreArgs); -% '$handle_undefined_function'(schema_encode, Args) -> -% error({args_count_error, {schema_encode, Args}}); - -% '$handle_undefined_function'(sprintf, [Format|Args]) -> -% erlang:apply(fun sprintf_s/2, [Format, Args]); - -% '$handle_undefined_function'(Fun, Args) -> -% error({sql_function_not_supported, function_literal(Fun, Args)}). - +-if(?EMQX_RELEASE_EDITION == ee). +%% EE +'$handle_undefined_function'(schema_decode, [SchemaId, Data | MoreArgs]) -> + emqx_ee_schema_registry_serde:decode(SchemaId, Data, MoreArgs); +'$handle_undefined_function'(schema_decode, Args) -> + error({args_count_error, {schema_decode, Args}}); +'$handle_undefined_function'(schema_encode, [SchemaId, Term | MoreArgs]) -> + emqx_ee_schema_registry_serde:encode(SchemaId, Term, MoreArgs); +'$handle_undefined_function'(schema_encode, Args) -> + error({args_count_error, {schema_encode, Args}}); '$handle_undefined_function'(sprintf, [Format | Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> error({sql_function_not_supported, function_literal(Fun, Args)}). +-else. +%% CE +'$handle_undefined_function'(sprintf, [Format | Args]) -> + erlang:apply(fun sprintf_s/2, [Format, Args]); +'$handle_undefined_function'(Fun, Args) -> + error({sql_function_not_supported, function_literal(Fun, Args)}). +-endif. map_path(Key) -> {path, [{key, P} || P <- string:split(Key, ".", all)]}. diff --git a/changes/ee/feat-10337.en.md b/changes/ee/feat-10337.en.md new file mode 100644 index 000000000..299933351 --- /dev/null +++ b/changes/ee/feat-10337.en.md @@ -0,0 +1,3 @@ +Add schema registry feature. + +With schema registry, one can encode and decode special serialization formats in payloads when transforming messages in Rule Engine. Currently, only [Apache Avro](https://avro.apache.org/) is supported. diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src index 324e7e308..771fdcb27 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_conf, [ {description, "EMQX Enterprise Edition configuration schema"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl index 5137574e3..7bf41deb5 100644 --- a/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl +++ b/lib-ee/emqx_ee_conf/src/emqx_ee_conf_schema.erl @@ -8,7 +8,7 @@ -export([namespace/0, roots/0, fields/1, translations/0, translation/1]). --define(EE_SCHEMA_MODULES, [emqx_license_schema]). +-define(EE_SCHEMA_MODULES, [emqx_license_schema, emqx_ee_schema_registry_schema]). namespace() -> emqx_conf_schema:namespace(). diff --git a/lib-ee/emqx_ee_schema_registry/.gitignore b/lib-ee/emqx_ee_schema_registry/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/lib-ee/emqx_ee_schema_registry/README.md b/lib-ee/emqx_ee_schema_registry/README.md new file mode 100644 index 000000000..9f477208c --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/README.md @@ -0,0 +1,69 @@ +# EMQX Schema Registry + +EMQX Schema Registry for managing various of schemas for +decoding/encoding messages. + +To use schema in rule engine, a schema name should be passed to the +SQL functions that decode/encode data, like: + +```sql +SELECT + schema_decode('sensor_notify', payload) as payload +FROM + "message.publish" +WHERE + topic = 't/1' +``` + +## Using schema registry with rule engine + +``` + +---------------------------+ + | | + Events/Msgs | | Events/Msgs + --------------------> EMQX |------------------> + | | + | | + +-------------|-------------+ + | + HOOK | + | + +-------------v-------------+ +----------+ + | | Data | | + | Rule Engine ------------- Backends | + | | | | + +------|-------------|------+ +----------+ + |^ |^ + Decode|| ||Encode + || || + +------v|------------v|-----+ + | | + | Schema Registry | + | | + +---------------------------+ +``` + +## Architecture + +``` + | | + Decode | [APIs] | Encode + | | + | | [Registry] + +------v--------------v------+ + REGISTER SCHEMA | | + -------------------> | +--------+ + | | | | +[Management APIs] | Schema Registry ------ Schema | + | | | | + -------------------> | +--------+ + LOAD PARSERS | | + +----------------------------+ + / | \ + +---/---+ +---|----+ +---\---+ + | | | | | | + [Decoders] | Avro | |ProtoBuf| |Others | + | | | | | | + +-------+ +--------+ +-------+ + +``` diff --git a/lib-ee/emqx_ee_schema_registry/etc/emqx_ee_schema_registry.conf b/lib-ee/emqx_ee_schema_registry/etc/emqx_ee_schema_registry.conf new file mode 100644 index 000000000..e69de29bb diff --git a/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl b/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl new file mode 100644 index 000000000..af49db6dd --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl @@ -0,0 +1,39 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_EE_SCHEMA_REGISTRY_HRL). +-define(EMQX_EE_SCHEMA_REGISTRY_HRL, true). + +-define(CONF_KEY_ROOT, schema_registry). +-define(CONF_KEY_PATH, [?CONF_KEY_ROOT]). + +-define(SCHEMA_REGISTRY_SHARD, emqx_ee_schema_registry_shard). +-define(SERDE_TAB, emqx_ee_schema_registry_serde_tab). + +-type schema_name() :: binary(). +-type schema_source() :: binary(). + +-type encoded_data() :: iodata(). +-type decoded_data() :: map(). +-type serializer() :: fun((decoded_data()) -> encoded_data()). +-type deserializer() :: fun((encoded_data()) -> decoded_data()). +-type destructor() :: fun(() -> ok). +-type serde_type() :: avro. +-type serde_opts() :: map(). + +-record(serde, { + name :: schema_name(), + serializer :: serializer(), + deserializer :: deserializer(), + destructor :: destructor() +}). +-type serde() :: #serde{}. +-type serde_map() :: #{ + name := schema_name(), + serializer := serializer(), + deserializer := deserializer(), + destructor := destructor() +}. + +-endif. diff --git a/lib-ee/emqx_ee_schema_registry/rebar.config b/lib-ee/emqx_ee_schema_registry/rebar.config new file mode 100644 index 000000000..b19fb05ae --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/rebar.config @@ -0,0 +1,12 @@ +%% -*- mode: erlang -*- + +{erl_opts, [debug_info]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {erlavro, {git, "https://github.com/klarna/erlavro.git", {tag, "2.9.8"}}} +]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_ee_schema_registry]} +]}. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src new file mode 100644 index 000000000..c40fb808a --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src @@ -0,0 +1,15 @@ +{application, emqx_ee_schema_registry, [ + {description, "EMQX Schema Registry"}, + {vsn, "0.1.0"}, + {registered, [emqx_ee_schema_registry_sup]}, + {mod, {emqx_ee_schema_registry_app, []}}, + {applications, [ + kernel, + stdlib, + erlavro + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl new file mode 100644 index 000000000..436777e9f --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -0,0 +1,242 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry). + +-behaviour(gen_server). +-behaviour(emqx_config_handler). + +-include("emqx_ee_schema_registry.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% API +-export([ + start_link/0, + + get_serde/1, + + add_schema/2, + delete_schema/1, + list_schemas/0 +]). + +%% `gen_server' API +-export([ + init/1, + handle_call/3, + handle_cast/2, + terminate/2 +]). + +%% `emqx_config_handler' API +-export([post_config_update/5]). + +-type schema() :: #{ + type := serde_type(), + source := binary(), + description => binary() +}. + +%%------------------------------------------------------------------------------------------------- +%% API +%%------------------------------------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec get_serde(schema_name()) -> {ok, serde_map()} | {error, not_found}. +get_serde(SchemaName) -> + case ets:lookup(?SERDE_TAB, to_bin(SchemaName)) of + [] -> + {error, not_found}; + [Serde] -> + {ok, serde_to_map(Serde)} + end. + +-spec add_schema(schema_name(), schema()) -> ok | {error, term()}. +add_schema(Name, Schema) -> + RawSchema = emqx_map_lib:binary_key_map(Schema), + Res = emqx_conf:update( + [?CONF_KEY_ROOT, schemas, Name], + RawSchema, + #{override_to => cluster} + ), + case Res of + {ok, _} -> + ok; + Error -> + Error + end. + +-spec delete_schema(schema_name()) -> ok | {error, term()}. +delete_schema(Name) -> + Res = emqx_conf:remove( + [?CONF_KEY_ROOT, schemas, Name], + #{override_to => cluster} + ), + case Res of + {ok, _} -> + ok; + Error -> + Error + end. + +-spec list_schemas() -> #{schema_name() => schema()}. +list_schemas() -> + emqx_config:get([?CONF_KEY_ROOT, schemas], #{}). + +%%------------------------------------------------------------------------------------------------- +%% `emqx_config_handler' API +%%------------------------------------------------------------------------------------------------- + +post_config_update( + [?CONF_KEY_ROOT, schemas] = _Path, + _Cmd, + NewConf = #{schemas := NewSchemas}, + OldConf = #{}, + _AppEnvs +) -> + OldSchemas = maps:get(schemas, OldConf, #{}), + #{ + added := Added, + changed := Changed0, + removed := Removed + } = emqx_map_lib:diff_maps(NewSchemas, OldSchemas), + Changed = maps:map(fun(_N, {_Old, New}) -> New end, Changed0), + RemovedNames = maps:keys(Removed), + case RemovedNames of + [] -> + ok; + _ -> + async_delete_serdes(RemovedNames) + end, + SchemasToBuild = maps:to_list(maps:merge(Changed, Added)), + case build_serdes(SchemasToBuild) of + ok -> + {ok, NewConf}; + {error, Reason, SerdesToRollback} -> + lists:foreach(fun ensure_serde_absent/1, SerdesToRollback), + {error, Reason} + end; +post_config_update(_Path, _Cmd, NewConf, _OldConf, _AppEnvs) -> + {ok, NewConf}. + +%%------------------------------------------------------------------------------------------------- +%% `gen_server' API +%%------------------------------------------------------------------------------------------------- + +init(_) -> + process_flag(trap_exit, true), + create_tables(), + Schemas = emqx_conf:get([?CONF_KEY_ROOT, schemas], #{}), + async_build_serdes(Schemas), + State = #{}, + {ok, State}. + +handle_call(_Call, _From, State) -> + {reply, {error, unknown_call}, State}. + +handle_cast({delete_serdes, Names}, State) -> + lists:foreach(fun ensure_serde_absent/1, Names), + ?tp(schema_registry_serdes_deleted, #{}), + {noreply, State}; +handle_cast({build_serdes, Schemas}, State) -> + do_build_serdes(Schemas), + {noreply, State}; +handle_cast(_Cast, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +%%------------------------------------------------------------------------------------------------- +%% Internal fns +%%------------------------------------------------------------------------------------------------- + +create_tables() -> + ok = mria:create_table(?SERDE_TAB, [ + {type, ordered_set}, + {rlog_shard, ?SCHEMA_REGISTRY_SHARD}, + {storage, ram_copies}, + {record_name, serde}, + {attributes, record_info(fields, serde)} + ]), + ok = mria:wait_for_tables([?SERDE_TAB]), + ok. + +do_build_serdes(Schemas) -> + %% TODO: use some kind of mutex to make each core build a + %% different serde to avoid duplicate work. Maybe ekka_locker? + maps:foreach(fun do_build_serde/2, Schemas), + ?tp(schema_registry_serdes_built, #{}). + +build_serdes(Serdes) -> + build_serdes(Serdes, []). + +build_serdes([{Name, Params} | Rest], Acc0) -> + Acc = [Name | Acc0], + case do_build_serde(Name, Params) of + ok -> + build_serdes(Rest, Acc); + {error, Error} -> + {error, Error, Acc} + end; +build_serdes([], _Acc) -> + ok. + +do_build_serde(Name0, #{type := Type, source := Source}) -> + try + Name = to_bin(Name0), + {Serializer, Deserializer, Destructor} = + emqx_ee_schema_registry_serde:make_serde(Type, Name, Source), + Serde = #serde{ + name = Name, + serializer = Serializer, + deserializer = Deserializer, + destructor = Destructor + }, + ok = mria:dirty_write(?SERDE_TAB, Serde), + ok + catch + Kind:Error:Stacktrace -> + ?SLOG( + error, + #{ + msg => "error_building_serde", + name => Name0, + type => Type, + kind => Kind, + error => Error, + stacktrace => Stacktrace + } + ), + {error, Error} + end. + +ensure_serde_absent(Name) -> + case get_serde(Name) of + {ok, #{destructor := Destructor}} -> + Destructor(), + ok = mria:dirty_delete(?SERDE_TAB, to_bin(Name)); + {error, not_found} -> + ok + end. + +async_build_serdes(Schemas) -> + gen_server:cast(?MODULE, {build_serdes, Schemas}). + +async_delete_serdes(Names) -> + gen_server:cast(?MODULE, {delete_serdes, Names}). + +to_bin(A) when is_atom(A) -> atom_to_binary(A); +to_bin(B) when is_binary(B) -> B. + +-spec serde_to_map(serde()) -> serde_map(). +serde_to_map(#serde{} = Serde) -> + #{ + name => Serde#serde.name, + serializer => Serde#serde.serializer, + deserializer => Serde#serde.deserializer, + destructor => Serde#serde.destructor + }. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl new file mode 100644 index 000000000..e82ed95bd --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_app.erl @@ -0,0 +1,19 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_app). + +-behaviour(application). + +-include("emqx_ee_schema_registry.hrl"). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + ok = mria_rlog:wait_for_shards([?SCHEMA_REGISTRY_SHARD], infinity), + emqx_conf:add_handler(?CONF_KEY_PATH, emqx_ee_schema_registry), + emqx_ee_schema_registry_sup:start_link(). + +stop(_State) -> + emqx_conf:remove_handler(?CONF_KEY_PATH), + ok. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl new file mode 100644 index 000000000..fca66a0b1 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl @@ -0,0 +1,251 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_http_api). + +-behaviour(minirest_api). + +-include("emqx_ee_schema_registry.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_api_lib.hrl"). + +-export([ + namespace/0, + api_spec/0, + paths/0, + schema/1 +]). + +-export([ + '/schema_registry'/2, + '/schema_registry/:name'/2 +]). + +%%------------------------------------------------------------------------------------------------- +%% `minirest' and `minirest_trails' API +%%------------------------------------------------------------------------------------------------- + +namespace() -> "schema_registry_http_api". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/schema_registry", + "/schema_registry/:name" + ]. + +schema("/schema_registry") -> + #{ + 'operationId' => '/schema_registry', + get => #{ + tags => [<<"schema_registry">>], + summary => <<"List registered schemas">>, + description => ?DESC("desc_schema_registry_api_list"), + responses => + #{ + 200 => + emqx_dashboard_swagger:schema_with_examples( + hoconsc:array(emqx_ee_schema_registry_schema:api_schema("get")), + #{ + sample => + #{value => sample_list_schemas_response()} + } + ) + } + }, + post => #{ + tags => [<<"schema_registry">>], + summary => <<"Register a new schema">>, + description => ?DESC("desc_schema_registry_api_post"), + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("post"), + post_examples() + ), + responses => + #{ + 201 => + emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("post"), + post_examples() + ), + 400 => error_schema('ALREADY_EXISTS', "Schema already exists") + } + } + }; +schema("/schema_registry/:name") -> + #{ + 'operationId' => '/schema_registry/:name', + get => #{ + tags => [<<"schema_registry">>], + summary => <<"Get registered schema">>, + description => ?DESC("desc_schema_registry_api_get"), + parameters => [param_path_schema_name()], + responses => + #{ + 200 => + emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("get"), + get_examples() + ), + 404 => error_schema('NOT_FOUND', "Schema not found") + } + }, + put => #{ + tags => [<<"schema_registry">>], + summary => <<"Update a schema">>, + description => ?DESC("desc_schema_registry_api_put"), + parameters => [param_path_schema_name()], + 'requestBody' => emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("put"), + post_examples() + ), + responses => + #{ + 200 => + emqx_dashboard_swagger:schema_with_examples( + emqx_ee_schema_registry_schema:api_schema("put"), + put_examples() + ), + 404 => error_schema('NOT_FOUND', "Schema not found") + } + }, + delete => #{ + tags => [<<"schema_registry">>], + summary => <<"Delete registered schema">>, + description => ?DESC("desc_schema_registry_api_delete"), + parameters => [param_path_schema_name()], + responses => + #{ + 204 => <<"Schema deleted">>, + 404 => error_schema('NOT_FOUND', "Schema not found") + } + } + }. + +%%------------------------------------------------------------------------------------------------- +%% API +%%------------------------------------------------------------------------------------------------- + +'/schema_registry'(get, _Params) -> + Schemas = emqx_ee_schema_registry:list_schemas(), + Response = + lists:map( + fun({Name, Params}) -> + Params#{name => Name} + end, + maps:to_list(Schemas) + ), + ?OK(Response); +'/schema_registry'(post, #{body := Params0 = #{<<"name">> := Name}}) -> + Params = maps:without([<<"name">>], Params0), + case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of + undefined -> + case emqx_ee_schema_registry:add_schema(Name, Params) of + ok -> + Res = emqx_config:get([?CONF_KEY_ROOT, schemas, Name]), + {201, Res#{name => Name}}; + {error, Error} -> + ?BAD_REQUEST(Error) + end; + _ -> + ?BAD_REQUEST('ALREADY_EXISTS', <<"Schema already exists">>) + end. + +'/schema_registry/:name'(get, #{bindings := #{name := Name}}) -> + case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of + undefined -> + ?NOT_FOUND(<<"Schema not found">>); + Res -> + ?OK(Res#{name => Name}) + end; +'/schema_registry/:name'(put, #{bindings := #{name := Name}, body := Params}) -> + case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of + undefined -> + ?NOT_FOUND(<<"Schema not found">>); + _ -> + case emqx_ee_schema_registry:add_schema(Name, Params) of + ok -> + Res = emqx_config:get([?CONF_KEY_ROOT, schemas, Name]), + ?OK(Res#{name => Name}); + {error, Error} -> + ?BAD_REQUEST(Error) + end + end; +'/schema_registry/:name'(delete, #{bindings := #{name := Name}}) -> + case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of + undefined -> + ?NOT_FOUND(<<"Schema not found">>); + _ -> + case emqx_ee_schema_registry:delete_schema(Name) of + ok -> + ?NO_CONTENT; + {error, Error} -> + ?BAD_REQUEST(Error) + end + end. + +%%------------------------------------------------------------------------------------------------- +%% Examples +%%------------------------------------------------------------------------------------------------- + +sample_list_schemas_response() -> + [sample_get_schema_response(avro)]. + +sample_get_schema_response(avro) -> + #{ + type => <<"avro">>, + name => <<"my_avro_schema">>, + description => <<"My Avro Schema">>, + source => << + "{\"type\":\"record\"," + "\"fields\":[{\"type\":\"int\",\"name\":\"i\"}," + "{\"type\":\"string\",\"name\":\"s\"}]}" + >> + }. + +put_examples() -> + post_examples(). + +post_examples() -> + get_examples(). + +get_examples() -> + #{ + <<"avro_schema">> => + #{ + summary => <<"Avro">>, + value => sample_get_schema_response(avro) + } + }. + +%%------------------------------------------------------------------------------------------------- +%% Schemas and hocon types +%%------------------------------------------------------------------------------------------------- + +param_path_schema_name() -> + {name, + mk( + binary(), + #{ + in => path, + required => true, + example => <<"my_schema">>, + desc => ?DESC("desc_param_path_schema_name") + } + )}. + +%%------------------------------------------------------------------------------------------------- +%% Internal fns +%%------------------------------------------------------------------------------------------------- + +mk(Type, Meta) -> hoconsc:mk(Type, Meta). + +error_schema(Code, Message) when is_atom(Code) -> + error_schema([Code], Message); +error_schema(Codes, Message) when is_list(Message) -> + error_schema(Codes, list_to_binary(Message)); +error_schema(Codes, Message) when is_list(Codes) andalso is_binary(Message) -> + emqx_dashboard_swagger:error_codes(Codes, Message). diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl new file mode 100644 index 000000000..01177345a --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl @@ -0,0 +1,127 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_schema_registry_schema). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_ee_schema_registry.hrl"). + +%% `hocon_schema' API +-export([ + roots/0, + fields/1, + desc/1, + tags/0, + union_member_selector/1 +]). + +%% `minirest_trails' API +-export([ + api_schema/1 +]). + +%%------------------------------------------------------------------------------ +%% `hocon_schema' APIs +%%------------------------------------------------------------------------------ + +roots() -> + [{?CONF_KEY_ROOT, mk(ref(?CONF_KEY_ROOT), #{required => false})}]. + +tags() -> + [<<"Schema Registry">>]. + +fields(?CONF_KEY_ROOT) -> + [ + {schemas, + mk( + hoconsc:map( + name, + hoconsc:union(fun union_member_selector/1) + ), + #{ + default => #{}, + desc => ?DESC("schema_registry_schemas") + } + )} + ]; +fields(avro) -> + [ + {type, mk(hoconsc:enum([avro]), #{required => true, desc => ?DESC("schema_type")})}, + {source, + mk(emqx_schema:json_binary(), #{required => true, desc => ?DESC("schema_source")})}, + {description, mk(binary(), #{default => <<>>, desc => ?DESC("schema_description")})} + ]; +fields("get_avro") -> + [{name, mk(binary(), #{required => true, desc => ?DESC("schema_name")})} | fields(avro)]; +fields("put_avro") -> + fields(avro); +fields("post_" ++ Type) -> + fields("get_" ++ Type). + +desc(?CONF_KEY_ROOT) -> + ?DESC("schema_registry_root"); +desc(avro) -> + ?DESC("avro_type"); +desc(_) -> + undefined. + +union_member_selector(all_union_members) -> + refs(); +union_member_selector({value, V}) -> + refs(V). + +union_member_selector_get_api(all_union_members) -> + refs_get_api(); +union_member_selector_get_api({value, V}) -> + refs_get_api(V). + +%%------------------------------------------------------------------------------ +%% `minirest_trails' "APIs" +%%------------------------------------------------------------------------------ + +api_schema("get") -> + hoconsc:union(fun union_member_selector_get_api/1); +api_schema("post") -> + api_schema("get"); +api_schema("put") -> + hoconsc:union(fun union_member_selector/1). + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ + +mk(Type, Meta) -> hoconsc:mk(Type, Meta). +ref(Name) -> hoconsc:ref(?MODULE, Name). + +supported_serde_types() -> + [avro]. + +refs() -> + [ref(Type) || Type <- supported_serde_types()]. + +refs(#{<<"type">> := TypeAtom} = Value) when is_atom(TypeAtom) -> + refs(Value#{<<"type">> := atom_to_binary(TypeAtom)}); +refs(#{<<"type">> := <<"avro">>}) -> + [ref(avro)]; +refs(_) -> + Expected = lists:join(" | ", [atom_to_list(T) || T <- supported_serde_types()]), + throw(#{ + field_name => type, + expected => Expected + }). + +refs_get_api() -> + [ref("get_avro")]. + +refs_get_api(#{<<"type">> := TypeAtom} = Value) when is_atom(TypeAtom) -> + refs(Value#{<<"type">> := atom_to_binary(TypeAtom)}); +refs_get_api(#{<<"type">> := <<"avro">>}) -> + [ref("get_avro")]; +refs_get_api(_) -> + Expected = lists:join(" | ", [atom_to_list(T) || T <- supported_serde_types()]), + throw(#{ + field_name => type, + expected => Expected + }). diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl new file mode 100644 index 000000000..43145fb16 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl @@ -0,0 +1,70 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_serde). + +-include("emqx_ee_schema_registry.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% API +-export([ + decode/2, + decode/3, + encode/2, + encode/3, + make_serde/3 +]). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +-spec decode(schema_name(), encoded_data()) -> decoded_data(). +decode(SerdeName, RawData) -> + decode(SerdeName, RawData, []). + +-spec decode(schema_name(), encoded_data(), [term()]) -> decoded_data(). +decode(SerdeName, RawData, VarArgs) when is_list(VarArgs) -> + case emqx_ee_schema_registry:get_serde(SerdeName) of + {error, not_found} -> + error({serde_not_found, SerdeName}); + {ok, #{deserializer := Deserializer}} -> + apply(Deserializer, [RawData | VarArgs]) + end. + +-spec encode(schema_name(), decoded_data()) -> encoded_data(). +encode(SerdeName, RawData) -> + encode(SerdeName, RawData, []). + +-spec encode(schema_name(), decoded_data(), [term()]) -> encoded_data(). +encode(SerdeName, EncodedData, VarArgs) when is_list(VarArgs) -> + case emqx_ee_schema_registry:get_serde(SerdeName) of + {error, not_found} -> + error({serde_not_found, SerdeName}); + {ok, #{serializer := Serializer}} -> + apply(Serializer, [EncodedData | VarArgs]) + end. + +-spec make_serde(serde_type(), schema_name(), schema_source()) -> + {serializer(), deserializer(), destructor()}. +make_serde(avro, Name, Source0) -> + Source = inject_avro_name(Name, Source0), + Serializer = avro:make_simple_encoder(Source, _Opts = []), + Deserializer = avro:make_simple_decoder(Source, [{map_type, map}, {record_type, map}]), + Destructor = fun() -> + ?tp(serde_destroyed, #{type => avro, name => Name}), + ok + end, + {Serializer, Deserializer, Destructor}. + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ + +-spec inject_avro_name(schema_name(), schema_source()) -> schema_source(). +inject_avro_name(Name, Source0) -> + %% The schema checks that the source is a valid JSON when + %% typechecking, so we shouldn't need to validate here. + Schema0 = emqx_json:decode(Source0, [return_maps]), + Schema = Schema0#{<<"name">> => Name}, + emqx_json:encode(Schema). diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl new file mode 100644 index 000000000..0dfc601d3 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_sup.erl @@ -0,0 +1,43 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +%% sup_flags() = #{strategy => strategy(), % optional +%% intensity => non_neg_integer(), % optional +%% period => pos_integer()} % optional +%% child_spec() = #{id => child_id(), % mandatory +%% start => mfargs(), % mandatory +%% restart => restart(), % optional +%% shutdown => shutdown(), % optional +%% type => worker(), % optional +%% modules => modules()} % optional +init([]) -> + SupFlags = #{ + strategy => one_for_one, + intensity => 10, + period => 100 + }, + ChildSpecs = [child_spec(emqx_ee_schema_registry)], + {ok, {SupFlags, ChildSpecs}}. + +child_spec(Mod) -> + #{ + id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => 5_000, + type => worker, + modules => [Mod] + }. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl new file mode 100644 index 000000000..9b2f64c03 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -0,0 +1,433 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include("emqx_ee_schema_registry.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +-define(APPS, [emqx_conf, emqx_rule_engine, emqx_ee_schema_registry]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [{group, avro}]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + [{avro, TCs}]. + +init_per_suite(Config) -> + emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), + emqx_mgmt_api_test_util:init_suite(?APPS), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)), + ok. + +init_per_group(avro, Config) -> + [{serde_type, avro} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + ok = snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TestCase, _Config) -> + ok = snabbkaffe:stop(), + emqx_common_test_helpers:call_janitor(), + clear_schemas(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +trace_rule(Data, Envs, _Args) -> + Now = erlang:monotonic_time(), + ets:insert(recorded_actions, {Now, #{data => Data, envs => Envs}}), + TestPid = persistent_term:get({?MODULE, test_pid}), + TestPid ! {action, #{data => Data, envs => Envs}}, + ok. + +make_trace_fn_action() -> + persistent_term:put({?MODULE, test_pid}, self()), + Fn = <<(atom_to_binary(?MODULE))/binary, ":trace_rule">>, + emqx_tables:new(recorded_actions, [named_table, public, ordered_set]), + #{function => Fn, args => #{}}. + +create_rule_http(RuleParams) -> + RepublishTopic = <<"republish/schema_registry">>, + emqx:subscribe(RepublishTopic), + DefaultParams = #{ + enable => true, + actions => [ + make_trace_fn_action(), + #{ + <<"function">> => <<"republish">>, + <<"args">> => + #{ + <<"topic">> => RepublishTopic, + <<"payload">> => <<>>, + <<"qos">> => 0, + <<"retain">> => false, + <<"user_properties">> => <<>> + } + } + ] + }, + Params = maps:merge(DefaultParams, RuleParams), + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + Error -> Error + end. + +schema_params(avro) -> + Source = #{ + type => record, + fields => [ + #{name => <<"i">>, type => <<"int">>}, + #{name => <<"s">>, type => <<"string">>} + ] + }, + SourceBin = emqx_json:encode(Source), + #{type => avro, source => SourceBin}. + +create_serde(SerdeType, SerdeName) -> + Schema = schema_params(SerdeType), + ok = emqx_ee_schema_registry:add_schema(SerdeName, Schema), + ok. + +sql_for(avro, encode_decode1) -> + << + "select\n" + " schema_decode('my_serde',\n" + " schema_encode('my_serde', json_decode(payload))) as decoded,\n" + " decoded.i as decoded_int,\n" + " decoded.s as decoded_string\n" + " from t" + >>; +sql_for(avro, encode1) -> + << + "select\n" + " schema_encode('my_serde', json_decode(payload)) as encoded\n" + " from t" + >>; +sql_for(avro, decode1) -> + << + "select\n" + " schema_decode('my_serde', payload) as decoded\n" + " from t" + >>; +sql_for(Type, Name) -> + ct:fail("unimplemented: ~p", [{Type, Name}]). + +clear_schemas() -> + maps:foreach( + fun(Name, _Schema) -> + ok = emqx_ee_schema_registry:delete_schema(Name) + end, + emqx_ee_schema_registry:list_schemas() + ). + +receive_action_results() -> + receive + {action, #{data := _} = Res} -> + Res + after 1_000 -> + ct:fail("action didn't run") + end. + +receive_published(Line) -> + receive + {deliver, _Topic, Msg} -> + MsgMap = emqx_message:to_map(Msg), + maps:update_with( + payload, + fun(Raw) -> + case emqx_json:safe_decode(Raw, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> Raw + end + end, + MsgMap + ) + after 1_000 -> + ct:pal("mailbox: ~p", [process_info(self(), messages)]), + ct:fail("publish not received, line ~b", [Line]) + end. + +cluster(Config) -> + PrivDataDir = ?config(priv_dir, Config), + PeerModule = + case os:getenv("IS_CI") of + false -> + slave; + _ -> + ct_slave + end, + Cluster = emqx_common_test_helpers:emqx_cluster( + [core, core], + [ + {apps, ?APPS}, + {listener_ports, []}, + {peer_mod, PeerModule}, + {priv_data_dir, PrivDataDir}, + {load_schema, true}, + {start_autocluster, true}, + {schema_mod, emqx_ee_conf_schema}, + %% need to restart schema registry app in the tests so + %% that it re-registers the config handler that is lost + %% when emqx_conf restarts during join. + {env, [{emqx_machine, applications, [emqx_ee_schema_registry]}]}, + {load_apps, [emqx_machine | ?APPS]}, + {env_handler, fun + (emqx) -> + application:set_env(emqx, boot_modules, [broker, router]), + ok; + (emqx_conf) -> + ok; + (_) -> + ok + end} + ] + ), + ct:pal("cluster:\n ~p", [Cluster]), + Cluster. + +start_cluster(Cluster) -> + Nodes = [ + emqx_common_test_helpers:start_slave(Name, Opts) + || {Name, Opts} <- Cluster + ], + on_exit(fun() -> + emqx_misc:pmap( + fun(N) -> + ct:pal("stopping ~p", [N]), + ok = emqx_common_test_helpers:stop_slave(N) + end, + Nodes + ) + end), + erpc:multicall(Nodes, mria_rlog, wait_for_shards, [[?SCHEMA_REGISTRY_SHARD], 30_000]), + Nodes. + +wait_for_cluster_rpc(Node) -> + %% need to wait until the config handler is ready after + %% restarting during the cluster join. + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + true = is_pid(erpc:call(Node, erlang, whereis, [emqx_config_handler])) + ). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_unknown_calls(_Config) -> + Ref = monitor(process, emqx_ee_schema_registry), + %% for coverage + emqx_ee_schema_registry ! unknown, + gen_server:cast(emqx_ee_schema_registry, unknown), + ?assertEqual({error, unknown_call}, gen_server:call(emqx_ee_schema_registry, unknown)), + receive + {'DOWN', Ref, process, _, _} -> + ct:fail("registry shouldn't have died") + after 500 -> + ok + end. + +t_encode_decode(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode_decode1)}), + on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + PayloadBin = emqx_json:encode(Payload), + emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), + Res = receive_action_results(), + ?assertMatch( + #{ + data := + #{ + <<"decoded">> := + #{ + <<"i">> := 10, + <<"s">> := <<"text">> + }, + <<"decoded_int">> := 10, + <<"decoded_string">> := <<"text">> + } + }, + Res + ), + ok. + +t_delete_serde(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ?check_trace( + begin + ok = create_serde(SerdeType, SerdeName), + {ok, {ok, _}} = + ?wait_async_action( + emqx_ee_schema_registry:delete_schema(SerdeName), + #{?snk_kind := schema_registry_serdes_deleted}, + 1_000 + ), + ok + end, + fun(Trace) -> + ?assertMatch([_], ?of_kind(schema_registry_serdes_deleted, Trace)), + ?assertMatch([#{type := SerdeType}], ?of_kind(serde_destroyed, Trace)), + ok + end + ), + ok. + +t_encode(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode1)}), + on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + PayloadBin = emqx_json:encode(Payload), + emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), + Published = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"encoded">> := _}}, + Published + ), + #{payload := #{<<"encoded">> := Encoded}} = Published, + {ok, #{deserializer := Deserializer}} = emqx_ee_schema_registry:get_serde(SerdeName), + ?assertEqual(Payload, Deserializer(Encoded)), + ok. + +t_decode(Config) -> + SerdeType = ?config(serde_type, Config), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, decode1)}), + on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + {ok, #{serializer := Serializer}} = emqx_ee_schema_registry:get_serde(SerdeName), + EncodedBin = Serializer(Payload), + emqx:publish(emqx_message:make(<<"t">>, EncodedBin)), + Published = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"decoded">> := _}}, + Published + ), + #{payload := #{<<"decoded">> := Decoded}} = Published, + ?assertEqual(Payload, Decoded), + ok. + +t_fail_rollback(Config) -> + SerdeType = ?config(serde_type, Config), + OkSchema = emqx_map_lib:binary_key_map(schema_params(SerdeType)), + BrokenSchema = OkSchema#{<<"source">> := <<"{}">>}, + %% hopefully, for this small map, the key order is used. + Serdes = #{ + <<"a">> => OkSchema, + <<"z">> => BrokenSchema + }, + ?assertMatch( + {error, _}, + emqx_conf:update( + [?CONF_KEY_ROOT, schemas], + Serdes, + #{} + ) + ), + %% no serdes should be in the table + ?assertEqual({error, not_found}, emqx_ee_schema_registry:get_serde(<<"a">>)), + ?assertEqual({error, not_found}, emqx_ee_schema_registry:get_serde(<<"z">>)), + ok. + +t_cluster_serde_build(Config) -> + SerdeType = ?config(serde_type, Config), + Cluster = cluster(Config), + SerdeName = my_serde, + Schema = schema_params(SerdeType), + ?check_trace( + begin + Nodes = [N1, N2 | _] = start_cluster(Cluster), + NumNodes = length(Nodes), + wait_for_cluster_rpc(N2), + ?assertEqual( + ok, + erpc:call(N2, emqx_ee_schema_registry, add_schema, [SerdeName, Schema]) + ), + %% check that we can serialize/deserialize in all nodes + lists:foreach( + fun(N) -> + erpc:call(N, fun() -> + Res0 = emqx_ee_schema_registry:get_serde(SerdeName), + ?assertMatch({ok, #{}}, Res0, #{node => N}), + {ok, #{serializer := Serializer, deserializer := Deserializer}} = Res0, + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + ?assertEqual(Payload, Deserializer(Serializer(Payload)), #{node => N}), + ok + end) + end, + Nodes + ), + %% now we delete and check it's removed from the table + ?tp(will_delete_schema, #{}), + {ok, SRef1} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := schema_registry_serdes_deleted}), + NumNodes, + 5_000 + ), + ?assertEqual( + ok, + erpc:call(N1, emqx_ee_schema_registry, delete_schema, [SerdeName]) + ), + {ok, _} = snabbkaffe:receive_events(SRef1), + lists:foreach( + fun(N) -> + erpc:call(N, fun() -> + ?assertMatch( + {error, not_found}, + emqx_ee_schema_registry:get_serde(SerdeName), + #{node => N} + ), + ok + end) + end, + Nodes + ), + ok + end, + fun(Trace) -> + ?assert( + ?strict_causality( + #{?snk_kind := will_delete_schema}, + #{?snk_kind := serde_destroyed, type := SerdeType}, + Trace + ) + ), + ok + end + ), + ok. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl new file mode 100644 index 000000000..bbb6d5ef0 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl @@ -0,0 +1,250 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_http_api_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-import(emqx_mgmt_api_test_util, [uri/1]). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(APPS, [emqx_conf, emqx_ee_schema_registry]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), + emqx_mgmt_api_test_util:init_suite(?APPS), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)), + ok. + +init_per_testcase(_TestCase, Config) -> + clear_schemas(), + ok = snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TestCase, _Config) -> + clear_schemas(), + ok = snabbkaffe:stop(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +request(get) -> + do_request(get, _Parts = [], _Body = []); +request({get, Name}) -> + do_request(get, _Parts = [Name], _Body = []); +request({delete, Name}) -> + do_request(delete, _Parts = [Name], _Body = []); +request({put, Name, Params}) -> + do_request(put, _Parts = [Name], Params); +request({post, Params}) -> + do_request(post, _Parts = [], Params). + +do_request(Method, PathParts, Body) -> + Header = emqx_common_test_http:default_auth_header(), + URI = uri(["schema_registry" | PathParts]), + Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, + Res0 = emqx_mgmt_api_test_util:request_api(Method, URI, [], Header, Body, Opts), + case Res0 of + {ok, Code, <<>>} -> + {ok, Code, <<>>}; + {ok, Code, Res1} -> + Res2 = emqx_json:decode(Res1, [return_maps]), + Res3 = try_decode_error_message(Res2), + {ok, Code, Res3}; + Error -> + Error + end. + +try_decode_error_message(#{<<"message">> := Msg0} = Res0) -> + case emqx_json:safe_decode(Msg0, [return_maps]) of + {ok, Msg} -> + Res0#{<<"message">> := Msg}; + {error, _} -> + Res0 + end; +try_decode_error_message(Res) -> + Res. + +clear_schemas() -> + maps:foreach( + fun(Name, _Schema) -> + ok = emqx_ee_schema_registry:delete_schema(Name) + end, + emqx_ee_schema_registry:list_schemas() + ). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_crud(_Config) -> + SchemaName = <<"my_avro_schema">>, + Source = #{ + type => record, + fields => [ + #{name => <<"i">>, type => <<"int">>}, + #{name => <<"s">>, type => <<"string">>} + ] + }, + SourceBin = emqx_json:encode(Source), + Params = #{ + <<"type">> => <<"avro">>, + <<"source">> => SourceBin, + <<"name">> => SchemaName, + <<"description">> => <<"My schema">> + }, + UpdateParams = maps:without([<<"name">>], Params), + + %% no schemas at first + ?assertMatch({ok, 200, []}, request(get)), + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Schema not found">> + }}, + request({get, SchemaName}) + ), + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Schema not found">> + }}, + request({put, SchemaName, UpdateParams}) + ), + ?assertMatch( + {ok, 404, #{ + <<"code">> := <<"NOT_FOUND">>, + <<"message">> := <<"Schema not found">> + }}, + request({delete, SchemaName}) + ), + + %% create a schema + ?assertMatch( + {ok, 201, #{ + <<"type">> := <<"avro">>, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My schema">> + }}, + request({post, Params}) + ), + ?assertMatch( + {ok, 200, #{ + <<"type">> := <<"avro">>, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My schema">> + }}, + request({get, SchemaName}) + ), + ?assertMatch( + {ok, 200, [ + #{ + <<"type">> := <<"avro">>, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My schema">> + } + ]}, + request(get) + ), + UpdateParams1 = UpdateParams#{<<"description">> := <<"My new schema">>}, + ?assertMatch( + {ok, 200, #{ + <<"type">> := <<"avro">>, + <<"source">> := SourceBin, + <<"name">> := SchemaName, + <<"description">> := <<"My new schema">> + }}, + request({put, SchemaName, UpdateParams1}) + ), + + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"ALREADY_EXISTS">>, + <<"message">> := <<"Schema already exists">> + }}, + request({post, Params}) + ), + %% typechecks, but is invalid + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + <<"{post_config_update,emqx_ee_schema_registry,{not_found,<<\"type\">>}}">> + }}, + request({put, SchemaName, UpdateParams#{<<"source">> := <<"{}">>}}) + ), + + ?assertMatch( + {ok, 204, <<>>}, + request({delete, SchemaName}) + ), + + %% doesn't typecheck + lists:foreach( + fun(Field) -> + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := #{<<"reason">> := <<"required_field">>} + }}, + request({post, maps:without([Field], Params)}), + #{field => Field} + ) + end, + [<<"name">>, <<"source">>] + ), + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + #{ + <<"expected">> := [_ | _], + <<"field_name">> := <<"type">> + } + }}, + request({post, maps:without([<<"type">>], Params)}), + #{field => <<"type">>} + ), + %% typechecks, but is invalid + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + <<"{post_config_update,emqx_ee_schema_registry,{not_found,<<\"type\">>}}">> + }}, + request({post, Params#{<<"source">> := <<"{}">>}}) + ), + + %% unknown serde type + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"BAD_REQUEST">>, + <<"message">> := + #{ + <<"expected">> := [_ | _], + <<"field_name">> := <<"type">> + } + }}, + request({post, Params#{<<"type">> := <<"foo">>}}) + ), + + ok. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl new file mode 100644 index 000000000..be62717d3 --- /dev/null +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl @@ -0,0 +1,121 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_schema_registry_serde_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-include("emqx_ee_schema_registry.hrl"). + +-import(emqx_common_test_helpers, [on_exit/1]). + +-define(APPS, [emqx_conf, emqx_rule_engine, emqx_ee_schema_registry]). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), + emqx_mgmt_api_test_util:init_suite(?APPS), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)), + ok. +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + emqx_common_test_helpers:call_janitor(), + clear_schemas(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +clear_schemas() -> + maps:foreach( + fun(Name, _Schema) -> + ok = emqx_ee_schema_registry:delete_schema(Name) + end, + emqx_ee_schema_registry:list_schemas() + ). + +schema_params(avro) -> + Source = #{ + type => record, + fields => [ + #{name => <<"i">>, type => <<"int">>}, + #{name => <<"s">>, type => <<"string">>} + ] + }, + SourceBin = emqx_json:encode(Source), + #{type => avro, source => SourceBin}. + +assert_roundtrip(SerdeName, Original) -> + Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original), + Decoded = emqx_ee_schema_registry_serde:decode(SerdeName, Encoded), + ?assertEqual(Original, Decoded, #{original => Original}). + +assert_roundtrip(SerdeName, Original, ArgsSerialize, ArgsDeserialize) -> + Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original, ArgsSerialize), + Decoded = emqx_ee_schema_registry_serde:decode(SerdeName, Encoded, ArgsDeserialize), + ?assertEqual(Original, Decoded, #{original => Original}). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_roundtrip_avro(_Config) -> + SerdeName = my_serde, + Params = schema_params(avro), + ok = emqx_ee_schema_registry:add_schema(SerdeName, Params), + Original = #{<<"i">> => 10, <<"s">> => <<"hi">>}, + %% for coverage + assert_roundtrip(SerdeName, Original, _ArgsSerialize = [], _ArgsDeserialize = []), + assert_roundtrip(SerdeName, Original), + ok. + +t_avro_invalid_json_schema(_Config) -> + SerdeName = my_serde, + Params = schema_params(avro), + WrongParams = Params#{source := <<"{">>}, + ?assertMatch( + {error, #{reason := #{expected_type := _}}}, + emqx_ee_schema_registry:add_schema(SerdeName, WrongParams) + ), + ok. + +t_avro_invalid_schema(_Config) -> + SerdeName = my_serde, + Params = schema_params(avro), + WrongParams = Params#{source := <<"{}">>}, + ?assertMatch( + {error, {post_config_update, _, {not_found, <<"type">>}}}, + emqx_ee_schema_registry:add_schema(SerdeName, WrongParams) + ), + ok. + +t_serde_not_found(_Config) -> + %% for coverage + NonexistentSerde = <<"nonexistent">>, + Original = #{}, + ?assertError( + {serde_not_found, NonexistentSerde}, + emqx_ee_schema_registry_serde:encode(NonexistentSerde, Original) + ), + ?assertError( + {serde_not_found, NonexistentSerde}, + emqx_ee_schema_registry_serde:decode(NonexistentSerde, Original) + ), + ok. diff --git a/mix.exs b/mix.exs index 600218e52..712e1f4e2 100644 --- a/mix.exs +++ b/mix.exs @@ -83,6 +83,8 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqx and observer_cli {:recon, github: "ferd/recon", tag: "2.5.1", override: true}, {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true}, + # in conflict by erlavro and rocketmq + {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true}, # dependencies of dependencies; we choose specific refs to match # what rebar3 chooses. # in conflict by gun and emqtt @@ -307,7 +309,8 @@ defmodule EMQXUmbrella.MixProject do emqx_license: :permanent, emqx_ee_conf: :load, emqx_ee_connector: :permanent, - emqx_ee_bridge: :permanent + emqx_ee_bridge: :permanent, + emqx_ee_schema_registry: :permanent ], else: [] ) diff --git a/rebar.config b/rebar.config index 50a8124be..8ad8133fd 100644 --- a/rebar.config +++ b/rebar.config @@ -81,6 +81,8 @@ , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} , {telemetry, "1.1.0"} , {hackney, {git, "https://github.com/emqx/hackney.git", {tag, "1.18.1-1"}}} + %% in conflict by erlavro and rocketmq + , {jsone, {git, "https://github.com/emqx/jsone.git", {tag, "1.7.1"}}} ]}. {xref_ignores, diff --git a/rebar.config.erl b/rebar.config.erl index 98cd30570..e00fe730d 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -422,7 +422,8 @@ relx_apps_per_edition(ee) -> emqx_license, {emqx_ee_conf, load}, emqx_ee_connector, - emqx_ee_bridge + emqx_ee_bridge, + emqx_ee_schema_registry ]; relx_apps_per_edition(ce) -> []. diff --git a/rel/i18n/emqx_bridge_api.hocon b/rel/i18n/emqx_bridge_api.hocon index f5d372128..66960619a 100644 --- a/rel/i18n/emqx_bridge_api.hocon +++ b/rel/i18n/emqx_bridge_api.hocon @@ -57,7 +57,7 @@ emqx_bridge_api { desc_api1 { desc { en: """List all created bridges""" - zh: """列出所有 Birdge""" + zh: """列出所有 Bridge""" } label: { en: "List All Bridges" diff --git a/rel/i18n/emqx_ee_schema_registry_http_api.hocon b/rel/i18n/emqx_ee_schema_registry_http_api.hocon new file mode 100644 index 000000000..058796a66 --- /dev/null +++ b/rel/i18n/emqx_ee_schema_registry_http_api.hocon @@ -0,0 +1,69 @@ +emqx_ee_schema_registry_http_api { + # apis + desc_schema_registry_api_list { + desc { + en: "List all registered schemas" + zh: "列出所有注册的模式" + } + label { + en: "List schemas" + zh: "列表模式" + } + } + + desc_schema_registry_api_get { + desc { + en: "Get a schema by its name" + zh: "通过名称获取模式" + } + label { + en: "Get schema" + zh: "获取模式" + } + } + + desc_schema_registry_api_post { + desc { + en: "Register a new schema" + zh: "注册一个新的模式" + } + label { + en: "Register schema" + zh: "注册模式" + } + } + + desc_schema_registry_api_put { + desc { + en: "Update an existing schema" + zh: "更新一个现有的模式" + } + label { + en: "Update schema" + zh: "更新模式" + } + } + + desc_schema_registry_api_delete { + desc { + en: "Delete a schema" + zh: "删除一个模式" + } + label { + en: "Delete schema" + zh: "删除模式" + } + } + + # params + desc_param_path_schema_name { + desc { + en: "The schema name" + zh: "模式名称" + } + label { + en: "Schema name" + zh: "模式名称" + } + } +} diff --git a/rel/i18n/emqx_ee_schema_registry_schema.hocon b/rel/i18n/emqx_ee_schema_registry_schema.hocon new file mode 100644 index 000000000..1538fe5f9 --- /dev/null +++ b/rel/i18n/emqx_ee_schema_registry_schema.hocon @@ -0,0 +1,78 @@ +emqx_ee_schema_registry_schema { + schema_registry_root { + desc { + en: "Schema registry configurations." + zh: "模式注册表的配置。" + } + label { + en: "Schema registry" + zh: "模式注册表" + } + } + + schema_registry_schemas { + desc { + en: "Registered schemas." + zh: "注册的模式。" + } + label { + en: "Registered schemas" + zh: "注册的模式" + } + } + + schema_name { + desc { + en: "A name for the schema that will serve as its identifier." + zh: "模式的一个名称，将作为其标识符。" + } + label { + en: "Schema name" + zh: "模式名称" + } + } + + schema_type { + desc { + en: "Schema type." + zh: "模式类型。" + } + label { + en: "Schema type" + zh: "模式类型" + } + } + + schema_source { + desc { + en: "Source text for the schema." + zh: "模式的源文本。" + } + label { + en: "Schema source" + zh: "模式来源" + } + } + + schema_description { + desc { + en: "A description for this schema." + zh: "对该模式的描述。" + } + label { + en: "Schema description" + zh: "模式描述" + } + } + + avro_type { + desc { + en: "[Apache Avro](https://avro.apache.org/) serialization format." + zh: "[阿帕奇-阿夫罗](https://avro.apache.org/) 序列化格式。" + } + label { + en: "Apache Avro" + zh: "阿帕奇-阿夫罗" + } + } +} diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 79c8b7e3a..168275e1e 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -1,6 +1,7 @@ ACL AES APIs +Avro BPAPI BSON Backplane From 7c05304ff42a7e6e74583c6bf76b219bebc4cc40 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 6 Apr 2023 16:53:55 -0300 Subject: [PATCH 031/279] chore: bump version to e5.0.2-rc.6 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 6394d940e..fa339be36 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.21"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.2-rc.5"). +-define(EMQX_RELEASE_EE, "5.0.2-rc.6"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From 502cc2b8b8ea1bdf42ef34cefe6965357f64103d Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 7 Apr 2023 15:34:16 +0800 Subject: [PATCH 032/279] chore: fix common tests --- apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl | 1 + apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl | 1 + 6 files changed, 6 insertions(+) diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index bfcebd772..c5fabf2fd 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -46,6 +46,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> application:load(emqx), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_config:delete_override_conf_files(), emqx_config:erase(gateway), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), diff --git a/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl index c5e8d9a92..8b0dacd75 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cm_SUITE.erl @@ -34,6 +34,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([]), diff --git a/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl index 77f4058e7..35e32d3da 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cm_registry_SUITE.erl @@ -34,6 +34,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([]), Conf. diff --git a/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl index 0aa3172f1..35ce5fb31 100644 --- a/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_ctx_SUITE.erl @@ -28,6 +28,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> + emqx_gateway_test_utils:load_all_gateway_apps(), ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), ok = meck:expect( emqx_access_control, diff --git a/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl index 211315e6c..b82e049d3 100644 --- a/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_metrics_SUITE.erl @@ -33,6 +33,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> emqx_config:erase(gateway), + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([]), Conf. diff --git a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl index cc5f7bf37..a51621688 100644 --- a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl @@ -37,6 +37,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Cfg) -> + emqx_gateway_test_utils:load_all_gateway_apps(), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_common_test_helpers:start_apps([emqx_authn, emqx_gateway]), Cfg. From ffa8f21d2e0e1bbb1d7552bb557225a82201776f Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 7 Apr 2023 18:02:04 +0800 Subject: [PATCH 033/279] chore: upgrade dashboard to v1.2.1 for ce --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 410140616..28c4b16e7 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.2.0 +export EMQX_DASHBOARD_VERSION ?= v1.2.1 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.5 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 From c2ca9089caf4bcec046461dc38e648b3ab21f452 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 6 Apr 2023 18:29:44 +0200 Subject: [PATCH 034/279] docs(emqx_machine): Add readme --- apps/emqx_machine/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 apps/emqx_machine/README.md diff --git a/apps/emqx_machine/README.md b/apps/emqx_machine/README.md new file mode 100644 index 000000000..9ff33a5e5 --- /dev/null +++ b/apps/emqx_machine/README.md @@ -0,0 +1,5 @@ +# EMQX Machine + +This application manages other OTP applications in EMQX and serves as the entry point when BEAM VM starts up. +It prepares the node before starting mnesia/mria, as well as EMQX business logic. +It keeps track of the business applications storing data in Mnesia, which need to be restarted when the node joins the cluster by registering `ekka` callbacks. From c68909767a90dbf414260baa400c42d8c8483bc7 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 10 Apr 2023 10:07:58 +0800 Subject: [PATCH 035/279] feat: more straightforward log for force_shutdown reason --- apps/emqx/src/emqx_misc.erl | 2 +- apps/emqx/src/emqx_schema.erl | 18 ++++++++++-------- apps/emqx/test/emqx_misc_SUITE.erl | 5 ++++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index cdd62df11..d0c7397c8 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -246,7 +246,7 @@ do_check_oom([]) -> ok; do_check_oom([{Val, Max, Reason} | Rest]) -> case is_integer(Max) andalso (0 < Max) andalso (Max < Val) of - true -> {shutdown, Reason}; + true -> {shutdown, #{reason => Reason, value => Val, max => Max}}; false -> do_check_oom(Rest) end. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 6bfff38d3..354e903a8 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2642,20 +2642,22 @@ to_atom(Str) when is_list(Str) -> to_atom(Bin) when is_binary(Bin) -> binary_to_atom(Bin, utf8). -validate_heap_size(Siz) -> +validate_heap_size(Siz) when is_integer(Siz) -> MaxSiz = case erlang:system_info(wordsize) of % arch_64 - 8 -> - (1 bsl 59) - 1; + 8 -> (1 bsl 59) - 1; % arch_32 - 4 -> - (1 bsl 27) - 1 + 4 -> (1 bsl 27) - 1 end, case Siz > MaxSiz of - true -> error(io_lib:format("force_shutdown_policy: heap-size ~ts is too large", [Siz])); - false -> ok - end. + true -> + {error, #{reason => max_heap_size_too_large, maximum => MaxSiz}}; + false -> + ok + end; +validate_heap_size(_SizStr) -> + {error, invalid_heap_size}. validate_alarm_actions(Actions) -> UnSupported = lists:filter( diff --git a/apps/emqx/test/emqx_misc_SUITE.erl b/apps/emqx/test/emqx_misc_SUITE.erl index c2bd751fa..c068a087e 100644 --- a/apps/emqx/test/emqx_misc_SUITE.erl +++ b/apps/emqx/test/emqx_misc_SUITE.erl @@ -146,7 +146,10 @@ t_check(_) -> [self() ! {msg, I} || I <- lists:seq(1, 5)], ?assertEqual(ok, emqx_misc:check_oom(Policy)), [self() ! {msg, I} || I <- lists:seq(1, 6)], - ?assertEqual({shutdown, message_queue_too_long}, emqx_misc:check_oom(Policy)). + ?assertEqual( + {shutdown, #{reason => message_queue_too_long, value => 11, max => 10}}, + emqx_misc:check_oom(Policy) + ). drain() -> drain([]). From 5a58dfc3a44d485010fd447cb6213340ba8f0f44 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 10 Apr 2023 14:39:13 +0800 Subject: [PATCH 036/279] chore: update README for auto subscribe --- apps/emqx_auto_subscribe/README.md | 57 ++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/apps/emqx_auto_subscribe/README.md b/apps/emqx_auto_subscribe/README.md index 96d368715..981e4cb1f 100644 --- a/apps/emqx_auto_subscribe/README.md +++ b/apps/emqx_auto_subscribe/README.md @@ -1,9 +1,54 @@ -emqx_auto_subscribe -===== +# Auto Subscribe -An OTP application +This application can help clients automatically subscribe to topics compiled from user definitions when they connect, and the clients no longer need to send the MQTT `SUBSCRIBE ` request. -Build ------ +# How To Use - $ rebar3 compile +Add the following configuration items to the `emqx.conf` file + +```yaml +auto_subscribe { + topics = [ + { + topic = "c/${clientid}" + }, + { + topic = "client/${clientid}/username/${username}/host/${host}/port/${port}" + qos = 1 + rh = 0 + rap = 0 + nl = 0 + } + ] +} +``` + +This example defines two templates, all of which will be compiled into the final topic by replacing placeholders like `${clientid}` `${port}` with the actual values when the client connects. + +# Configuration + +## Configuration Definition + +| Field | Definition | Value Range | Default | +| -------------- | ----------------------------- | ----------------------------------------------------------- | ------- | +| auto_subscribe | Auto subscribe configuration | topics | topics | +| topics | Subscription Options | Subscription configurations list. See `Subscription Option` | [] | + +## Subscription Option + +| Field | Definition | Value Range | Default | +|-------|---------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------|------------------| +| topic | Required. Topic template. | String, placeholders supported | No default value | +| qos | Optional. Subscription QoS | 0 or 1 or 2. Refer to the MQTT QoS definition | 0 | +| rh | Optional. MQTT version 5.0. Whether to send retain message when a subscription is created. | 0: Not send the retain message
1: Send the retain message | 0 | +| rap | Optional. MQTT version 5.0. When forwarding messages, Whether to send with retain flag | 0: Set retain 0
1: Keep retain flag | 0 | +| nl | Optional. MQTT version 5.0. Whether the message can be forwarded to the client when published by itself | 0: Forwarded to self
1: Not forwarded to self | 0 | + +## Subscription Placeholders + +| Placeholder | Definition | +| ----------- | -------------------------------------- | +| ${clientid} | Client ID | +| ${username} | Client Username | +| ${ip} | Client TCP connection local IP address | +| ${port} | Client TCP connection local Port | From 30bdffe3187be7db88f85821417a909bd7b9e536 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 10 Apr 2023 15:08:10 +0800 Subject: [PATCH 037/279] feat: support async and batch callback for cassandra connector --- lib-ee/emqx_ee_bridge/rebar.config | 2 +- .../test/emqx_ee_bridge_cassa_SUITE.erl | 65 +++------ .../src/emqx_ee_connector_cassa.erl | 125 ++++++++++++++++-- 3 files changed, 133 insertions(+), 59 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index 985b387d4..2df1f9f6a 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -3,7 +3,7 @@ , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}} - , {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.4.2"}}} + , {ecql, {git, "https://github.com/emqx/ecql.git", {branch, "batch-support"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index f1ea6e930..826df9be8 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -72,10 +72,10 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), - NonBatchCases = [t_write_timeout], + NonBatchCases = [t_write_timeout, t_simple_sql_query], QueryModeGroups = [{group, async}, {group, sync}], BatchingGroups = [ - %{group, with_batch}, + {group, with_batch}, {group, without_batch} ], [ @@ -404,12 +404,7 @@ t_setup_via_config_and_publish(Config) -> end, fun(Trace0) -> Trace = ?of_kind(cassandra_connector_query_return, Trace0), - case ?config(enable_batch, Config) of - true -> - ?assertMatch([#{result := {_, [ok]}}], Trace); - false -> - ?assertMatch([#{result := ok}], Trace) - end, + ?assertMatch([#{result := ok}], Trace), ok end ), @@ -448,12 +443,7 @@ t_setup_via_http_api_and_publish(Config) -> end, fun(Trace0) -> Trace = ?of_kind(cassandra_connector_query_return, Trace0), - case ?config(enable_batch, Config) of - true -> - ?assertMatch([#{result := {_, [{ok, 1}]}}], Trace); - false -> - ?assertMatch([#{result := ok}], Trace) - end, + ?assertMatch([#{result := ok}], Trace), ok end ), @@ -540,8 +530,8 @@ t_write_failure(Config) -> fun(Trace0) -> ct:pal("trace: ~p", [Trace0]), Trace = ?of_kind(buffer_worker_flush_nack, Trace0), - ?assertMatch([#{result := {error, _}} | _], Trace), - [#{result := {error, Error}} | _] = Trace, + ?assertMatch([#{result := {async_return, {error, _}}} | _], Trace), + [#{result := {async_return, {error, Error}}} | _] = Trace, case Error of {resource_error, _} -> ok; @@ -576,7 +566,6 @@ t_write_failure(Config) -> % ok. t_simple_sql_query(Config) -> - EnableBatch = ?config(enable_batch, Config), QueryMode = ?config(query_mode, Config), ?assertMatch( {ok, _}, @@ -592,12 +581,7 @@ t_simple_sql_query(Config) -> {ok, Res} = receive_result(Ref, 2_000), Res end, - case EnableBatch of - true -> - ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); - false -> - ?assertMatch({ok, {<<"system.local">>, _, [[1]]}}, Result) - end, + ?assertMatch({ok, {<<"system.local">>, _, [[1]]}}, Result), ok. t_missing_data(Config) -> @@ -607,27 +591,25 @@ t_missing_data(Config) -> ), %% emqx_ee_connector_cassa will send missed data as a `null` atom %% to ecql driver - {_, {ok, Event}} = - ?wait_async_action( - send_message(Config, #{}), - #{?snk_kind := buffer_worker_flush_ack}, - 2_000 - ), - ?assertMatch( - %% TODO: match error msgs + send_message(Config, #{}), + ?block_until( #{ - result := - {error, {unrecoverable_error, {8704, <<"Expected 8 or 0 byte long for date (4)">>}}} + ?snk_kind := buffer_worker_flush_ack, + result := {async_return, ok} }, - Event + 2_000 + ), + ?block_until( + #{ + ?snk_kind := handle_async_reply_enter, + result := {error, {8704, _}} + }, + 2_000 ), ok. t_bad_sql_parameter(Config) -> QueryMode = ?config(query_mode, Config), - EnableBatch = ?config(enable_batch, Config), - Name = ?config(cassa_name, Config), - ResourceId = emqx_bridge_resource:resource_id(cassandra, Name), ?assertMatch( {ok, _}, create_bridge( @@ -656,14 +638,7 @@ t_bad_sql_parameter(Config) -> ct:fail("no response received") end end, - case EnableBatch of - true -> - ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); - false -> - ?assertMatch( - {error, {unrecoverable_error, _}}, Result - ) - end, + ?assertMatch({error, _}, Result), ok. t_nasty_sql_string(Config) -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index a6a77f233..4b3804c01 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -33,8 +33,9 @@ on_start/2, on_stop/2, on_query/3, - %% TODO: not_supported_now - %%on_batch_query/3, + on_query_async/4, + on_batch_query/3, + on_batch_query_async/4, on_get_status/2 ]). @@ -45,7 +46,7 @@ ]). %% callbacks for query executing --export([query/3, prepared_query/3]). +-export([query/4, prepared_query/4, batch_query/3]). -export([do_get_status/1]). @@ -96,7 +97,7 @@ keyspace(_) -> undefined. %%-------------------------------------------------------------------- %% callbacks for emqx_resource -callback_mode() -> always_sync. +callback_mode() -> async_if_possible. -spec on_start(binary(), hoconsc:config()) -> {ok, state()} | {error, _}. on_start( @@ -172,6 +173,28 @@ on_stop(InstId, #{poolname := PoolName}) -> on_query( InstId, Request, + State +) -> + do_signle_query(InstId, Request, sync, State). + +-spec on_query_async( + emqx_resource:resource_id(), + request(), + {function(), list()}, + state() +) -> ok | {error, {recoverable_error | unrecoverable_error, term()}}. +on_query_async( + InstId, + Request, + Callback, + State +) -> + do_signle_query(InstId, Request, {async, Callback}, State). + +do_signle_query( + InstId, + Request, + Async, #{poolname := PoolName} = State ) -> {Type, PreparedKeyOrSQL, Params} = parse_request_to_cql(Request), @@ -187,7 +210,59 @@ on_query( } ), {PreparedKeyOrSQL1, Data} = proc_cql_params(Type, PreparedKeyOrSQL, Params, State), - Res = exec_cql_query(InstId, PoolName, Type, PreparedKeyOrSQL1, Data), + Res = exec_cql_query(InstId, PoolName, Type, Async, PreparedKeyOrSQL1, Data), + handle_result(Res). + +-spec on_batch_query( + emqx_resource:resource_id(), + [request()], + state() +) -> ok | {error, {recoverable_error | unrecoverable_error, term()}}. +on_batch_query( + InstId, + Requests, + State +) -> + do_batch_query(InstId, Requests, sync, State). + +-spec on_batch_query_async( + emqx_resource:resource_id(), + [request()], + {function(), list()}, + state() +) -> ok | {error, {recoverable_error | unrecoverable_error, term()}}. +on_batch_query_async( + InstId, + Requests, + Callback, + State +) -> + do_batch_query(InstId, Requests, {async, Callback}, State). + +do_batch_query( + InstId, + Requests, + Async, + #{poolname := PoolName} = State +) -> + CQLs = + lists:map( + fun(Request) -> + {Type, PreparedKeyOrSQL, Params} = parse_request_to_cql(Request), + proc_cql_params(Type, PreparedKeyOrSQL, Params, State) + end, + Requests + ), + ?tp( + debug, + cassandra_connector_received_cql_batch_query, + #{ + connector => InstId, + cqls => CQLs, + state => State + } + ), + Res = exec_cql_batch_query(InstId, PoolName, Async, CQLs), handle_result(Res). parse_request_to_cql({send_message, Params}) -> @@ -203,17 +278,32 @@ proc_cql_params( Params, #{prepare_statement := Prepares, params_tokens := ParamsTokens} ) -> - PreparedKey = maps:get(PreparedKey0, Prepares), + %% assert + _PreparedKey = maps:get(PreparedKey0, Prepares), Tokens = maps:get(PreparedKey0, ParamsTokens), - {PreparedKey, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}; + {PreparedKey0, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}; proc_cql_params(query, SQL, Params, _State) -> {SQL1, Tokens} = emqx_plugin_libs_rule:preproc_sql(SQL, '?'), {SQL1, assign_type_for_params(emqx_plugin_libs_rule:proc_sql(Tokens, Params))}. -exec_cql_query(InstId, PoolName, Type, PreparedKey, Data) when +exec_cql_query(InstId, PoolName, Type, Async, PreparedKey, Data) when Type == query; Type == prepared_query -> - case ecpool:pick_and_do(PoolName, {?MODULE, Type, [PreparedKey, Data]}, no_handover) of + case ecpool:pick_and_do(PoolName, {?MODULE, Type, [Async, PreparedKey, Data]}, no_handover) of + {error, Reason} = Result -> + ?tp( + error, + cassandra_connector_query_return, + #{connector => InstId, error => Reason} + ), + Result; + Result -> + ?tp(debug, cassandra_connector_query_return, #{result => Result}), + Result + end. + +exec_cql_batch_query(InstId, PoolName, Async, CQLs) -> + case ecpool:pick_and_do(PoolName, {?MODULE, batch_query, [Async, CQLs]}, no_handover) of {error, Reason} = Result -> ?tp( error, @@ -261,11 +351,20 @@ do_check_prepares(State = #{poolname := PoolName, prepare_cql := {error, Prepare %%-------------------------------------------------------------------- %% callbacks query -query(Conn, SQL, Params) -> - ecql:query(Conn, SQL, Params). +query(Conn, sync, CQL, Params) -> + ecql:query(Conn, CQL, Params); +query(Conn, {async, Callback}, CQL, Params) -> + ecql:async_query(Conn, CQL, Params, one, Callback). -prepared_query(Conn, PreparedKey, Params) -> - ecql:execute(Conn, PreparedKey, Params). +prepared_query(Conn, sync, PreparedKey, Params) -> + ecql:execute(Conn, PreparedKey, Params); +prepared_query(Conn, {async, Callback}, PreparedKey, Params) -> + ecql:async_execute(Conn, PreparedKey, Params, Callback). + +batch_query(Conn, sync, Rows) -> + ecql:batch(Conn, Rows); +batch_query(Conn, {async, Callback}, Rows) -> + ecql:async_batch(Conn, Rows, Callback). %%-------------------------------------------------------------------- %% callbacks for ecpool From c50326360f30fc353f293ec4d7955fd3997431a5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 10 Apr 2023 18:03:58 +0800 Subject: [PATCH 038/279] chore: update ecql version --- lib-ee/emqx_ee_bridge/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index 2df1f9f6a..1c7d130ae 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -3,7 +3,7 @@ , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}} - , {ecql, {git, "https://github.com/emqx/ecql.git", {branch, "batch-support"}}} + , {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.1"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} From a711ce6b936b4f347b5619013afa0ab975884b36 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 10 Apr 2023 18:10:31 +0200 Subject: [PATCH 039/279] chore: bump version to e5.0.2 --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index fa339be36..84f4468e4 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.21"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.2-rc.6"). +-define(EMQX_RELEASE_EE, "5.0.2"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). From d501c1ee9c969995f16de42aa5d5b3738b35a627 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 Apr 2023 10:31:21 +0800 Subject: [PATCH 040/279] chore: update changes --- apps/emqx/test/emqx_banned_SUITE.erl | 3 +-- changes/ee/feat-10140.en.md | 2 -- changes/ee/feat-10140.zh.md | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/emqx/test/emqx_banned_SUITE.erl b/apps/emqx/test/emqx_banned_SUITE.erl index 80427ac47..0c14f64c9 100644 --- a/apps/emqx/test/emqx_banned_SUITE.erl +++ b/apps/emqx/test/emqx_banned_SUITE.erl @@ -186,9 +186,8 @@ t_session_taken(_) -> false end end, - 3000 + 6000 ), - Publish(), C2 = Connect(), diff --git a/changes/ee/feat-10140.en.md b/changes/ee/feat-10140.en.md index f2616fda1..42238a21f 100644 --- a/changes/ee/feat-10140.en.md +++ b/changes/ee/feat-10140.en.md @@ -1,4 +1,2 @@ Integrate `Cassandra` into `bridges` as a new backend. At the current stage: - Only support Cassandra version 3.x, not yet 4.x. -- Only support storing data in synchronously, not yet asynchronous and batch - method to store data to Cassandra. diff --git a/changes/ee/feat-10140.zh.md b/changes/ee/feat-10140.zh.md index 5b070133a..0d0ece3a0 100644 --- a/changes/ee/feat-10140.zh.md +++ b/changes/ee/feat-10140.zh.md @@ -1,3 +1,2 @@ 支持 Cassandra 数据桥接。在当前阶段： - 仅支持 Cassandra 3.x 版本，暂不支持 4.x。 -- 仅支持以同步的方式存储数据，暂不支持异步和批量的方式来存储数据到 Cassandra。 From aee85dc328fc0118f5088b5df11bcd934beada9c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 Apr 2023 14:49:07 +0800 Subject: [PATCH 041/279] chore: fix flaky tests --- apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 2427a10ee..1a4bab5f3 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -66,6 +66,7 @@ end_per_group(AuthName, Conf) -> Conf. init_per_suite(Config) -> + emqx_gateway_test_utils:load_all_gateway_apps(), emqx_config:erase(gateway), init_gateway_conf(), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), From e282d361f8501685871cd5b417b0606d653554f3 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 11 Apr 2023 09:34:51 +0200 Subject: [PATCH 042/279] style: fix Chinese translations Co-authored-by: LenaLenaPan <120552185+LenaLenaPan@users.noreply.github.com> --- rel/i18n/emqx_rule_api_schema.hocon | 2 +- rel/i18n/emqx_rule_engine_api.hocon | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index 3c2b3e7e4..0d8253223 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -641,7 +641,7 @@ emqx_rule_api_schema { root_rule_engine { desc { en: "Rule engine configurations. This API can be used to change EMQX rule engine settings. But not for the rules. To list, create, or update rules, call the '/rules' API instead." - zh: "规则引擎配置。该 API 可用于查看和修改规则引擎相关的一些设置。但不可用于规则，请调用 '/rules' API 来对规则进行操作。" + zh: "规则引擎配置。该 API 可用于查看和修改规则引擎相关的一些设置。但不可用于规则，如需查看或修改规则，请调用 '/rules' API 进行操作。" } label: { en: "Rule engine configuration" diff --git a/rel/i18n/emqx_rule_engine_api.hocon b/rel/i18n/emqx_rule_engine_api.hocon index 7be0588c9..8a57f8e31 100644 --- a/rel/i18n/emqx_rule_engine_api.hocon +++ b/rel/i18n/emqx_rule_engine_api.hocon @@ -53,7 +53,7 @@ emqx_rule_engine_api { api1_resp { desc { en: "List of rules" - zh: "列出所有规则" + zh: "规则列表" } label: { en: "List Rules" @@ -122,7 +122,7 @@ emqx_rule_engine_api { } label: { en: "Delete Cluster Rule" - zh: "删除集群规则" + zh: "基于给定 ID 新建一条规则" } } api7 { From e70deae1c3f9788c45c37730ea07838f71e9d77d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Mon, 10 Apr 2023 17:59:33 +0300 Subject: [PATCH 043/279] feat(resource): ask for metrics only when needed --- apps/emqx/priv/bpapi.versions | 1 + apps/emqx_authn/src/emqx_authn_api.erl | 4 +- .../emqx_authz/src/emqx_authz_api_sources.erl | 4 +- apps/emqx_bridge/src/emqx_bridge.erl | 5 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 141 +++++++++--------- .../src/proto/emqx_bridge_proto_v3.erl | 4 + .../src/proto/emqx_bridge_proto_v4.erl | 135 +++++++++++++++++ .../test/emqx_bridge_api_SUITE.erl | 15 +- apps/emqx_resource/src/emqx_resource.erl | 8 +- .../src/emqx_resource_manager.erl | 16 +- .../test/emqx_resource_SUITE.erl | 6 +- 11 files changed, 239 insertions(+), 100 deletions(-) create mode 100644 apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 6190925d2..041ed8ced 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -5,6 +5,7 @@ {emqx_bridge,1}. {emqx_bridge,2}. {emqx_bridge,3}. +{emqx_bridge,4}. {emqx_broker,1}. {emqx_cm,1}. {emqx_conf,1}. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index ad9cd8579..fc026a671 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -872,8 +872,8 @@ lookup_from_local_node(ChainName, AuthenticatorID) -> case emqx_resource:get_instance(ResourceId) of {error, not_found} -> {error, {NodeId, not_found_resource}}; - {ok, _, #{status := Status, metrics := ResourceMetrics}} -> - {ok, {NodeId, Status, Metrics, ResourceMetrics}} + {ok, _, #{status := Status}} -> + {ok, {NodeId, Status, Metrics, emqx_resource:get_metrics(ResourceId)}} end end; {error, Reason} -> diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 58fa471fc..0c2dee340 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -344,8 +344,8 @@ lookup_from_local_node(Type) -> case emqx_resource:get_instance(ResourceId) of {error, not_found} -> {error, {NodeId, not_found_resource}}; - {ok, _, #{status := Status, metrics := ResourceMetrics}} -> - {ok, {NodeId, Status, Metrics, ResourceMetrics}} + {ok, _, #{status := Status}} -> + {ok, {NodeId, Status, Metrics, emqx_resource:get_metrics(ResourceId)}} end; _ -> Metrics = emqx_metrics_worker:get_metrics(authz_metrics, Type), diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index bf91d20f7..6ff81a10b 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -34,7 +34,7 @@ unload/0, lookup/1, lookup/2, - lookup/3, + get_metrics/2, create/3, disable_enable/3, remove/2, @@ -271,6 +271,9 @@ lookup(Type, Name, RawConf) -> }} end. +get_metrics(Type, Name) -> + emqx_resource:get_metrics(emqx_bridge_resource:resource_id(Type, Name)). + maybe_upgrade(mqtt, Config) -> emqx_bridge_compatible_config:maybe_upgrade(Config); maybe_upgrade(webhook, Config) -> diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 44a478bca..b29cefacd 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -46,6 +46,7 @@ ]). -export([lookup_from_local_node/2]). +-export([get_metrics_from_local_node/2]). -define(BRIDGE_NOT_ENABLED, ?BAD_REQUEST(<<"Forbidden operation, bridge not enabled">>) @@ -467,7 +468,7 @@ schema("/bridges_probe") -> end; '/bridges'(get, _Params) -> Nodes = mria:running_nodes(), - NodeReplies = emqx_bridge_proto_v3:list_bridges_on_nodes(Nodes), + NodeReplies = emqx_bridge_proto_v4:list_bridges_on_nodes(Nodes), case is_ok(NodeReplies) of {ok, NodeBridges} -> AllBridges = [ @@ -524,7 +525,7 @@ schema("/bridges_probe") -> ). '/bridges/:id/metrics'(get, #{bindings := #{id := Id}}) -> - ?TRY_PARSE_ID(Id, lookup_from_all_nodes_metrics(BridgeType, BridgeName, 200)). + ?TRY_PARSE_ID(Id, get_metrics_from_all_nodes(BridgeType, BridgeName)). '/bridges/:id/metrics/reset'(put, #{bindings := #{id := Id}}) -> ?TRY_PARSE_ID( @@ -564,19 +565,21 @@ maybe_deobfuscate_bridge_probe(#{<<"type">> := BridgeType, <<"name">> := BridgeN maybe_deobfuscate_bridge_probe(Params) -> Params. -lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> - FormatFun = fun format_bridge_info/1, - do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun). - -lookup_from_all_nodes_metrics(BridgeType, BridgeName, SuccCode) -> - FormatFun = fun format_bridge_metrics/1, - do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun). - -do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun) -> +get_metrics_from_all_nodes(BridgeType, BridgeName) -> Nodes = mria:running_nodes(), - case is_ok(emqx_bridge_proto_v3:lookup_from_all_nodes(Nodes, BridgeType, BridgeName)) of + Result = do_bpapi_call(all, get_metrics_from_all_nodes, [Nodes, BridgeType, BridgeName]), + case Result of + Metrics when is_list(Metrics) -> + {200, format_bridge_metrics(lists:zip(Nodes, Metrics))}; + {error, Reason} -> + ?INTERNAL_ERROR(Reason) + end. + +lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) -> + Nodes = mria:running_nodes(), + case is_ok(emqx_bridge_proto_v4:lookup_from_all_nodes(Nodes, BridgeType, BridgeName)) of {ok, [{ok, _} | _] = Results} -> - {SuccCode, FormatFun([R || {ok, R} <- Results])}; + {SuccCode, format_bridge_info([R || {ok, R} <- Results])}; {ok, [{error, not_found} | _]} -> ?BRIDGE_NOT_FOUND(BridgeType, BridgeName); {error, Reason} -> @@ -603,6 +606,9 @@ create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) -> ?BAD_REQUEST(map_to_json(Reason)) end. +get_metrics_from_local_node(BridgeType, BridgeName) -> + format_metrics(emqx_bridge:get_metrics(BridgeType, BridgeName)). + '/bridges/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) -> ?TRY_PARSE_ID( Id, @@ -739,7 +745,7 @@ pick_bridges_by_id(Type, Name, BridgesAllNodes) -> ). format_bridge_info([FirstBridge | _] = Bridges) -> - Res = maps:without([node, metrics], FirstBridge), + Res = maps:remove(node, FirstBridge), NodeStatus = node_status(Bridges), redact(Res#{ status => aggregate_status(NodeStatus), @@ -766,7 +772,7 @@ aggregate_status(AllStatus) -> end. collect_metrics(Bridges) -> - [maps:with([node, metrics], B) || B <- Bridges]. + [#{node => Node, metrics => Metrics} || {Node, Metrics} <- Bridges]. aggregate_metrics(AllMetrics) -> InitMetrics = ?EMPTY_METRICS, @@ -800,9 +806,7 @@ aggregate_metrics( M15 + N15, M16 + N16, M17 + N17 - ); -aggregate_metrics(#{}, Metrics) -> - Metrics. + ). format_resource( #{ @@ -826,63 +830,57 @@ format_resource( ). format_resource_data(ResData) -> - maps:fold(fun format_resource_data/3, #{}, maps:with([status, metrics, error], ResData)). + maps:fold(fun format_resource_data/3, #{}, maps:with([status, error], ResData)). format_resource_data(error, undefined, Result) -> Result; format_resource_data(error, Error, Result) -> Result#{status_reason => emqx_misc:readable_error_msg(Error)}; -format_resource_data( - metrics, - #{ - counters := #{ - 'dropped' := Dropped, - 'dropped.other' := DroppedOther, - 'dropped.expired' := DroppedExpired, - 'dropped.queue_full' := DroppedQueueFull, - 'dropped.resource_not_found' := DroppedResourceNotFound, - 'dropped.resource_stopped' := DroppedResourceStopped, - 'matched' := Matched, - 'retried' := Retried, - 'late_reply' := LateReply, - 'failed' := SentFailed, - 'success' := SentSucc, - 'received' := Rcvd - }, - gauges := Gauges, - rate := #{ - matched := #{current := Rate, last5m := Rate5m, max := RateMax} - } - }, - Result -) -> - Queued = maps:get('queuing', Gauges, 0), - SentInflight = maps:get('inflight', Gauges, 0), - Result#{ - metrics => - ?METRICS( - Dropped, - DroppedOther, - DroppedExpired, - DroppedQueueFull, - DroppedResourceNotFound, - DroppedResourceStopped, - Matched, - Queued, - Retried, - LateReply, - SentFailed, - SentInflight, - SentSucc, - Rate, - Rate5m, - RateMax, - Rcvd - ) - }; format_resource_data(K, V, Result) -> Result#{K => V}. +format_metrics(#{ + counters := #{ + 'dropped' := Dropped, + 'dropped.other' := DroppedOther, + 'dropped.expired' := DroppedExpired, + 'dropped.queue_full' := DroppedQueueFull, + 'dropped.resource_not_found' := DroppedResourceNotFound, + 'dropped.resource_stopped' := DroppedResourceStopped, + 'matched' := Matched, + 'retried' := Retried, + 'late_reply' := LateReply, + 'failed' := SentFailed, + 'success' := SentSucc, + 'received' := Rcvd + }, + gauges := Gauges, + rate := #{ + matched := #{current := Rate, last5m := Rate5m, max := RateMax} + } +}) -> + Queued = maps:get('queuing', Gauges, 0), + SentInflight = maps:get('inflight', Gauges, 0), + ?METRICS( + Dropped, + DroppedOther, + DroppedExpired, + DroppedQueueFull, + DroppedResourceNotFound, + DroppedResourceStopped, + Matched, + Queued, + Retried, + LateReply, + SentFailed, + SentInflight, + SentSucc, + Rate, + Rate5m, + RateMax, + Rcvd + ). + fill_defaults(Type, RawConf) -> PackedConf = pack_bridge_conf(Type, RawConf), FullConf = emqx_config:fill_defaults(emqx_bridge_schema, PackedConf, #{}), @@ -990,7 +988,7 @@ do_bpapi_call(Node, Call, Args) -> do_bpapi_call_vsn(SupportedVersion, Call, Args) -> case lists:member(SupportedVersion, supported_versions(Call)) of true -> - apply(emqx_bridge_proto_v3, Call, Args); + apply(emqx_bridge_proto_v4, Call, Args); false -> {error, not_implemented} end. @@ -1000,9 +998,10 @@ maybe_unwrap({error, not_implemented}) -> maybe_unwrap(RpcMulticallResult) -> emqx_rpc:unwrap_erpc(RpcMulticallResult). -supported_versions(start_bridge_to_node) -> [2, 3]; -supported_versions(start_bridges_to_all_nodes) -> [2, 3]; -supported_versions(_Call) -> [1, 2, 3]. +supported_versions(start_bridge_to_node) -> [2, 3, 4]; +supported_versions(start_bridges_to_all_nodes) -> [2, 3, 4]; +supported_versions(get_metrics_from_all_nodes) -> [4]; +supported_versions(_Call) -> [1, 2, 3, 4]. redact(Term) -> emqx_misc:redact(Term). diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl index a35db5d96..0b496364a 100644 --- a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v3.erl @@ -20,6 +20,7 @@ -export([ introduced_in/0, + deprecated_since/0, list_bridges/1, list_bridges_on_nodes/1, @@ -39,6 +40,9 @@ introduced_in() -> "5.0.21". +deprecated_since() -> + "5.0.22". + -spec list_bridges(node()) -> list() | emqx_rpc:badrpc(). list_bridges(Node) -> rpc:call(Node, emqx_bridge, list, [], ?TIMEOUT). diff --git a/apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl new file mode 100644 index 000000000..937065e41 --- /dev/null +++ b/apps/emqx_bridge/src/proto/emqx_bridge_proto_v4.erl @@ -0,0 +1,135 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_proto_v4). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + list_bridges_on_nodes/1, + restart_bridge_to_node/3, + start_bridge_to_node/3, + stop_bridge_to_node/3, + lookup_from_all_nodes/3, + get_metrics_from_all_nodes/3, + restart_bridges_to_all_nodes/3, + start_bridges_to_all_nodes/3, + stop_bridges_to_all_nodes/3 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +-define(TIMEOUT, 15000). + +introduced_in() -> + "5.0.22". + +-spec list_bridges_on_nodes([node()]) -> + emqx_rpc:erpc_multicall([emqx_resource:resource_data()]). +list_bridges_on_nodes(Nodes) -> + erpc:multicall(Nodes, emqx_bridge, list, [], ?TIMEOUT). + +-type key() :: atom() | binary() | [byte()]. + +-spec restart_bridge_to_node(node(), key(), key()) -> + term(). +restart_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridge_to_node(node(), key(), key()) -> + term(). +start_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridge_to_node(node(), key(), key()) -> + term(). +stop_bridge_to_node(Node, BridgeType, BridgeName) -> + rpc:call( + Node, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec restart_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +restart_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + restart, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec start_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +start_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + start, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec stop_bridges_to_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +stop_bridges_to_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_resource, + stop, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec lookup_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(). +lookup_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + lookup_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). + +-spec get_metrics_from_all_nodes([node()], key(), key()) -> + emqx_rpc:erpc_multicall(emqx_metrics_worker:metrics()). +get_metrics_from_all_nodes(Nodes, BridgeType, BridgeName) -> + erpc:multicall( + Nodes, + emqx_bridge_api, + get_metrics_from_local_node, + [BridgeType, BridgeName], + ?TIMEOUT + ). diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 8899cd24a..4d5495ffe 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1201,13 +1201,16 @@ t_metrics(Config) -> request_json(get, uri(["bridges", BridgeID, "metrics"]), Config) ), - %% check that metrics isn't returned when listing all bridges + %% check for absence of metrics when listing all bridges {ok, 200, Bridges} = request_json(get, uri(["bridges"]), Config), - ?assert( - lists:all( - fun(E) -> not maps:is_key(<<"metrics">>, E) end, - Bridges - ) + ?assertNotMatch( + [ + #{ + <<"metrics">> := #{}, + <<"node_metrics">> := [_ | _] + } + ], + Bridges ), ok. diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 0f7e93a9b..054bd228b 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -103,6 +103,7 @@ list_instances_verbose/0, %% return the data of the instance get_instance/1, + get_metrics/1, fetch_creation_opts/1, %% return all the instances of the same resource type list_instances_by_type/1, @@ -311,7 +312,12 @@ set_resource_status_connecting(ResId) -> -spec get_instance(resource_id()) -> {ok, resource_group(), resource_data()} | {error, Reason :: term()}. get_instance(ResId) -> - emqx_resource_manager:lookup_cached(ResId, [metrics]). + emqx_resource_manager:lookup_cached(ResId). + +-spec get_metrics(resource_id()) -> + emqx_metrics_worker:metrics(). +get_metrics(ResId) -> + emqx_resource_manager:get_metrics(ResId). -spec fetch_creation_opts(map()) -> creation_opts(). fetch_creation_opts(Opts) -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 6a4919b41..b5ccff05c 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -37,7 +37,6 @@ list_all/0, list_group/1, lookup_cached/1, - lookup_cached/2, get_metrics/1, reset_metrics/1 ]). @@ -231,25 +230,14 @@ set_resource_status_connecting(ResId) -> -spec lookup(resource_id()) -> {ok, resource_group(), resource_data()} | {error, not_found}. lookup(ResId) -> case safe_call(ResId, lookup, ?T_LOOKUP) of - {error, timeout} -> lookup_cached(ResId, [metrics]); + {error, timeout} -> lookup_cached(ResId); Result -> Result end. %% @doc Lookup the group and data of a resource from the cache -spec lookup_cached(resource_id()) -> {ok, resource_group(), resource_data()} | {error, not_found}. lookup_cached(ResId) -> - lookup_cached(ResId, []). - -%% @doc Lookup the group and data of a resource from the cache --spec lookup_cached(resource_id(), [Option]) -> - {ok, resource_group(), resource_data()} | {error, not_found} -when - Option :: metrics. -lookup_cached(ResId, Options) -> - NeedMetrics = lists:member(metrics, Options), case read_cache(ResId) of - {Group, Data} when NeedMetrics -> - {ok, Group, data_record_to_external_map_with_metrics(Data)}; {Group, Data} -> {ok, Group, data_record_to_external_map(Data)}; not_found -> @@ -366,7 +354,7 @@ handle_event({call, From}, {remove, ClearMetrics}, _State, Data) -> handle_remove_event(From, ClearMetrics, Data); % Called when the state-data of the resource is being looked up. handle_event({call, From}, lookup, _State, #data{group = Group} = Data) -> - Reply = {ok, Group, data_record_to_external_map_with_metrics(Data)}, + Reply = {ok, Group, data_record_to_external_map(Data)}, {keep_state_and_data, [{reply, From, Reply}]}; % Called when doing a manually health check. handle_event({call, From}, health_check, stopped, _Data) -> diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 8638c381f..5d8e85697 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -349,7 +349,7 @@ t_query_counter_async_query(_) -> ?assertMatch([#{query := {query, _, get_counter, _, _}}], QueryTrace) end ), - {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID), + #{counters := C} = emqx_resource:get_metrics(?ID), ?assertMatch(#{matched := 1002, 'success' := 1002, 'failed' := 0}, C), ok = emqx_resource:remove_local(?ID). @@ -402,7 +402,7 @@ t_query_counter_async_callback(_) -> ?assertMatch([#{query := {query, _, get_counter, _, _}}], QueryTrace) end ), - {ok, _, #{metrics := #{counters := C}}} = emqx_resource:get_instance(?ID), + #{counters := C} = emqx_resource:get_metrics(?ID), ?assertMatch(#{matched := 1002, 'success' := 1002, 'failed' := 0}, C), ?assertMatch(1000, ets:info(Tab0, size)), ?assert( @@ -2702,7 +2702,7 @@ config() -> Config. tap_metrics(Line) -> - {ok, _, #{metrics := #{counters := C, gauges := G}}} = emqx_resource:get_instance(?ID), + #{counters := C, gauges := G} = emqx_resource:get_metrics(?ID), ct:pal("metrics (l. ~b): ~p", [Line, #{counters => C, gauges => G}]), #{counters => C, gauges => G}. From 945c6bc757e06ac348a2f09b634b7a821632ed70 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 11 Apr 2023 17:26:33 +0800 Subject: [PATCH 044/279] docs: update gateway README --- apps/emqx_gateway/README.md | 54 ++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index 57e8febab..581f0245d 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -1,4 +1,4 @@ -# emqx_gateway +# Gateway EMQX Gateway is an application that managing all gateways in EMQX. @@ -11,16 +11,42 @@ protocol access on EMQX. For example: - Configuration & Schema - HTTP/CLI management interfaces -There are some standard implementations available, such as [Stomp](../emqx_stomp/README.md), -[MQTT-SN](../emqx_mqttsn/README.md), [CoAP](../emqx_coap/README.md), -and [LwM2M](../emqx_lwm2m/README.md) gateway. - The emqx_gateway application depends on `emqx`, `emqx_authn`, `emqx_ctl` that provide the foundation for protocol access. -## Three ways to create your gateway +More introduction: [Extended Protocol Gateway](https://www.emqx.io/docs/en/v5.0/gateway/gateway.html) -## Raw Erlang Application +## Usage + +This application is just a Framework, we provide some standard implementations, +such as [Stomp](../emqx_stomp/README.md), [MQTT-SN](../emqx_mqttsn/README.md), +[CoAP](../emqx_coap/README.md) and [LwM2M](../emqx_lwm2m/README.md) gateway. + +These applications are all packaged by default in the EMQX distribution. If you +need to start a certain gateway, you only need to enable it via +Dashboard, HTTP API or emqx.conf file. + +For instance, enable the Stomp gateway in emqx.conf: +```hocon +gateway.stomp { + + mountpoint = "stomp/" + + listeners.tcp.default { + bind = 61613 + acceptors = 16 + max_connections = 1024000 + max_conn_rate = 1000 + } +} +``` + +## How to develop your Gateway application + +There are three ways to develop Gateway application to accept your private protocol +clients. + +### Raw Erlang Application This approach is the same as in EMQX 4.x. You need to implement an Erlang application, which is packaged in EMQX as a [Plugin](todo) or as a source code dependency. @@ -30,7 +56,7 @@ and you can freely implement the features you need. Steps guide: [Implement Gateway via Raw Application](doc/implement_gateway_via_raw_appliction.md) -## Respect emqx_gateway framework +### Respect emqx_gateway framework Similar to the first approach, you still need to implement an application using Erlang and package it into EMQX. @@ -43,7 +69,7 @@ by the emqx_gateway framework, even if it may require you to understand more det Steps guide: [Implement Gateway via Gateway framework](doc/implement_gateway_via_gateway_framekwork.md) -## Use ExProto Gateway (Non-Erlang developers) +### Use ExProto Gateway (Non-Erlang developers) If you want to implement your gateway using other programming languages such as Java, Python, Go, etc. @@ -53,6 +79,14 @@ your device protocol and integrate it with EMQX. Refer to: [ExProto Gateway](../emqx_exproto/README.md) -## Cookbook for emqx_gateway framework +## Understard emqx_gateway framework *WIP* + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + +## License + +See [LICENSE](../../LICENSE) From 00301935bbe6dde4402d9b17d9bb3f1eaf5d029d Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 11 Apr 2023 08:56:46 +0200 Subject: [PATCH 045/279] chore: update changelog entries --- changes/ce/feat-9986.en.md | 2 +- changes/ce/fix-10014.en.md | 2 +- changes/ce/fix-10014.zh.md | 2 +- changes/ce/fix-10055.en.md | 2 +- changes/ce/fix-10107.en.md | 2 +- changes/ce/fix-10154.zh.md | 1 + changes/ce/fix-10237.zh.md | 1 + changes/ce/fix-10251.zh.md | 1 + changes/ce/fix-10314.zh.md | 1 + changes/ce/fix-10327.zh.md | 1 + changes/ee/feat-9564.en.md | 2 +- changes/ee/feat-9881.en.md | 2 +- 12 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 changes/ce/fix-10154.zh.md create mode 100644 changes/ce/fix-10237.zh.md create mode 100644 changes/ce/fix-10251.zh.md create mode 100644 changes/ce/fix-10314.zh.md create mode 100644 changes/ce/fix-10327.zh.md diff --git a/changes/ce/feat-9986.en.md b/changes/ce/feat-9986.en.md index ee7a6be71..7a63c2e74 100644 --- a/changes/ce/feat-9986.en.md +++ b/changes/ce/feat-9986.en.md @@ -1 +1 @@ -For helm charts, add MQTT ingress bridge; and removed stale `mgmt` references. +Add MQTT ingress to helm charts and update helm charts documentation diff --git a/changes/ce/fix-10014.en.md b/changes/ce/fix-10014.en.md index d52452bf9..2a3674772 100644 --- a/changes/ce/fix-10014.en.md +++ b/changes/ce/fix-10014.en.md @@ -1 +1 @@ -In dashboard API for `/monitor(_current)/nodes/:node` return `404` instead of `400` if node does not exist. +Ensure Monitor API `/monitor(_current)/nodes/:node` returns `404` instead of `400` if node does not exist. diff --git a/changes/ce/fix-10014.zh.md b/changes/ce/fix-10014.zh.md index 5e6a1660f..1ce2a3c43 100644 --- a/changes/ce/fix-10014.zh.md +++ b/changes/ce/fix-10014.zh.md @@ -1 +1 @@ -如果 API 查询的节点不存在，将会返回 404 而不再是 400。 +如果 API 查询的节点不存在，将会返回 `404` 而不再是 `400`。 diff --git a/changes/ce/fix-10055.en.md b/changes/ce/fix-10055.en.md index 15237c135..b22a319c6 100644 --- a/changes/ce/fix-10055.en.md +++ b/changes/ce/fix-10055.en.md @@ -1 +1 @@ -Fix: configuration parameter `mqtt.max_awaiting_rel` had no effect. +The configuration parameter `mqtt.max_awaiting_rel` was not functional and has now been corrected. diff --git a/changes/ce/fix-10107.en.md b/changes/ce/fix-10107.en.md index 1bcbbad60..6f76bc083 100644 --- a/changes/ce/fix-10107.en.md +++ b/changes/ce/fix-10107.en.md @@ -1,4 +1,4 @@ -For operations on `bridges API` if `bridge-id` is unknown we now return `404` +For operations on Bridges API if `bridge-id` is unknown we now return `404` instead of `400`. Also a bug was fixed that caused a crash if that was a node operation. Additionally we now also check if the given bridge is enabled when doing the cluster operation `start` . Affected endpoints: diff --git a/changes/ce/fix-10154.zh.md b/changes/ce/fix-10154.zh.md new file mode 100644 index 000000000..8adf365ae --- /dev/null +++ b/changes/ce/fix-10154.zh.md @@ -0,0 +1 @@ +将数据桥接和连接器的 resume_interval 参数值设为 health_check_interval 和 request_timeout / 3 中的较小值，以解决请求超时的问题。 diff --git a/changes/ce/fix-10237.zh.md b/changes/ce/fix-10237.zh.md new file mode 100644 index 000000000..c4bae060f --- /dev/null +++ b/changes/ce/fix-10237.zh.md @@ -0,0 +1 @@ +当调用 `/nodes/:node[/metrics|/stats]` API ，若节点不存在则返回 `404` 状态码。 diff --git a/changes/ce/fix-10251.zh.md b/changes/ce/fix-10251.zh.md new file mode 100644 index 000000000..0c2e0ba09 --- /dev/null +++ b/changes/ce/fix-10251.zh.md @@ -0,0 +1 @@ +修复了当删除一个使用中的 ingress 类型的桥接时，未提示存在规则依赖的问题。 diff --git a/changes/ce/fix-10314.zh.md b/changes/ce/fix-10314.zh.md new file mode 100644 index 000000000..8b99197ca --- /dev/null +++ b/changes/ce/fix-10314.zh.md @@ -0,0 +1 @@ +修复 `/monitor_current` API ，使其仅查看当前 节点。修复了 `/stats` API，以防止当集群中的一个或多个节点关闭时出现崩溃。 diff --git a/changes/ce/fix-10327.zh.md b/changes/ce/fix-10327.zh.md new file mode 100644 index 000000000..e99bda40a --- /dev/null +++ b/changes/ce/fix-10327.zh.md @@ -0,0 +1 @@ +在收到不可恢复的错误时，不要增加 'actions.failed.unknown' 规则指标计数。 diff --git a/changes/ee/feat-9564.en.md b/changes/ee/feat-9564.en.md index 4405e3e07..93cbcdb0d 100644 --- a/changes/ee/feat-9564.en.md +++ b/changes/ee/feat-9564.en.md @@ -1,2 +1,2 @@ -Implemented Kafka Consumer bridge. +Implement Kafka Consumer bridge. Now it's possible to consume messages from Kafka and publish them to MQTT topics. diff --git a/changes/ee/feat-9881.en.md b/changes/ee/feat-9881.en.md index 546178965..b5c6de484 100644 --- a/changes/ee/feat-9881.en.md +++ b/changes/ee/feat-9881.en.md @@ -1,4 +1,4 @@ -In this pull request, we have enhanced the error logs related to InfluxDB connectivity health checks. +Enhance the error logs related to InfluxDB connectivity health checks. Previously, if InfluxDB failed to pass the health checks using the specified parameters, the only message provided was "timed out waiting for it to become healthy". With the updated implementation, the error message will be displayed in both the logs and the dashboard, enabling easier identification and resolution of the issue. From b48fb17f4a0dc2281b4e6904932e01137e6bd9e1 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 11 Apr 2023 10:20:25 +0200 Subject: [PATCH 046/279] fix: CHECK_PARAMS macro defines unused var --- .../src/emqx_rule_engine_api.erl | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index b138d992e..b19816542 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -447,20 +447,12 @@ param_path_id() -> '/rule_engine'(get, _Params) -> {200, format_rule_engine_resp(emqx_conf:get([rule_engine]))}; '/rule_engine'(put, #{body := Params}) -> - ?CHECK_PARAMS( - Params, - rule_engine, - case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of - {ok, #{config := Config}} -> - {200, format_rule_engine_resp(Config)}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "update_rule_engine_failed", - reason => Reason - }), - {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} - end - ). + case rule_engine_update(Params) of + {ok, Config} -> + {200, format_rule_engine_resp(Config)}; + {error, Reason} -> + {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} + end. %%------------------------------------------------------------------------------ %% Internal functions @@ -731,3 +723,20 @@ run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) - run_fuzzy_match(E, Fuzzy); run_fuzzy_match(E, [_ | Fuzzy]) -> run_fuzzy_match(E, Fuzzy). + +rule_engine_update(Params) -> + case emqx_rule_api_schema:check_params(Params, rule_engine) of + {ok, _CheckedParams} -> + case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of + {ok, #{config := Config}} -> + {ok, Config}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "update_rule_engine_failed", + reason => Reason + }), + {error, Reason} + end; + {error, Reason} -> + {error, Reason} + end. From 8ef36f29ce252490b6e05188893f2dbd92ebaf48 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 11 Apr 2023 10:40:11 +0200 Subject: [PATCH 047/279] fix: add 'rule_engine' as possible tag() value for spec --- apps/emqx_rule_engine/src/emqx_rule_api_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 8a8822044..c9926f56f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -26,7 +26,7 @@ -export([roots/0, fields/1]). --type tag() :: rule_creation | rule_test. +-type tag() :: rule_creation | rule_test | rule_engine. -spec check_params(map(), tag()) -> {ok, map()} | {error, term()}. check_params(Params, Tag) -> From 33811ebf5977426a7a9360086167cf0c319007fa Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 11 Apr 2023 13:21:57 +0200 Subject: [PATCH 048/279] docs: Generate changelog for e5.0.2 --- changes/e5.0.2.en.md | 135 +++++++++++++++++++++++++++++++++++++++++++ changes/e5.0.2.zh.md | 118 +++++++++++++++++++++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 changes/e5.0.2.en.md create mode 100644 changes/e5.0.2.zh.md diff --git a/changes/e5.0.2.en.md b/changes/e5.0.2.en.md new file mode 100644 index 000000000..8d552ec17 --- /dev/null +++ b/changes/e5.0.2.en.md @@ -0,0 +1,135 @@ +# e5.0.2 + +## Enhancements + +- [#10022](https://github.com/emqx/emqx/pull/10022) Start releasing Rocky Linux 9 (compatible with Enterprise Linux 9) and MacOS 12 packages + +- [#10139](https://github.com/emqx/emqx/pull/10139) Add `extraVolumeMounts` to EMQX Helm Chart, it will have the ability to mount the user-own files into the EMQX instance, for example, ACL rule files as mentioned in [#9052](https://github.com/emqx/emqx/issues/9052) + Done of [#10116](https://github.com/emqx/emqx/issues/10116) + +- [#9893](https://github.com/emqx/emqx/pull/9893) When connecting with the flag `clean_start=false`, EMQX will filter out messages that published by banned clients. + Previously, the messages sent by banned clients may still be delivered to subscribers in this scenario. + +- [#9986](https://github.com/emqx/emqx/pull/9986) Add MQTT ingress to helm charts and update helm charts documentation + +- [#10083](https://github.com/emqx/emqx/pull/10083) Add `DynamoDB` support for Data-Brdige. + +- [#9564](https://github.com/emqx/emqx/pull/9564) Implement Kafka Consumer bridge. + Now it's possible to consume messages from Kafka and publish them to MQTT topics. + +- [#9881](https://github.com/emqx/emqx/pull/9881) Enhance the error logs related to InfluxDB connectivity health checks. + Previously, if InfluxDB failed to pass the health checks using the specified parameters, the only message provided was "timed out waiting for it to become healthy". + With the updated implementation, the error message will be displayed in both the logs and the dashboard, enabling easier identification and resolution of the issue. + +- [#10123](https://github.com/emqx/emqx/pull/10123) Improve the performance of `/bridges` API. + Earlier, when the number of nodes in the cluster was large or the node was busy, the API may have a request timeout. + +- [#9998](https://github.com/emqx/emqx/pull/9998) Redact the HTTP request body in the authentication error logs for security reasons. + +## Bug Fixes + +- [#10013](https://github.com/emqx/emqx/pull/10013) Fix return type structure for error case in API schema for `/gateways/:name/clients`. + +- [#10014](https://github.com/emqx/emqx/pull/10014) Ensure Monitor API `/monitor(_current)/nodes/:node` returns `404` instead of `400` if node does not exist. + +- [#10026](https://github.com/emqx/emqx/pull/10026) Metrics are now only exposed via the /bridges/:id/metrics endpoint. Metrics are no longer returned in other API operations such as getting the list of all bridges, or in the response when a bridge has been created. + +- [#10027](https://github.com/emqx/emqx/pull/10027) Allow setting node name from `EMQX_NODE__NAME` when running in docker. + Prior to this fix, only `EMQX_NODE_NAME` is allowed. + +- [#10050](https://github.com/emqx/emqx/pull/10050) Ensure Bridge API returns `404` status code consistently for resources that don't exist. + +- [#10052](https://github.com/emqx/emqx/pull/10052) Improve daemon mode startup failure logs. + + Before this change, it was difficult for users to understand the reason for EMQX 'start' command failed to boot the node. + The only information they received was that the node did not start within the expected time frame, + and they were instructed to boot the node with 'console' command in the hope of obtaining some logs. + However, the node might actually be running, which could cause 'console' mode to fail for a different reason. + + With this new change, when daemon mode fails to boot, a diagnosis is issued. Here are the possible scenarios: + + * If the node cannot be found from `ps -ef`, the user is instructed to find information in log files `erlang.log.*`. + * If the node is found to be running but not responding to pings, the user is advised to check if the host name is resolvable and reachable. + * If the node is responding to pings, but the EMQX app is not running, it is likely a bug. In this case, the user is advised to report a Github issue. + +- [#10055](https://github.com/emqx/emqx/pull/10055) The configuration parameter `mqtt.max_awaiting_rel` was not functional and has now been corrected. + +- [#10056](https://github.com/emqx/emqx/pull/10056) Fix `/bridges` API status code. + - Return `400` instead of `403` in case of removing a data bridge that is dependent on an active rule. + - Return `400` instead of `403` in case of calling operations (start|stop|restart) when Data-Bridging is not enabled. + +- [#10066](https://github.com/emqx/emqx/pull/10066) Improve error messages for `/briges_probe` and `[/node/:node]/bridges/:id/:operation` API calls to make them more readable. And set HTTP status code to `400` instead of `500`. + +- [#10074](https://github.com/emqx/emqx/pull/10074) Check if type in `PUT /authorization/sources/:type` matches `type` given in body of request. + +- [#10079](https://github.com/emqx/emqx/pull/10079) Fix description of `shared_subscription_strategy`. + +- [#10085](https://github.com/emqx/emqx/pull/10085) Consistently return `404` for all requests on non existent source in `/authorization/sources/:source[/*]`. + +- [#10098](https://github.com/emqx/emqx/pull/10098) A crash with an error in the log file that happened when the MongoDB authorization module queried the database has been fixed. + +- [#10100](https://github.com/emqx/emqx/pull/10100) Fix channel crash for slow clients with enhanced authentication. + Previously, when the client was using enhanced authentication, but the Auth message was sent slowly or the Auth message was lost, the client process would crash. + +- [#10107](https://github.com/emqx/emqx/pull/10107) For operations on Bridges API if `bridge-id` is unknown we now return `404` + instead of `400`. Also a bug was fixed that caused a crash if that was a node + operation. Additionally we now also check if the given bridge is enabled when + doing the cluster operation `start` . Affected endpoints: + * [cluster] `/bridges/:id/:operation`, + * [node] `/nodes/:node/bridges/:id/:operation`, where `operation` is one of + `[start|stop|restart]`. + Moreover, for a node operation, EMQX checks if node name is in our cluster and + return `404` instead of `501`. + +- [#10117](https://github.com/emqx/emqx/pull/10117) Fix an error occurring when a joining node doesn't have plugins that are installed on other nodes in the cluster. + After this fix, the joining node will copy all the necessary plugins from other nodes. + +- [#10118](https://github.com/emqx/emqx/pull/10118) Fix problems related to manual joining of EMQX replicant nodes to the cluster. + Previously, after manually executing joining and then leaving the cluster, the `replicant` node can only run normally after restarting the node after joining the cluster again. + + [Mria PR](https://github.com/emqx/mria/pull/128) + +- [#10119](https://github.com/emqx/emqx/pull/10119) Fix crash when `statsd.server` is set to an empty string. + +- [#10124](https://github.com/emqx/emqx/pull/10124) The default heartbeat period for MongoDB has been increased to reduce the risk of too excessive logging to the MongoDB log file. + +- [#10130](https://github.com/emqx/emqx/pull/10130) Fix garbled config display in dashboard when the value is originally from environment variables. + For example, `env EMQX_STATSD__SERVER='127.0.0.1:8124' . /bin/emqx start` results in unreadable string (not '127.0.0.1:8124') displayed in Dashboard's Statsd settings page. + Related PR: [HOCON#234](https://github.com/emqx/hocon/pull/234). + +- [#10132](https://github.com/emqx/emqx/pull/10132) Fix some error logs generated by `systemctl stop emqx` command. + Prior to the fix, the command was not stopping jq and os_mon applications properly. + +- [#10144](https://github.com/emqx/emqx/pull/10144) Add `-setcookie` emulator flag when invoking `emqx ctl` to prevent problems with emqx cli when home directory is read only. Fixes [#10142](https://github.com/emqx/emqx/issues/10142). + +- [#10154](https://github.com/emqx/emqx/pull/10154) Change the default `resume_interval` for bridges and connectors to be + the minimum of `health_check_interval` and `request_timeout / 3`. + Also exposes it as a hidden configuration to allow fine tuning. + + Before this change, the default values for `resume_interval` meant + that, if a buffer ever got blocked due to resource errors or high + message volumes, then, by the time the buffer would try to resume its + normal operations, almost all requests would have timed out. + +- [#10157](https://github.com/emqx/emqx/pull/10157) Fixed default rate limit configuration not being applied correctly when creating a new listener. + +- [#10237](https://github.com/emqx/emqx/pull/10237) Ensure we return `404` status code for unknown node names in `/nodes/:node[/metrics|/stats]` API. + +- [#10251](https://github.com/emqx/emqx/pull/10251) Consider bridges referenced in `FROM` rule clauses as dependencies. + + Before this fix, when one tried to delete an ingress rule referenced in an action like `select * from "$bridges/mqtt:ingress"`, the UI would not trigger a warning about dependent rule actions. + +- [#10313](https://github.com/emqx/emqx/pull/10313) Ensure that when the core or replicant node starting, the `cluster-override.conf` file is only copied from the core node. + Previously, when sorting nodes by startup time, the core node may have copied this file from the replicant node. + +- [#10314](https://github.com/emqx/emqx/pull/10314) Fix /monitor_current API so that it only looks at the current node. + Fix /stats API to not crash when one or more nodes in the cluster are down. + +- [#10327](https://github.com/emqx/emqx/pull/10327) Don't increment 'actions.failed.unknown' rule metrics counter upon receiving unrecoverable bridge errors. + This counter is displayed on the dashboard's rule overview tab ('Action statistics' - 'Unknown'). + The fix is only applicable for synchronous bridges, as all rule actions for asynchronous bridges + are counted as successful (they increment 'actions.success' which is displayed as 'Action statistics' - 'Success'). + +- [#10095](https://github.com/emqx/emqx/pull/10095) Stop MySQL client from bombarding server repeatedly with unnecessary `PREPARE` queries on every batch, trashing the server and exhausting its internal limits. This was happening when the MySQL bridge was in the batch mode. + + Ensure safer and more careful escaping of strings and binaries in batch insert queries when the MySQL bridge is in the batch mode. diff --git a/changes/e5.0.2.zh.md b/changes/e5.0.2.zh.md new file mode 100644 index 000000000..a73372570 --- /dev/null +++ b/changes/e5.0.2.zh.md @@ -0,0 +1,118 @@ +# e5.0.2 + +## 增强 + +- [#10022](https://github.com/emqx/emqx/pull/10022) 开始发布Rocky Linux 9（与Enterprise Linux 9兼容）和 MacOS 12 软件包。 + +- [#10139](https://github.com/emqx/emqx/pull/10139) 将 `extraVolumeMounts` 添加到 EMQX Helm Chart 中，它将能够挂载用户自己的文件到 EMQX 实例中，例如在 [#9052](https://github.com/emqx/emqx/issues/9052) 中提到的 ACL 规则文件。 + 修复了 issue [#10116](https://github.com/emqx/emqx/issues/10116) + +- [#9893](https://github.com/emqx/emqx/pull/9893) 当使用 `clean_start=false` 标志连接时，EMQX 将会从消息队列中过滤出被封禁客户端发出的消息，使它们不能被下发给订阅者。 + 此前被封禁客户端发出的消息仍可能在这一场景下被下发给订阅者。 + +- [#9986](https://github.com/emqx/emqx/pull/9986) 在 helm chart 中新增了 MQTT 桥接 ingress 的配置参数；并删除了旧版本遗留的 `mgmt` 配置。 + +- [#10083](https://github.com/emqx/emqx/pull/10083) 为数据桥接增加 `DynamoDB` 支持。 + +- [#9564](https://github.com/emqx/emqx/pull/9564) 实现了 Kafka 消费者桥接。 + 现在可以从 Kafka 消费消息并将其发布到 MQTT 主题。 + +- [#9881](https://github.com/emqx/emqx/pull/9881) 增强了与 InfluxDB 连接健康检查相关的错误日志。 + 在此更改之前，如果使用配置的参数 InfluxDB 未能通过健康检查，用户仅能获得一个“超时”的信息。 + 现在，详细的错误消息将显示在日志和控制台，从而让用户更容易地识别和解决问题。 + +- [#10123](https://github.com/emqx/emqx/pull/10123) 改进 `/bridges` API 的性能。 + 此前，当集群中节点数目较多或节点忙时，该 API 可能出现请求超时的情况。 + +- [#9998](https://github.com/emqx/emqx/pull/9998) 出于安全原因，在身份验证错误日志中模糊 HTTP 请求正文。 + +## 修复 + +- [#10013](https://github.com/emqx/emqx/pull/10013) 修复 API `/gateways/:name/clients` 返回值的类型结构错误。 + +- [#10014](https://github.com/emqx/emqx/pull/10014) 如果 API 查询的节点不存在，将会返回 `404` 而不再是 `400`。 + +- [#10026](https://github.com/emqx/emqx/pull/10026) 现在只有显式调用 `/bridges/:id/metrics` 接口时才可以获得指标数据，而其他 API 接口将不再返回相关数据。 + +- [#10027](https://github.com/emqx/emqx/pull/10027) 在 docker 中启动时，允许使用 `EMQX_NODE__NAME` 环境变量来配置节点名。 + 在此修复前，只能使 `EMQX_NODE_NAME`。 + +- [#10050](https://github.com/emqx/emqx/pull/10050) 确保 Bridge API 对不存在的资源一致返回 `404` 状态代码。 + +- [#10052](https://github.com/emqx/emqx/pull/10052) 优化 EMQX daemon 模式启动启动失败的日志。 + + 在进行此更改之前，当 EMQX 用 `start` 命令启动失败时，用户很难理解出错的原因。 + 所知道的仅仅是节点未能在预期时间内启动，然后被指示以 `console` 式引导节点以获取一些日志。 + 然而，节点实际上可能正在运行，这可能会导致 `console` 模式因不同的原因而失败。 + + 此次修复后，启动脚本会发出诊断： + + * 如果无法从 `ps -ef` 中找到节点，则指示用户在 `erlang.log.*` 中查找信息。 + * 如果发现节点正在运行但不响应 ping，则建议用户检查节点主机名是否有效并可达。 + * 如果节点响应 ping 但 EMQX 应用程序未运行，则很可能是一个错误。在这种情况下，建议用户报告一个Github issue。 + +- [#10055](https://github.com/emqx/emqx/pull/10055) 修复配置项 `mqtt.max_awaiting_rel` 更新不生效问题。 + +- [#10056](https://github.com/emqx/emqx/pull/10056) 修复 `/bridges` API 的 HTTP 状态码。 + - 当删除被活动中的规则依赖的数据桥接时，将返回 `400` 而不是 `403` 。 + - 当数据桥接未启用时，调用操作（启动|停止|重启）将返回 `400` 而不是 `403`。 + +- [#10066](https://github.com/emqx/emqx/pull/10066) 改进 `/briges_probe` 和 `[/node/:node]/bridges/:id/:operation` API 调用的错误信息，使之更加易读。并将 HTTP 状态代码设置为 `400` 而不是 `500`。 + +- [#10074](https://github.com/emqx/emqx/pull/10074) 检查 `PUT /authorization/sources/:type` 中的类型是否与请求正文中的 `type` 相符。 + +- [#10079](https://github.com/emqx/emqx/pull/10079) 修正对 `shared_subscription_strategy` 的描述。 + + +- [#10085](https://github.com/emqx/emqx/pull/10085) 如果向 `/authorization/sources/:source[/*]` 请求的 `source` 不存在，将一致地返回 `404`。 + +- [#10098](https://github.com/emqx/emqx/pull/10098) 当 MongoDB 授权模块查询数据库时，在日志文件中发生的崩溃与错误已经被修复。 + +- [#10100](https://github.com/emqx/emqx/pull/10100) 修复响应较慢的客户端在使用增强认证时可能出现崩溃的问题。 + 此前，当客户端使用增强认证功能，但发送 Auth 报文较慢或 Auth 报文丢失时会导致客户端进程崩溃。 + +- [#10107](https://github.com/emqx/emqx/pull/10107) 现在对桥接的 API 进行调用时，如果 `bridge-id` 不存在，将会返回 `404`，而不再是`400`。 + 然后，还修复了这种情况下，在节点级别上进行 API 调用时，可能导致崩溃的问题。 + 另外，在启动某个桥接时，会先检查指定桥接是否已启用。 + 受影响的接口有： + * [cluster] `/bridges/:id/:operation`, + * [node] `/nodes/:node/bridges/:id/:operation`, + 其中 `operation` 是 `[start|stop|restart]` 之一。 + 此外，对于节点操作，EMQX 将检查节点是否存在于集群中，如果不在，则会返回`404`，而不再是`501`。 + +- [#10117](https://github.com/emqx/emqx/pull/10117) 修复节点加入集群时，由于缺少集其它节点已安装的插件所导致的错误。 + 在此修复后，加入集群的节点将从其它节点复制所有必须的插件。 + +- [#10118](https://github.com/emqx/emqx/pull/10118) 修复 `replicant` 节点因为手动加入 EMQX 集群导致的相关问题。 + 此前，手动执行 `加入集群-离开集群` 后，`replicant` 节点再次加入集群后只有重启节点才能正常运行。 + + [Mria PR](https://github.com/emqx/mria/pull/128) + +- [#10119](https://github.com/emqx/emqx/pull/10119) 修复 `statsd.server` 配置为空字符串时启动崩溃的问题。 + +- [#10124](https://github.com/emqx/emqx/pull/10124) 增加了 MongoDB 的默认心跳周期，以减少 MongoDB 日志文件记录过多的风险。 + +- [#10130](https://github.com/emqx/emqx/pull/10130) 修复通过环境变量配置启动的 EMQX 节点无法通过HTTP API获取到正确的配置信息。 + 比如：`EMQX_STATSD__SERVER='127.0.0.1:8124' ./bin/emqx start` 后通过 Dashboard看到的 Statsd 配置信息是乱码。 + 相关 PR: [HOCON:234](https://github.com/emqx/hocon/pull/234). + +- [#10132](https://github.com/emqx/emqx/pull/10132) 修复 `systemctl stop emqx` 命令没有正常停止 jq、os_mon 组件所产生一些错误日志。 + +- [#10144](https://github.com/emqx/emqx/pull/10144) 为 emqx 可执行文件加入 `-setcookie` 标志，以避免由于 home 目录只读，导致 emqx cli 所提供的 `emqx ctl` 等命令在执行时出现的一些问题。修复 [#10142](https://github.com/emqx/emqx/issues/10142)。 + +- [#10154](https://github.com/emqx/emqx/pull/10154) 将数据桥接和连接器的 resume_interval 参数值设为 health_check_interval 和 request_timeout / 3 中的较小值，以解决请求超时的问题。 + +- [#10157](https://github.com/emqx/emqx/pull/10157) 修复在创建新的监听器时，没有正确应用速率限制默认配置的问题。 + +- [#10237](https://github.com/emqx/emqx/pull/10237) 当调用 `/nodes/:node[/metrics|/stats]` API ，若节点不存在则返回 `404` 状态码。 + +- [#10251](https://github.com/emqx/emqx/pull/10251) 修复了当删除一个使用中的 ingress 类型的桥接时，未提示存在规则依赖的问题。 + +- [#10313](https://github.com/emqx/emqx/pull/10313) 确保当 core 或 replicant 节点启动时，仅从 core 节点复制 `cluster-override.conf` 文件。 + 此前按照节点启动时间排序时，core 节点可能从 replicant 节点复制该文件。 + +- [#10314](https://github.com/emqx/emqx/pull/10314) 修复 `/monitor_current` API ，使其仅查看当前 节点。修复了 `/stats` API，以防止当集群中的一个或多个节点关闭时出现崩溃。 + +- [#10327](https://github.com/emqx/emqx/pull/10327) 在收到不可恢复的错误时，不要增加 'actions.failed.unknown' 规则指标计数。 + +- [#10095](https://github.com/emqx/emqx/pull/10095) 优化 MySQL 桥接在批量模式下能更高效的使用预处理语句 ，减少了对 MySQL 服务器的写入压力， 并确保对 SQL 语句进行更安全和谨慎的转义。 From d5ae5ebfd824683c2a689b5eabee2cb7cd5a9e77 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 11 Apr 2023 16:17:07 +0300 Subject: [PATCH 049/279] chore: add changelog entry Co-Authored-By: ieQu1 <99872536+ieQu1@users.noreply.github.com> --- changes/ce/feat-10359.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-10359.en.md diff --git a/changes/ce/feat-10359.en.md b/changes/ce/feat-10359.en.md new file mode 100644 index 000000000..21a253556 --- /dev/null +++ b/changes/ce/feat-10359.en.md @@ -0,0 +1 @@ +Metrics now are not implicitly collected in places where API handlers don't make any use of them. Instead, a separate backplane RPC gathers cluster-wide metrics. \ No newline at end of file From e6f8682c47535e2101379495e2ceb9961946b12b Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Tue, 11 Apr 2023 15:42:56 +0200 Subject: [PATCH 050/279] fix: ensure we don't return 'rules' in rule_engine --- .../src/emqx_rule_engine_api.erl | 44 +------ .../test/emqx_rule_engine_api_SUITE.erl | 120 +++++++++--------- 2 files changed, 69 insertions(+), 95 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index b19816542..f640f8303 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -475,8 +475,6 @@ encode_nested_error(RuleError, Reason) -> {RuleError, Reason} end. -format_rule_info_resp(Rules) when is_list(Rules) -> - [format_rule_info_resp(R) || R <- Rules]; format_rule_info_resp({Id, Rule}) -> format_rule_info_resp(Rule#{id => Id}); format_rule_info_resp(#{ @@ -500,32 +498,8 @@ format_rule_info_resp(#{ description => Descr }. -format_rule_engine_resp(#{rules := Rules} = Config) -> - Config#{rules => maps:map(fun format_rule_resp/2, Rules)}. - -format_rule_resp( - _Id, - #{ - name := Name, - actions := Action, - sql := SQL, - enable := Enable, - description := Descr - } = Rule -) -> - Format = #{ - name => Name, - actions => format_action(Action), - sql => SQL, - enable => Enable, - description => Descr - }, - case Rule of - #{metadata := MetaData = #{created_at := CreatedAt}} -> - Format#{metadata => MetaData#{created_at => format_datetime(CreatedAt, millisecond)}}; - _ -> - Format - end. +format_rule_engine_resp(Config) -> + maps:remove(rules, Config). format_datetime(Timestamp, Unit) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, Unit}])). @@ -727,16 +701,10 @@ run_fuzzy_match(E, [_ | Fuzzy]) -> rule_engine_update(Params) -> case emqx_rule_api_schema:check_params(Params, rule_engine) of {ok, _CheckedParams} -> - case emqx_conf:update([rule_engine], Params, #{override_to => cluster}) of - {ok, #{config := Config}} -> - {ok, Config}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "update_rule_engine_failed", - reason => Reason - }), - {error, Reason} - end; + {ok, #{config := Config}} = emqx_conf:update([rule_engine], Params, #{ + override_to => cluster + }), + {ok, Config}; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index e59b5c6df..e94806a7b 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -23,6 +23,14 @@ -include_lib("common_test/include/ct.hrl"). -define(CONF_DEFAULT, <<"rule_engine {rules {}}">>). +-define(SIMPLE_RULE(NAME_SUFFIX), #{ + <<"description">> => <<"A simple rule">>, + <<"enable">> => true, + <<"actions">> => [#{<<"function">> => <<"console">>}], + <<"sql">> => <<"SELECT * from \"t/1\"">>, + <<"name">> => <<"test_rule", NAME_SUFFIX/binary>> +}). +-define(SIMPLE_RULE(ID, NAME_SUFFIX), ?SIMPLE_RULE(NAME_SUFFIX)#{<<"id">> => ID}). all() -> emqx_common_test_helpers:all(?MODULE). @@ -37,6 +45,9 @@ end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([emqx_conf, emqx_rule_engine]), ok. +init_per_testcase(t_crud_rule_api, Config) -> + meck:new(emqx_json, [passthrough]), + init_per_testcase(common, Config); init_per_testcase(_, Config) -> Config. @@ -48,7 +59,7 @@ end_per_testcase(_, _Config) -> emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), lists:foreach( fun(#{id := Id}) -> - emqx_rule_engine_api:'/rules/:id'( + {204} = emqx_rule_engine_api:'/rules/:id'( delete, #{bindings => #{id => Id}} ) @@ -57,45 +68,38 @@ end_per_testcase(_, _Config) -> ). t_crud_rule_api(_Config) -> - RuleID = <<"my_rule">>, - Params0 = #{ - <<"description">> => <<"A simple rule">>, - <<"enable">> => true, - <<"id">> => RuleID, - <<"actions">> => [#{<<"function">> => <<"console">>}], - <<"sql">> => <<"SELECT * from \"t/1\"">>, - <<"name">> => <<"test_rule">> - }, - {201, Rule} = emqx_rule_engine_api:'/rules'(post, #{body => Params0}), - %% if we post again with the same params, it return with 400 "rule id already exists" - ?assertMatch( - {400, #{code := _, message := _Message}}, - emqx_rule_engine_api:'/rules'(post, #{body => Params0}) - ), + RuleId = <<"my_rule">>, + Rule = simple_rule_fixture(RuleId, <<>>), + ?assertEqual(RuleId, maps:get(id, Rule)), - ?assertEqual(RuleID, maps:get(id, Rule)), {200, #{data := Rules}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), ct:pal("RList : ~p", [Rules]), ?assert(length(Rules) > 0), + %% if we post again with the same id, it return with 400 "rule id already exists" + ?assertMatch( + {400, #{code := _, message := _Message}}, + emqx_rule_engine_api:'/rules'(post, #{body => ?SIMPLE_RULE(RuleId, <<"some_other">>)}) + ), + {204} = emqx_rule_engine_api:'/rules/:id/metrics/reset'(put, #{ - bindings => #{id => RuleID} + bindings => #{id => RuleId} }), - {200, Rule1} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}), + {200, Rule1} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}), ct:pal("RShow : ~p", [Rule1]), ?assertEqual(Rule, Rule1), - {200, Metrics} = emqx_rule_engine_api:'/rules/:id/metrics'(get, #{bindings => #{id => RuleID}}), + {200, Metrics} = emqx_rule_engine_api:'/rules/:id/metrics'(get, #{bindings => #{id => RuleId}}), ct:pal("RMetrics : ~p", [Metrics]), - ?assertMatch(#{id := RuleID, metrics := _, node_metrics := _}, Metrics), + ?assertMatch(#{id := RuleId, metrics := _, node_metrics := _}, Metrics), {200, Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ - bindings => #{id => RuleID}, - body => Params0#{<<"sql">> => <<"select * from \"t/b\"">>} + bindings => #{id => RuleId}, + body => ?SIMPLE_RULE(RuleId)#{<<"sql">> => <<"select * from \"t/b\"">>} }), - {200, Rule3} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}), + {200, Rule3} = emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}), %ct:pal("RShow : ~p", [Rule3]), ?assertEqual(Rule3, Rule2), ?assertEqual(<<"select * from \"t/b\"">>, maps:get(sql, Rule3)), @@ -112,14 +116,14 @@ t_crud_rule_api(_Config) -> {204}, emqx_rule_engine_api:'/rules/:id'( delete, - #{bindings => #{id => RuleID}} + #{bindings => #{id => RuleId}} ) ), %ct:pal("Show After Deleted: ~p", [NotFound]), ?assertMatch( {404, #{code := _, message := _Message}}, - emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleID}}) + emqx_rule_engine_api:'/rules/:id'(get, #{bindings => #{id => RuleId}}) ), {400, #{ @@ -174,30 +178,15 @@ t_crud_rule_api(_Config) -> ok. t_list_rule_api(_Config) -> - AddIds = - lists:map( - fun(Seq0) -> - Seq = integer_to_binary(Seq0), - Params = #{ - <<"description">> => <<"A simple rule">>, - <<"enable">> => true, - <<"actions">> => [#{<<"function">> => <<"console">>}], - <<"sql">> => <<"SELECT * from \"t/1\"">>, - <<"name">> => <<"test_rule", Seq/binary>> - }, - {201, #{id := Id}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), - Id - end, - lists:seq(1, 20) - ), - + AddIds = rules_fixture(20), + ct:pal("rule ids: ~p", [AddIds]), {200, #{data := Rules, meta := #{count := Count}}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}), ?assertEqual(20, length(AddIds)), ?assertEqual(20, length(Rules)), ?assertEqual(20, Count), - [RuleID | _] = AddIds, + [RuleId | _] = AddIds, UpdateParams = #{ <<"description">> => <<"中文的描述也能搜索"/utf8>>, <<"enable">> => false, @@ -206,7 +195,7 @@ t_list_rule_api(_Config) -> <<"name">> => <<"test_rule_update1">> }, {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ - bindings => #{id => RuleID}, + bindings => #{id => RuleId}, body => UpdateParams }), QueryStr1 = #{query_string => #{<<"enable">> => false}}, @@ -229,20 +218,13 @@ t_list_rule_api(_Config) -> {200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5), ?assertEqual(maps:get(data, Result1), maps:get(data, Result5)), - QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}}, + QueryStr6 = #{query_string => #{<<"like_id">> => RuleId}}, {200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6), ?assertEqual(maps:get(data, Result1), maps:get(data, Result6)), ok. t_reset_metrics_on_disable(_Config) -> - Params = #{ - <<"description">> => <<"A simple rule">>, - <<"enable">> => true, - <<"actions">> => [#{<<"function">> => <<"console">>}], - <<"sql">> => <<"SELECT * from \"t/1\"">>, - <<"name">> => atom_to_binary(?FUNCTION_NAME) - }, - {201, #{id := RuleId}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), + #{id := RuleId} = simple_rule_fixture(), %% generate some fake metrics emqx_metrics_worker:inc(rule_metrics, RuleId, 'matched', 10), @@ -256,7 +238,7 @@ t_reset_metrics_on_disable(_Config) -> %% disable the rule; metrics should be reset {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{ bindings => #{id => RuleId}, - body => Params#{<<"enable">> := false} + body => #{<<"enable">> => false} }), {200, #{metrics := Metrics1}} = emqx_rule_engine_api:'/rules/:id/metrics'( @@ -283,9 +265,10 @@ test_rule_params(Sql, Payload) -> }. t_rule_engine(_) -> - {200, _} = emqx_rule_engine_api:'/rule_engine'(get, foo), + _ = simple_rule_fixture(), + {200, Config} = emqx_rule_engine_api:'/rule_engine'(get, #{}), + ?assert(not maps:is_key(rules, Config)), {200, #{ - %, jq_function_default_timeout := 12000 % hidden! jq_implementation_module := jq_port }} = emqx_rule_engine_api:'/rule_engine'(put, #{ @@ -299,3 +282,26 @@ t_rule_engine(_) -> body => #{<<"rules">> => #{<<"some_rule">> => SomeRule}} }), {400, _} = emqx_rule_engine_api:'/rule_engine'(put, #{body => #{<<"something">> => <<"weird">>}}). + +rules_fixture(N) -> + lists:map( + fun(Seq0) -> + Seq = integer_to_binary(Seq0), + #{id := Id} = simple_rule_fixture(Seq), + Id + end, + lists:seq(1, N) + ). + +simple_rule_fixture() -> + simple_rule_fixture(<<>>). + +simple_rule_fixture(NameSuffix) -> + create_rule(?SIMPLE_RULE(NameSuffix)). + +simple_rule_fixture(Id, NameSuffix) -> + create_rule(?SIMPLE_RULE(Id, NameSuffix)). + +create_rule(Params) -> + {201, Rule} = emqx_rule_engine_api:'/rules'(post, #{body => Params}), + Rule. From 97b94b88825a0b1f7b4eda0dc7baa58df8cdfa15 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 4 Apr 2023 08:55:43 -0300 Subject: [PATCH 051/279] feat: implement schema registry for 5.0 (avro) Part of https://emqx.atlassian.net/browse/EMQX-9251 This ports part of the Schema Registry app from 4.x to 5.0. Here, only support for Avro is added. Subsequent PRs will follow to add support for other formats. --- lib-ee/emqx_ee_schema_registry/README.md | 12 +++++--- .../src/emqx_ee_schema_registry.erl | 21 +++++++++---- .../src/emqx_ee_schema_registry_http_api.erl | 30 +++++++++---------- .../src/emqx_ee_schema_registry_schema.erl | 2 +- 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/lib-ee/emqx_ee_schema_registry/README.md b/lib-ee/emqx_ee_schema_registry/README.md index 9f477208c..c1c409c7d 100644 --- a/lib-ee/emqx_ee_schema_registry/README.md +++ b/lib-ee/emqx_ee_schema_registry/README.md @@ -52,12 +52,12 @@ WHERE | | [Registry] +------v--------------v------+ REGISTER SCHEMA | | - -------------------> | +--------+ - | | | | + INSTANCE | | +--------+ + -------------------> | | | [Management APIs] | Schema Registry ------ Schema | | | | | - -------------------> | +--------+ - LOAD PARSERS | | + | | +--------+ + | | +----------------------------+ / | \ +---/---+ +---|----+ +---\---+ @@ -67,3 +67,7 @@ WHERE +-------+ +--------+ +-------+ ``` + +- Register schema instance: adds a new instance of a schema of a + certain type. For example, when the user may have several Avro or + Protobuf schemas that they wish to use with different data flows. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl index 436777e9f..3569b246e 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -17,6 +17,7 @@ get_serde/1, add_schema/2, + get_schema/1, delete_schema/1, list_schemas/0 ]). @@ -26,6 +27,7 @@ init/1, handle_call/3, handle_cast/2, + handle_continue/2, terminate/2 ]). @@ -54,6 +56,15 @@ get_serde(SchemaName) -> {ok, serde_to_map(Serde)} end. +-spec get_schema(schema_name()) -> {ok, map()} | {error, not_found}. +get_schema(SchemaName) -> + case emqx_config:get([?CONF_KEY_ROOT, schemas, SchemaName], undefined) of + undefined -> + {error, not_found}; + Config -> + {ok, Config} + end. + -spec add_schema(schema_name(), schema()) -> ok | {error, term()}. add_schema(Name, Schema) -> RawSchema = emqx_map_lib:binary_key_map(Schema), @@ -130,9 +141,12 @@ init(_) -> process_flag(trap_exit, true), create_tables(), Schemas = emqx_conf:get([?CONF_KEY_ROOT, schemas], #{}), - async_build_serdes(Schemas), State = #{}, - {ok, State}. + {ok, State, {continue, {build_serdes, Schemas}}}. + +handle_continue({build_serdes, Schemas}, State) -> + do_build_serdes(Schemas), + {noreply, State}. handle_call(_Call, _From, State) -> {reply, {error, unknown_call}, State}. @@ -223,9 +237,6 @@ ensure_serde_absent(Name) -> ok end. -async_build_serdes(Schemas) -> - gen_server:cast(?MODULE, {build_serdes, Schemas}). - async_delete_serdes(Names) -> gen_server:cast(?MODULE, {delete_serdes, Names}). diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl index fca66a0b1..897d29e07 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl @@ -141,44 +141,44 @@ schema("/schema_registry/:name") -> ?OK(Response); '/schema_registry'(post, #{body := Params0 = #{<<"name">> := Name}}) -> Params = maps:without([<<"name">>], Params0), - case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of - undefined -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> case emqx_ee_schema_registry:add_schema(Name, Params) of ok -> - Res = emqx_config:get([?CONF_KEY_ROOT, schemas, Name]), + {ok, Res} = emqx_ee_schema_registry:get_schema(Name), {201, Res#{name => Name}}; {error, Error} -> ?BAD_REQUEST(Error) end; - _ -> + {ok, _} -> ?BAD_REQUEST('ALREADY_EXISTS', <<"Schema already exists">>) end. '/schema_registry/:name'(get, #{bindings := #{name := Name}}) -> - case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of - undefined -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> ?NOT_FOUND(<<"Schema not found">>); - Res -> - ?OK(Res#{name => Name}) + {ok, Schema} -> + ?OK(Schema#{name => Name}) end; '/schema_registry/:name'(put, #{bindings := #{name := Name}, body := Params}) -> - case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of - undefined -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> ?NOT_FOUND(<<"Schema not found">>); - _ -> + {ok, _} -> case emqx_ee_schema_registry:add_schema(Name, Params) of ok -> - Res = emqx_config:get([?CONF_KEY_ROOT, schemas, Name]), + {ok, Res} = emqx_ee_schema_registry:get_schema(Name), ?OK(Res#{name => Name}); {error, Error} -> ?BAD_REQUEST(Error) end end; '/schema_registry/:name'(delete, #{bindings := #{name := Name}}) -> - case emqx_config:get([?CONF_KEY_ROOT, schemas, Name], undefined) of - undefined -> + case emqx_ee_schema_registry:get_schema(Name) of + {error, not_found} -> ?NOT_FOUND(<<"Schema not found">>); - _ -> + {ok, _} -> case emqx_ee_schema_registry:delete_schema(Name) of ok -> ?NO_CONTENT; diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl index 01177345a..bcdc63166 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl @@ -48,7 +48,7 @@ fields(?CONF_KEY_ROOT) -> ]; fields(avro) -> [ - {type, mk(hoconsc:enum([avro]), #{required => true, desc => ?DESC("schema_type")})}, + {type, mk(avro, #{required => true, desc => ?DESC("schema_type")})}, {source, mk(emqx_schema:json_binary(), #{required => true, desc => ?DESC("schema_source")})}, {description, mk(binary(), #{default => <<>>, desc => ?DESC("schema_description")})} From 80f964bdbe4bfa75fa9151e6ebe2d37a1e9e3548 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 12 Apr 2023 10:44:37 +0800 Subject: [PATCH 052/279] chore: update apps/emqx_gateway/README.md Co-authored-by: Thales Macedo Garitezi --- apps/emqx_gateway/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index 581f0245d..5aed6421b 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -79,7 +79,7 @@ your device protocol and integrate it with EMQX. Refer to: [ExProto Gateway](../emqx_exproto/README.md) -## Understard emqx_gateway framework +## Understand emqx_gateway framework *WIP* From 8bf65c507e0fb7dd398f5a0efe11d4832b063c3f Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 11 Apr 2023 18:46:38 +0200 Subject: [PATCH 053/279] chore: update changelog --- changes/e5.0.2.en.md | 114 +++++++++++++----------------------------- changes/e5.0.2.zh.md | 115 ++++++++++++++----------------------------- 2 files changed, 71 insertions(+), 158 deletions(-) diff --git a/changes/e5.0.2.en.md b/changes/e5.0.2.en.md index 8d552ec17..83eecdfe9 100644 --- a/changes/e5.0.2.en.md +++ b/changes/e5.0.2.en.md @@ -2,29 +2,28 @@ ## Enhancements -- [#10022](https://github.com/emqx/emqx/pull/10022) Start releasing Rocky Linux 9 (compatible with Enterprise Linux 9) and MacOS 12 packages +- [#10022](https://github.com/emqx/emqx/pull/10022) Release installation packages for Rocky Linux 9 (compatible with Red Hat Enterprise Linux 9) and macOS 12 for Intel platform. -- [#10139](https://github.com/emqx/emqx/pull/10139) Add `extraVolumeMounts` to EMQX Helm Chart, it will have the ability to mount the user-own files into the EMQX instance, for example, ACL rule files as mentioned in [#9052](https://github.com/emqx/emqx/issues/9052) - Done of [#10116](https://github.com/emqx/emqx/issues/10116) +- [#10139](https://github.com/emqx/emqx/pull/10139) Add `extraVolumeMounts` to EMQX Helm Chart, you can mount user's own files to EMQX instance, such as ACL rule files mentioned in [#9052](https://github.com/emqx/emqx/issues/9052). -- [#9893](https://github.com/emqx/emqx/pull/9893) When connecting with the flag `clean_start=false`, EMQX will filter out messages that published by banned clients. - Previously, the messages sent by banned clients may still be delivered to subscribers in this scenario. +- [#9893](https://github.com/emqx/emqx/pull/9893) When connecting with the flag `clean_start=false`, EMQX will filter out messages that published by clients banned by the blacklist feature in the session. +Previously, messages sent by clients banned by the blacklist feature could still be delivered to subscribers in this case. -- [#9986](https://github.com/emqx/emqx/pull/9986) Add MQTT ingress to helm charts and update helm charts documentation +- [#9986](https://github.com/emqx/emqx/pull/9986) Add MQTT ingress to helm charts and remove obsolete mgmt references. -- [#10083](https://github.com/emqx/emqx/pull/10083) Add `DynamoDB` support for Data-Brdige. +- [#9564](https://github.com/emqx/emqx/pull/9564) Implement Kafka Consumer Bridge, which supports consuming messages from Kafka and publishing them to MQTT topics. -- [#9564](https://github.com/emqx/emqx/pull/9564) Implement Kafka Consumer bridge. - Now it's possible to consume messages from Kafka and publish them to MQTT topics. - -- [#9881](https://github.com/emqx/emqx/pull/9881) Enhance the error logs related to InfluxDB connectivity health checks. - Previously, if InfluxDB failed to pass the health checks using the specified parameters, the only message provided was "timed out waiting for it to become healthy". - With the updated implementation, the error message will be displayed in both the logs and the dashboard, enabling easier identification and resolution of the issue. +- [#9881](https://github.com/emqx/emqx/pull/9881) Improve error logging related to health checks for InfluxDB connections. - [#10123](https://github.com/emqx/emqx/pull/10123) Improve the performance of `/bridges` API. - Earlier, when the number of nodes in the cluster was large or the node was busy, the API may have a request timeout. +Earlier, when the number of nodes in the cluster was large or the node was busy, the API may had a request timeout. + +- [#9998](https://github.com/emqx/emqx/pull/9998) Obfuscate request body in error log when using HTTP service for client authentication for security reasons. + +- [#10026](https://github.com/emqx/emqx/pull/10026) Metrics are now only exposed via the `/bridges/:id/metrics` endpoint, and no longer returned in other API operations. + +- [#10052](https://github.com/emqx/emqx/pull/10052) Improve startup failure logs in daemon mode. -- [#9998](https://github.com/emqx/emqx/pull/9998) Redact the HTTP request body in the authentication error logs for security reasons. ## Bug Fixes @@ -32,104 +31,57 @@ - [#10014](https://github.com/emqx/emqx/pull/10014) Ensure Monitor API `/monitor(_current)/nodes/:node` returns `404` instead of `400` if node does not exist. -- [#10026](https://github.com/emqx/emqx/pull/10026) Metrics are now only exposed via the /bridges/:id/metrics endpoint. Metrics are no longer returned in other API operations such as getting the list of all bridges, or in the response when a bridge has been created. - -- [#10027](https://github.com/emqx/emqx/pull/10027) Allow setting node name from `EMQX_NODE__NAME` when running in docker. - Prior to this fix, only `EMQX_NODE_NAME` is allowed. +- [#10027](https://github.com/emqx/emqx/pull/10027) Allow setting node name via environment variable `EMQX_NODE__NAME` in Docker. - [#10050](https://github.com/emqx/emqx/pull/10050) Ensure Bridge API returns `404` status code consistently for resources that don't exist. -- [#10052](https://github.com/emqx/emqx/pull/10052) Improve daemon mode startup failure logs. - - Before this change, it was difficult for users to understand the reason for EMQX 'start' command failed to boot the node. - The only information they received was that the node did not start within the expected time frame, - and they were instructed to boot the node with 'console' command in the hope of obtaining some logs. - However, the node might actually be running, which could cause 'console' mode to fail for a different reason. - - With this new change, when daemon mode fails to boot, a diagnosis is issued. Here are the possible scenarios: - - * If the node cannot be found from `ps -ef`, the user is instructed to find information in log files `erlang.log.*`. - * If the node is found to be running but not responding to pings, the user is advised to check if the host name is resolvable and reachable. - * If the node is responding to pings, but the EMQX app is not running, it is likely a bug. In this case, the user is advised to report a Github issue. - - [#10055](https://github.com/emqx/emqx/pull/10055) The configuration parameter `mqtt.max_awaiting_rel` was not functional and has now been corrected. - [#10056](https://github.com/emqx/emqx/pull/10056) Fix `/bridges` API status code. - - Return `400` instead of `403` in case of removing a data bridge that is dependent on an active rule. - - Return `400` instead of `403` in case of calling operations (start|stop|restart) when Data-Bridging is not enabled. + Return `400` instead of `403` in case of removing a data bridge that is dependent on an active rule. + Return `400` instead of `403` in case of calling operations (start|stop|restart) when Data-Bridging is not enabled. - [#10066](https://github.com/emqx/emqx/pull/10066) Improve error messages for `/briges_probe` and `[/node/:node]/bridges/:id/:operation` API calls to make them more readable. And set HTTP status code to `400` instead of `500`. -- [#10074](https://github.com/emqx/emqx/pull/10074) Check if type in `PUT /authorization/sources/:type` matches `type` given in body of request. +- [#10074](https://github.com/emqx/emqx/pull/10074) Check if type in `PUT /authorization/sources/:type` matches `type` given in the request body. -- [#10079](https://github.com/emqx/emqx/pull/10079) Fix description of `shared_subscription_strategy`. +- [#10079](https://github.com/emqx/emqx/pull/10079) Fix wrong description about `shared_subscription_strategy`. -- [#10085](https://github.com/emqx/emqx/pull/10085) Consistently return `404` for all requests on non existent source in `/authorization/sources/:source[/*]`. +- [#10085](https://github.com/emqx/emqx/pull/10085) Consistently return `404` for all requests on non-existent source in `/authorization/sources/:source[/*]`. -- [#10098](https://github.com/emqx/emqx/pull/10098) A crash with an error in the log file that happened when the MongoDB authorization module queried the database has been fixed. +- [#10098](https://github.com/emqx/emqx/pull/10098) Fix an issue where the MongoDB connector crashed when MongoDB authorization was configured. -- [#10100](https://github.com/emqx/emqx/pull/10100) Fix channel crash for slow clients with enhanced authentication. - Previously, when the client was using enhanced authentication, but the Auth message was sent slowly or the Auth message was lost, the client process would crash. +- [#10100](https://github.com/emqx/emqx/pull/10100) Fix channel crash for slow clients with enhanced authentication. +Previously, when the client was using enhanced authentication, but the Auth message was sent slowly or the Auth message was lost, the client process would crash. -- [#10107](https://github.com/emqx/emqx/pull/10107) For operations on Bridges API if `bridge-id` is unknown we now return `404` - instead of `400`. Also a bug was fixed that caused a crash if that was a node - operation. Additionally we now also check if the given bridge is enabled when - doing the cluster operation `start` . Affected endpoints: - * [cluster] `/bridges/:id/:operation`, - * [node] `/nodes/:node/bridges/:id/:operation`, where `operation` is one of - `[start|stop|restart]`. - Moreover, for a node operation, EMQX checks if node name is in our cluster and - return `404` instead of `501`. +- [#10107](https://github.com/emqx/emqx/pull/10107) For operations on Bridges API if `bridge-id` is unknown we now return `404` instead of `400`. - [#10117](https://github.com/emqx/emqx/pull/10117) Fix an error occurring when a joining node doesn't have plugins that are installed on other nodes in the cluster. - After this fix, the joining node will copy all the necessary plugins from other nodes. +After this fix, the joining node will copy all the necessary plugins from other nodes. - [#10118](https://github.com/emqx/emqx/pull/10118) Fix problems related to manual joining of EMQX replicant nodes to the cluster. - Previously, after manually executing joining and then leaving the cluster, the `replicant` node can only run normally after restarting the node after joining the cluster again. - - [Mria PR](https://github.com/emqx/mria/pull/128) - [#10119](https://github.com/emqx/emqx/pull/10119) Fix crash when `statsd.server` is set to an empty string. - [#10124](https://github.com/emqx/emqx/pull/10124) The default heartbeat period for MongoDB has been increased to reduce the risk of too excessive logging to the MongoDB log file. - [#10130](https://github.com/emqx/emqx/pull/10130) Fix garbled config display in dashboard when the value is originally from environment variables. - For example, `env EMQX_STATSD__SERVER='127.0.0.1:8124' . /bin/emqx start` results in unreadable string (not '127.0.0.1:8124') displayed in Dashboard's Statsd settings page. - Related PR: [HOCON#234](https://github.com/emqx/hocon/pull/234). -- [#10132](https://github.com/emqx/emqx/pull/10132) Fix some error logs generated by `systemctl stop emqx` command. - Prior to the fix, the command was not stopping jq and os_mon applications properly. +- [#10132](https://github.com/emqx/emqx/pull/10132) Fix some error logs generated by `systemctl stop emqx` command. +Prior to the fix, the command was not stopping `jq` and `os_mon` applications properly. -- [#10144](https://github.com/emqx/emqx/pull/10144) Add `-setcookie` emulator flag when invoking `emqx ctl` to prevent problems with emqx cli when home directory is read only. Fixes [#10142](https://github.com/emqx/emqx/issues/10142). +- [#10144](https://github.com/emqx/emqx/pull/10144) Fix an issue where emqx cli failed to set the Erlang cookie when the emqx directory was read-only. -- [#10154](https://github.com/emqx/emqx/pull/10154) Change the default `resume_interval` for bridges and connectors to be - the minimum of `health_check_interval` and `request_timeout / 3`. - Also exposes it as a hidden configuration to allow fine tuning. +- [#10154](https://github.com/emqx/emqx/pull/10154) Change the default `resume_interval` for bridges and connectors to be the minimum of `health_check_interval` and `request_timeout / 3` to resolve issue of request timeout. - Before this change, the default values for `resume_interval` meant - that, if a buffer ever got blocked due to resource errors or high - message volumes, then, by the time the buffer would try to resume its - normal operations, almost all requests would have timed out. - -- [#10157](https://github.com/emqx/emqx/pull/10157) Fixed default rate limit configuration not being applied correctly when creating a new listener. +- [#10157](https://github.com/emqx/emqx/pull/10157) Fix default rate limit configuration not being applied correctly when creating a new listener. - [#10237](https://github.com/emqx/emqx/pull/10237) Ensure we return `404` status code for unknown node names in `/nodes/:node[/metrics|/stats]` API. -- [#10251](https://github.com/emqx/emqx/pull/10251) Consider bridges referenced in `FROM` rule clauses as dependencies. - - Before this fix, when one tried to delete an ingress rule referenced in an action like `select * from "$bridges/mqtt:ingress"`, the UI would not trigger a warning about dependent rule actions. +- [#10251](https://github.com/emqx/emqx/pull/10251) Fix an issue where rule dependencies were not prompted when deleting an ingress-type bridge in use. - [#10313](https://github.com/emqx/emqx/pull/10313) Ensure that when the core or replicant node starting, the `cluster-override.conf` file is only copied from the core node. - Previously, when sorting nodes by startup time, the core node may have copied this file from the replicant node. -- [#10314](https://github.com/emqx/emqx/pull/10314) Fix /monitor_current API so that it only looks at the current node. - Fix /stats API to not crash when one or more nodes in the cluster are down. +- [#10327](https://github.com/emqx/emqx/pull/10327) Don't increase “actions.failed.unknown” rule metrics counter upon receiving unrecoverable data bridge errors. -- [#10327](https://github.com/emqx/emqx/pull/10327) Don't increment 'actions.failed.unknown' rule metrics counter upon receiving unrecoverable bridge errors. - This counter is displayed on the dashboard's rule overview tab ('Action statistics' - 'Unknown'). - The fix is only applicable for synchronous bridges, as all rule actions for asynchronous bridges - are counted as successful (they increment 'actions.success' which is displayed as 'Action statistics' - 'Success'). - -- [#10095](https://github.com/emqx/emqx/pull/10095) Stop MySQL client from bombarding server repeatedly with unnecessary `PREPARE` queries on every batch, trashing the server and exhausting its internal limits. This was happening when the MySQL bridge was in the batch mode. - - Ensure safer and more careful escaping of strings and binaries in batch insert queries when the MySQL bridge is in the batch mode. +- [#10095](https://github.com/emqx/emqx/pull/10095) Fix an issue where when the MySQL connector was in batch mode, clients would keep querying the server with unnecessary `PREPARE` statements on each batch, possibly causing server resource exhaustion. diff --git a/changes/e5.0.2.zh.md b/changes/e5.0.2.zh.md index a73372570..f6dcc3e72 100644 --- a/changes/e5.0.2.zh.md +++ b/changes/e5.0.2.zh.md @@ -1,118 +1,79 @@ # e5.0.2 -## 增强 +## 优化 -- [#10022](https://github.com/emqx/emqx/pull/10022) 开始发布Rocky Linux 9（与Enterprise Linux 9兼容）和 MacOS 12 软件包。 +- [#10022](https://github.com/emqx/emqx/pull/10022) 发布 Rocky Linux 9 (兼容Red Hat Enterprise Linux 9) 以及 macOS 12 Intel 平台的安装包。 -- [#10139](https://github.com/emqx/emqx/pull/10139) 将 `extraVolumeMounts` 添加到 EMQX Helm Chart 中，它将能够挂载用户自己的文件到 EMQX 实例中，例如在 [#9052](https://github.com/emqx/emqx/issues/9052) 中提到的 ACL 规则文件。 - 修复了 issue [#10116](https://github.com/emqx/emqx/issues/10116) +- [#10139](https://github.com/emqx/emqx/pull/10139) 在 EMQX Helm Chart 中添加 `extraVolumeMounts`，可以挂载用户自己的文件到 EMQX 实例中，例如在 [#9052](https://github.com/emqx/emqx/issues/9052) 中提到的 ACL 规则文件。 -- [#9893](https://github.com/emqx/emqx/pull/9893) 当使用 `clean_start=false` 标志连接时，EMQX 将会从消息队列中过滤出被封禁客户端发出的消息，使它们不能被下发给订阅者。 - 此前被封禁客户端发出的消息仍可能在这一场景下被下发给订阅者。 +- [#9893](https://github.com/emqx/emqx/pull/9893) 当使用 `clean_start=false` 标志连接时，EMQX 将过滤掉会话中被被黑名单功能禁止的客户端发布的消息。以前，在这种情况下，被黑名单功能禁止的客户端发送的消息仍可能被传递给订阅者。 -- [#9986](https://github.com/emqx/emqx/pull/9986) 在 helm chart 中新增了 MQTT 桥接 ingress 的配置参数；并删除了旧版本遗留的 `mgmt` 配置。 +- [#9986](https://github.com/emqx/emqx/pull/9986) 在 helm charts 中增加 MQTT ingress 并删除过时的 `mgmt` 引用。 -- [#10083](https://github.com/emqx/emqx/pull/10083) 为数据桥接增加 `DynamoDB` 支持。 +- [#9564](https://github.com/emqx/emqx/pull/9564) 数据桥接新增 Kafka Consumer，支持从 Kafka 消费消息并将它们发布到 MQTT 主题。 -- [#9564](https://github.com/emqx/emqx/pull/9564) 实现了 Kafka 消费者桥接。 - 现在可以从 Kafka 消费消息并将其发布到 MQTT 主题。 +- [#9881](https://github.com/emqx/emqx/pull/9881) 改进了与 InfluxDB 连接的健康检查相关的错误日志。 -- [#9881](https://github.com/emqx/emqx/pull/9881) 增强了与 InfluxDB 连接健康检查相关的错误日志。 - 在此更改之前，如果使用配置的参数 InfluxDB 未能通过健康检查，用户仅能获得一个“超时”的信息。 - 现在，详细的错误消息将显示在日志和控制台，从而让用户更容易地识别和解决问题。 +- [#10123](https://github.com/emqx/emqx/pull/10123) 改进了 `/bridges` API 的性能。避免了当集群节点数量较多时可能出现的请求响应超时。 -- [#10123](https://github.com/emqx/emqx/pull/10123) 改进 `/bridges` API 的性能。 - 此前，当集群中节点数目较多或节点忙时，该 API 可能出现请求超时的情况。 +- [#9998](https://github.com/emqx/emqx/pull/9998) 出于安全原因，在使用 HTTP 服务进行客户端认证时，对错误日志中的请求体进行脱敏处理。 -- [#9998](https://github.com/emqx/emqx/pull/9998) 出于安全原因，在身份验证错误日志中模糊 HTTP 请求正文。 +- [#10026](https://github.com/emqx/emqx/pull/10026) 仅在 `/bridges/:id/metrics` API 中返回指标数据。 + +- [#10052](https://github.com/emqx/emqx/pull/10052) 改进守护进程模式下启动失败后的日志。 ## 修复 -- [#10013](https://github.com/emqx/emqx/pull/10013) 修复 API `/gateways/:name/clients` 返回值的类型结构错误。 +- [#10013](https://github.com/emqx/emqx/pull/10013) 修复 `/gateways/:name/clients` API 在错误情况下的返回类型结构。 -- [#10014](https://github.com/emqx/emqx/pull/10014) 如果 API 查询的节点不存在，将会返回 `404` 而不再是 `400`。 +- [#10014](https://github.com/emqx/emqx/pull/10014) 当节点不存在时，`/monitor(_current)/nodes/:node` API 返回 `404` 错误代码而不是 `400` 。 -- [#10026](https://github.com/emqx/emqx/pull/10026) 现在只有显式调用 `/bridges/:id/metrics` 接口时才可以获得指标数据，而其他 API 接口将不再返回相关数据。 +- [#10027](https://github.com/emqx/emqx/pull/10027) 允许在 Docker 中通过 `EMQX_NODE__NAME` 设置节点名称。 -- [#10027](https://github.com/emqx/emqx/pull/10027) 在 docker 中启动时，允许使用 `EMQX_NODE__NAME` 环境变量来配置节点名。 - 在此修复前，只能使 `EMQX_NODE_NAME`。 +- [#10050](https://github.com/emqx/emqx/pull/10050) 当调用 Bridge API 时若资源不能存在则返回 `404` 状态码。 -- [#10050](https://github.com/emqx/emqx/pull/10050) 确保 Bridge API 对不存在的资源一致返回 `404` 状态代码。 +- [#10055](https://github.com/emqx/emqx/pull/10055) 修复配置项 `mqtt.max_awaiting_rel` 设置无效的问题。 -- [#10052](https://github.com/emqx/emqx/pull/10052) 优化 EMQX daemon 模式启动启动失败的日志。 +- [#10056](https://github.com/emqx/emqx/pull/10056) 修复 `/bridges` API 状态码返回错误。当被删除的 Bridge 存在依赖，或 Bridge 未启用时进行启用、停止、重启等操作时返回 `400` 状态码。 - 在进行此更改之前，当 EMQX 用 `start` 命令启动失败时，用户很难理解出错的原因。 - 所知道的仅仅是节点未能在预期时间内启动，然后被指示以 `console` 式引导节点以获取一些日志。 - 然而，节点实际上可能正在运行，这可能会导致 `console` 模式因不同的原因而失败。 +- [#10066](https://github.com/emqx/emqx/pull/10066) 优化 `/briges_probe` 和 `[/node/:node]/bridges/:id/:operation` API 调用的错误消息，使其更易理解。并修正错误状态码为`400`。 - 此次修复后，启动脚本会发出诊断： +- [#10074](https://github.com/emqx/emqx/pull/10074) 增加 `PUT /authorization/sources/:type` 请求参数 `type` 的值与请求体中的实际类型的一致性检查。 - * 如果无法从 `ps -ef` 中找到节点，则指示用户在 `erlang.log.*` 中查找信息。 - * 如果发现节点正在运行但不响应 ping，则建议用户检查节点主机名是否有效并可达。 - * 如果节点响应 ping 但 EMQX 应用程序未运行，则很可能是一个错误。在这种情况下，建议用户报告一个Github issue。 +- [#10079](https://github.com/emqx/emqx/pull/10079) 修复文档中关于 `shared_subscription_strategy` 的描述错误。 -- [#10055](https://github.com/emqx/emqx/pull/10055) 修复配置项 `mqtt.max_awaiting_rel` 更新不生效问题。 +- [#10085](https://github.com/emqx/emqx/pull/10085) 对于 `/authorization/sources/:source[/*]` API 中不存在的资源的所有请求，始终返回 `404`。 -- [#10056](https://github.com/emqx/emqx/pull/10056) 修复 `/bridges` API 的 HTTP 状态码。 - - 当删除被活动中的规则依赖的数据桥接时，将返回 `400` 而不是 `403` 。 - - 当数据桥接未启用时，调用操作（启动|停止|重启）将返回 `400` 而不是 `403`。 +- [#10098](https://github.com/emqx/emqx/pull/10098) 修复了当配置 MongoDB 授权时 MongoDB 连接器崩溃的问题。 -- [#10066](https://github.com/emqx/emqx/pull/10066) 改进 `/briges_probe` 和 `[/node/:node]/bridges/:id/:operation` API 调用的错误信息，使之更加易读。并将 HTTP 状态代码设置为 `400` 而不是 `500`。 +- [#10100](https://github.com/emqx/emqx/pull/10100) 修复了当客户端使用增强认证时，认证消息发送缓慢或者消息丢失时客户端进程崩溃的问题。 -- [#10074](https://github.com/emqx/emqx/pull/10074) 检查 `PUT /authorization/sources/:type` 中的类型是否与请求正文中的 `type` 相符。 +- [#10107](https://github.com/emqx/emqx/pull/10107) 当调用 `bridges API` 时如果 `bridge-id` 不存在则返回 `404` 状态码。 -- [#10079](https://github.com/emqx/emqx/pull/10079) 修正对 `shared_subscription_strategy` 的描述。 +- [#10117](https://github.com/emqx/emqx/pull/10117) 修复当加入的节点没有安装在集群其他节点上的插件时发生的错误。修复后，加入的节点将从其他节点复制所有必要的插件。 +- [#10118](https://github.com/emqx/emqx/pull/10118) 修复手动添加 EMQX 副本类型节点到集群的相关问题。 -- [#10085](https://github.com/emqx/emqx/pull/10085) 如果向 `/authorization/sources/:source[/*]` 请求的 `source` 不存在，将一致地返回 `404`。 +- [#10119](https://github.com/emqx/emqx/pull/10119) 修复了当 `statsd.server` 设置为空字符串时会崩溃的问题。 -- [#10098](https://github.com/emqx/emqx/pull/10098) 当 MongoDB 授权模块查询数据库时，在日志文件中发生的崩溃与错误已经被修复。 +- [#10124](https://github.com/emqx/emqx/pull/10124) 调整 MongoDB 的默认心跳周期，以降低日志文件记录过多的风险。 -- [#10100](https://github.com/emqx/emqx/pull/10100) 修复响应较慢的客户端在使用增强认证时可能出现崩溃的问题。 - 此前，当客户端使用增强认证功能，但发送 Auth 报文较慢或 Auth 报文丢失时会导致客户端进程崩溃。 +- [#10130](https://github.com/emqx/emqx/pull/10130) 修复了通过环境变量设置的值，在 Dashboard 中显示乱码的问题。 -- [#10107](https://github.com/emqx/emqx/pull/10107) 现在对桥接的 API 进行调用时，如果 `bridge-id` 不存在，将会返回 `404`，而不再是`400`。 - 然后，还修复了这种情况下，在节点级别上进行 API 调用时，可能导致崩溃的问题。 - 另外，在启动某个桥接时，会先检查指定桥接是否已启用。 - 受影响的接口有： - * [cluster] `/bridges/:id/:operation`, - * [node] `/nodes/:node/bridges/:id/:operation`, - 其中 `operation` 是 `[start|stop|restart]` 之一。 - 此外，对于节点操作，EMQX 将检查节点是否存在于集群中，如果不在，则会返回`404`，而不再是`501`。 +- [#10132](https://github.com/emqx/emqx/pull/10132) 修复了 `systemctl stop emqx` 命令无法正确停止 `jq` 和 `os_mon` 应用的问题。 -- [#10117](https://github.com/emqx/emqx/pull/10117) 修复节点加入集群时，由于缺少集其它节点已安装的插件所导致的错误。 - 在此修复后，加入集群的节点将从其它节点复制所有必须的插件。 +- [#10144](https://github.com/emqx/emqx/pull/10144) 修复了当 emqx 目录只读时， emqx cli 设置 Erlang cookie 失败的问题。 -- [#10118](https://github.com/emqx/emqx/pull/10118) 修复 `replicant` 节点因为手动加入 EMQX 集群导致的相关问题。 - 此前，手动执行 `加入集群-离开集群` 后，`replicant` 节点再次加入集群后只有重启节点才能正常运行。 +- [#10154](https://github.com/emqx/emqx/pull/10154) 将数据桥接和连接器的 `resume_interval` 参数值设为 `health_check_interval` 和 `request_timeout / 3` 中的较小值，以解决请求超时的问题。 - [Mria PR](https://github.com/emqx/mria/pull/128) +- [#10157](https://github.com/emqx/emqx/pull/10157) 修复在创建新的监听器时默认速率限制不生效的问题。 -- [#10119](https://github.com/emqx/emqx/pull/10119) 修复 `statsd.server` 配置为空字符串时启动崩溃的问题。 - -- [#10124](https://github.com/emqx/emqx/pull/10124) 增加了 MongoDB 的默认心跳周期，以减少 MongoDB 日志文件记录过多的风险。 - -- [#10130](https://github.com/emqx/emqx/pull/10130) 修复通过环境变量配置启动的 EMQX 节点无法通过HTTP API获取到正确的配置信息。 - 比如：`EMQX_STATSD__SERVER='127.0.0.1:8124' ./bin/emqx start` 后通过 Dashboard看到的 Statsd 配置信息是乱码。 - 相关 PR: [HOCON:234](https://github.com/emqx/hocon/pull/234). - -- [#10132](https://github.com/emqx/emqx/pull/10132) 修复 `systemctl stop emqx` 命令没有正常停止 jq、os_mon 组件所产生一些错误日志。 - -- [#10144](https://github.com/emqx/emqx/pull/10144) 为 emqx 可执行文件加入 `-setcookie` 标志，以避免由于 home 目录只读，导致 emqx cli 所提供的 `emqx ctl` 等命令在执行时出现的一些问题。修复 [#10142](https://github.com/emqx/emqx/issues/10142)。 - -- [#10154](https://github.com/emqx/emqx/pull/10154) 将数据桥接和连接器的 resume_interval 参数值设为 health_check_interval 和 request_timeout / 3 中的较小值，以解决请求超时的问题。 - -- [#10157](https://github.com/emqx/emqx/pull/10157) 修复在创建新的监听器时，没有正确应用速率限制默认配置的问题。 - -- [#10237](https://github.com/emqx/emqx/pull/10237) 当调用 `/nodes/:node[/metrics|/stats]` API ，若节点不存在则返回 `404` 状态码。 +- [#10237](https://github.com/emqx/emqx/pull/10237) 当调用 `/nodes/:node[/metrics|/stats]` API，若节点不存在则返回 `404` 状态码。 - [#10251](https://github.com/emqx/emqx/pull/10251) 修复了当删除一个使用中的 ingress 类型的桥接时，未提示存在规则依赖的问题。 -- [#10313](https://github.com/emqx/emqx/pull/10313) 确保当 core 或 replicant 节点启动时，仅从 core 节点复制 `cluster-override.conf` 文件。 - 此前按照节点启动时间排序时，core 节点可能从 replicant 节点复制该文件。 +- [#10313](https://github.com/emqx/emqx/pull/10313) 确保在核心节点或副本节点启动时，仅从核心节点复制 `cluster-override.conf` 文件。 -- [#10314](https://github.com/emqx/emqx/pull/10314) 修复 `/monitor_current` API ，使其仅查看当前 节点。修复了 `/stats` API，以防止当集群中的一个或多个节点关闭时出现崩溃。 +- [#10327](https://github.com/emqx/emqx/pull/10327) 数据桥接出现不可恢复的错误不再计入到 `actions.failed.unknown` 指标中。 -- [#10327](https://github.com/emqx/emqx/pull/10327) 在收到不可恢复的错误时，不要增加 'actions.failed.unknown' 规则指标计数。 - -- [#10095](https://github.com/emqx/emqx/pull/10095) 优化 MySQL 桥接在批量模式下能更高效的使用预处理语句 ，减少了对 MySQL 服务器的写入压力， 并确保对 SQL 语句进行更安全和谨慎的转义。 +- [#10095](https://github.com/emqx/emqx/pull/10095) 修复当 MySQL 连接器处于批处理模式时，会发生客户端在每个批次上不断使用不必要的 `PREPARE` 语句查询服务器，可能会导致服务器资源耗尽的问题。 From 9560fdc5a2a48f40b5649636e90cf9372a244de0 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 12 Apr 2023 11:34:04 +0800 Subject: [PATCH 054/279] chore: typo fixes --- .../test/emqx_ee_bridge_cassa_SUITE.erl | 32 +++++++++++-------- .../src/emqx_ee_connector_cassa.erl | 6 ++-- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index 826df9be8..208b68de6 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -591,20 +591,24 @@ t_missing_data(Config) -> ), %% emqx_ee_connector_cassa will send missed data as a `null` atom %% to ecql driver - send_message(Config, #{}), - ?block_until( - #{ - ?snk_kind := buffer_worker_flush_ack, - result := {async_return, ok} - }, - 2_000 - ), - ?block_until( - #{ - ?snk_kind := handle_async_reply_enter, - result := {error, {8704, _}} - }, - 2_000 + ?check_trace( + begin + ?wait_async_action( + send_message(Config, #{}), + #{?snk_kind := handle_async_reply, result := {error, {8704, _}}}, + 10_000 + ), + ok + end, + fun(Trace0) -> + %% 1. ecql driver will return `ok` first in async query + Trace = ?of_kind(cassandra_connector_query_return, Trace0), + ?assertMatch([#{result := ok}], Trace), + %% 2. then it will return an error in callback function + Trace1 = ?of_kind(handle_async_reply, Trace0), + ?assertMatch([#{result := {error, {8704, _}}}], Trace1), + ok + end ), ok. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index 4b3804c01..1e1882a1f 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -175,7 +175,7 @@ on_query( Request, State ) -> - do_signle_query(InstId, Request, sync, State). + do_single_query(InstId, Request, sync, State). -spec on_query_async( emqx_resource:resource_id(), @@ -189,9 +189,9 @@ on_query_async( Callback, State ) -> - do_signle_query(InstId, Request, {async, Callback}, State). + do_single_query(InstId, Request, {async, Callback}, State). -do_signle_query( +do_single_query( InstId, Request, Async, From f161399bccac08babef77b5741ea63abc8b3dd2a Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 6 Apr 2023 16:48:41 +0800 Subject: [PATCH 055/279] feat: deprecated trace config --- apps/emqx/src/emqx_schema.erl | 4 +- apps/emqx/src/emqx_trace/emqx_trace.erl | 38 ++++++++++++-- apps/emqx/src/emqx_trace/emqx_trace.hrl | 2 + .../src/emqx_trace/emqx_trace_handler.erl | 4 +- apps/emqx/test/emqx_trace_SUITE.erl | 50 ++++++++++++++++++- .../src/emqx_mgmt_api_trace.erl | 21 +++++++- .../test/emqx_mgmt_api_trace_SUITE.erl | 7 +++ 7 files changed, 117 insertions(+), 9 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 20018b2d5..b0b316321 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -229,7 +229,7 @@ roots(low) -> {"trace", sc( ref("trace"), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )}, {"crl_cache", sc( @@ -1853,6 +1853,8 @@ fields("trace") -> {"payload_encode", sc(hoconsc:enum([hex, text, hidden]), #{ default => text, + deprecated => {since, "5.0.22"}, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(fields_trace_payload_encode) })} ]. diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index ea6736038..a185d1910 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -59,7 +59,8 @@ -ifdef(TEST). -export([ log_file/2, - find_closest_time/2 + find_closest_time/2, + migrate_trace/0 ]). -endif. @@ -230,6 +231,7 @@ init([]) -> {attributes, record_info(fields, ?TRACE)} ]), ok = mria:wait_for_tables([?TRACE]), + migrate_trace(), {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), ok = filelib:ensure_dir(filename:join([trace_dir(), dummy])), ok = filelib:ensure_dir(filename:join([zip_dir(), dummy])), @@ -358,9 +360,10 @@ start_trace(Trace) -> name = Name, type = Type, filter = Filter, - start_at = Start + start_at = Start, + payload_encode = PayloadEncode } = Trace, - Who = #{name => Name, type => Type, filter => Filter}, + Who = #{name => Name, type => Type, filter => Filter, payload_encode => PayloadEncode}, emqx_trace_handler:install(Who, debug, log_file(Name, Start)). stop_trace(Finished, Started) -> @@ -490,6 +493,8 @@ to_trace(#{type := ip_address, ip_address := Filter} = Trace, Rec) -> end; to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])}; +to_trace(#{payload_encode := PayloadEncode} = Trace, Rec) -> + to_trace(maps:remove(payload_encode, Trace), Rec#?TRACE{payload_encode = PayloadEncode}); to_trace(#{start_at := StartAt} = Trace, Rec) -> {ok, Sec} = to_system_second(StartAt), to_trace(maps:remove(start_at, Trace), Rec#?TRACE{start_at = Sec}); @@ -573,3 +578,30 @@ filter_cli_handler(Names) -> now_second() -> os:system_time(second). + +migrate_trace() -> + Fields = record_info(fields, ?TRACE), + case mnesia:table_info(emqx_trace, attributes) =:= Fields of + true -> + ok; + false -> + TransFun = fun(Trace) -> + case Trace of + {?TRACE, Name, Type, Filter, Enable, StartAt, EndAt} -> + #?TRACE{ + name = Name, + type = Type, + filter = Filter, + enable = Enable, + start_at = StartAt, + end_at = EndAt, + payload_encode = text, + extra = #{} + }; + #?TRACE{} -> + Trace + end + end, + {atomic, ok} = mnesia:transform_table(?TRACE, TransFun, Fields, ?TRACE), + ok + end. diff --git a/apps/emqx/src/emqx_trace/emqx_trace.hrl b/apps/emqx/src/emqx_trace/emqx_trace.hrl index 096e786dd..62028bcc0 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.hrl +++ b/apps/emqx/src/emqx_trace/emqx_trace.hrl @@ -24,6 +24,8 @@ filter :: emqx_types:topic() | emqx_types:clientid() | emqx_trace:ip_address() | undefined | '_', enable = true :: boolean() | '_', + payload_encode = text :: hex | text | hidden | '_', + extra = #{} :: map() | '_', start_at :: integer() | undefined | '_', end_at :: integer() | undefined | '_' }). diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index 9c2d2358e..7f586f7a4 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -160,14 +160,14 @@ filters(#{type := topic, filter := Filter, name := Name}) -> filters(#{type := ip_address, filter := Filter, name := Name}) -> [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}]. -formatter(#{type := _Type}) -> +formatter(#{type := _Type, payload_encode := PayloadEncode}) -> {emqx_trace_formatter, #{ %% template is for ?SLOG message not ?TRACE. template => [time, " [", level, "] ", msg, "\n"], single_line => true, max_size => unlimited, depth => unlimited, - payload_encode => payload_encode() + payload_encode => PayloadEncode }}. filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) -> diff --git a/apps/emqx/test/emqx_trace_SUITE.erl b/apps/emqx/test/emqx_trace_SUITE.erl index c66808132..3594f0651 100644 --- a/apps/emqx/test/emqx_trace_SUITE.erl +++ b/apps/emqx/test/emqx_trace_SUITE.erl @@ -24,7 +24,9 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --record(emqx_trace, {name, type, filter, enable = true, start_at, end_at}). +-record(emqx_trace, { + name, type, filter, enable = true, payload_encode = text, extra = #{}, start_at, end_at +}). %%-------------------------------------------------------------------- %% Setups @@ -97,7 +99,9 @@ t_base_create_delete(_Config) -> type => clientid, name => <<"name1">>, start_at => Now, - end_at => Now + 30 * 60 + end_at => Now + 30 * 60, + payload_encode => text, + extra => #{} } ], ?assertEqual(ExpectFormat, emqx_trace:format([TraceRec])), @@ -385,6 +389,48 @@ t_find_closed_time(_Config) -> ?assertEqual(1000, emqx_trace:find_closest_time(Traces, Now)), ok. +t_migrate_trace(_Config) -> + build_new_trace_data(), + build_old_trace_data(), + reload(), + Traces = emqx_trace:format(emqx_trace:list()), + ?assertEqual(2, erlang:length(Traces)), + lists:foreach( + fun(#{name := Name, enable := Enable}) -> + ?assertEqual(true, Enable, Name) + end, + Traces + ), + LoggerIds = logger:get_handler_ids(), + lists:foreach( + fun(Id) -> + ?assertEqual(true, lists:member(Id, LoggerIds), LoggerIds) + end, + [ + trace_topic_test_topic_migrate_new, + trace_topic_test_topic_migrate_old + ] + ), + ok. + +build_new_trace_data() -> + Now = erlang:system_time(second), + {ok, _} = emqx_trace:create([ + {<<"name">>, <<"test_topic_migrate_new">>}, + {<<"type">>, topic}, + {<<"topic">>, <<"/test/migrate/new">>}, + {<<"start_at">>, Now - 10} + ]). + +build_old_trace_data() -> + Now = erlang:system_time(second), + OldAttrs = [name, type, filter, enable, start_at, end_at], + {atomic, ok} = mnesia:transform_table(emqx_trace, ignore, OldAttrs, emqx_trace), + OldTrace = + {emqx_trace, <<"test_topic_migrate_old">>, topic, <<"topic">>, true, Now - 10, Now + 100}, + ok = mnesia:dirty_write(OldTrace), + ok. + reload() -> catch ok = gen_server:stop(emqx_trace), {ok, _Pid} = emqx_trace:start_link(). diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index b93839b0b..73c767a52 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -94,7 +94,8 @@ schema("/trace") -> 409 => emqx_dashboard_swagger:error_codes( [ 'ALREADY_EXISTS', - 'DUPLICATE_CONDITION' + 'DUPLICATE_CONDITION', + 'BAD_TYPE' ], <<"trace already exists">> ) @@ -265,6 +266,19 @@ fields(trace) -> example => running } )}, + {payload_encode, + hoconsc:mk(hoconsc:enum([hex, text, hidden]), #{ + desc => + "" + "Determine the format of the payload format in the trace file.
\n" + "`text`: Text-based protocol or plain text protocol.\n" + " It is recommended when payload is JSON encoded.
\n" + "`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
\n" + "`hidden`: payload is obfuscated as `******`" + "", + default => text, + required => false + })}, {start_at, hoconsc:mk( emqx_datetime:epoch_second(), @@ -421,6 +435,11 @@ trace(post, #{body := Param}) -> code => 'DUPLICATE_CONDITION', message => ?TO_BIN([Name, " Duplication Condition"]) }}; + {error, {bad_type, _}} -> + {409, #{ + code => 'BAD_TYPE', + message => <<"Rolling upgrade in progress, create failed">> + }}; {error, Reason} -> {400, #{ code => 'INVALID_PARAMS', diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 162d07aaa..7922bbb40 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -174,6 +174,13 @@ t_create_failed(_Config) -> {error, {"HTTP/1.1", 409, _}}, request_api(post, api_path("trace"), [GoodName2 | Trace]) ), + %% new name but bad payload-encode + GoodName3 = {<<"name">>, <<"test-name-2">>}, + PayloadEncode = {<<"payload_encode">>, <<"bad">>}, + ?assertMatch( + {error, {"HTTP/1.1", 400, _}}, + request_api(post, api_path("trace"), [GoodName3, PayloadEncode | Trace]) + ), unload(), emqx_trace:clear(), From 45254c59362e9e0450abdd2e884bee509affb3af Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 12 Apr 2023 17:14:30 +0800 Subject: [PATCH 056/279] chore: add changelog for create trace --- apps/emqx/src/emqx_trace/emqx_trace_handler.erl | 10 ++++++++-- apps/emqx_management/src/emqx_mgmt_api_trace.erl | 3 ++- changes/ce/feat-10373.en.md | 2 ++ 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 changes/ce/feat-10373.en.md diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index 7f586f7a4..231fd5e7b 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -44,7 +44,8 @@ -type tracer() :: #{ name := binary(), type := clientid | topic | ip_address, - filter := emqx_types:clientid() | emqx_types:topic() | emqx_trace:ip_address() + filter := emqx_types:clientid() | emqx_types:topic() | emqx_trace:ip_address(), + payload_encode := text | hidden | hex }. -define(CONFIG(_LogFile_), #{ @@ -70,7 +71,12 @@ LogFilePath :: string() ) -> ok | {error, term()}. install(Name, Type, Filter, Level, LogFile) -> - Who = #{type => Type, filter => ensure_bin(Filter), name => ensure_bin(Name)}, + Who = #{ + type => Type, + filter => ensure_bin(Filter), + name => ensure_bin(Name), + payload_encode => payload_encode() + }, install(Who, Level, LogFile). -spec install( diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 73c767a52..619da92f0 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -273,7 +273,8 @@ fields(trace) -> "Determine the format of the payload format in the trace file.
\n" "`text`: Text-based protocol or plain text protocol.\n" " It is recommended when payload is JSON encoded.
\n" - "`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
\n" + "`hex`: Binary hexadecimal encode." + "It is recommended when payload is a custom binary protocol.
\n" "`hidden`: payload is obfuscated as `******`" "", default => text, diff --git a/changes/ce/feat-10373.en.md b/changes/ce/feat-10373.en.md new file mode 100644 index 000000000..7609e2a1d --- /dev/null +++ b/changes/ce/feat-10373.en.md @@ -0,0 +1,2 @@ +Deprecate the trace.payload_encode configuration. +Add payload_encode=[text,hidden,hex] option when creating a trace via HTTP API. From 9774ad2cdd79abdbb26660a6f4b695a241629b66 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 12 Apr 2023 20:54:08 +0800 Subject: [PATCH 057/279] chore: update apps/emqx_gateway/README.md --- apps/emqx_gateway/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index 5aed6421b..d1f327db2 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -11,7 +11,7 @@ protocol access on EMQX. For example: - Configuration & Schema - HTTP/CLI management interfaces -The emqx_gateway application depends on `emqx`, `emqx_authn`, `emqx_ctl` that +The emqx_gateway application depends on `emqx`, `emqx_authn`, `emqx_authz`, `emqx_ctl` that provide the foundation for protocol access. More introduction: [Extended Protocol Gateway](https://www.emqx.io/docs/en/v5.0/gateway/gateway.html) From 9c9f39d0f7431ae7f17b711ff61c17135a223761 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 11 Apr 2023 17:01:34 +0300 Subject: [PATCH 058/279] feat(resman): also move out metrics collection for debugging Now `emqx_resource:list_instances_verbose/0` will populate the metrics for each instance, for the sake of simplicity. --- apps/emqx_resource/include/emqx_resource.hrl | 3 +-- apps/emqx_resource/src/emqx_resource.erl | 7 +++++-- apps/emqx_resource/src/emqx_resource_manager.erl | 7 +------ changes/ce/feat-10359.en.md | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index d799e7d93..7f88e1440 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -43,8 +43,7 @@ config := resource_config(), error := term(), state := resource_state(), - status := resource_status(), - metrics => emqx_metrics_worker:metrics() + status := resource_status() }. -type resource_group() :: binary(). -type creation_opts() :: #{ diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 054bd228b..d8b91942b 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -327,9 +327,12 @@ fetch_creation_opts(Opts) -> list_instances() -> [Id || #{id := Id} <- list_instances_verbose()]. --spec list_instances_verbose() -> [resource_data()]. +-spec list_instances_verbose() -> [_ResourceDataWithMetrics :: map()]. list_instances_verbose() -> - emqx_resource_manager:list_all(). + [ + Res#{metrics => get_metrics(ResId)} + || #{id := ResId} = Res <- emqx_resource_manager:list_all() + ]. -spec list_instances_by_type(module()) -> [resource_id()]. list_instances_by_type(ResourceType) -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index b5ccff05c..9fe0fbdf2 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -258,7 +258,7 @@ reset_metrics(ResId) -> list_all() -> try [ - data_record_to_external_map_with_metrics(Data) + data_record_to_external_map(Data) || {_Id, _Group, Data} <- ets:tab2list(?ETS_TABLE) ] catch @@ -669,11 +669,6 @@ data_record_to_external_map(Data) -> state => Data#data.state }. --spec data_record_to_external_map_with_metrics(data()) -> resource_data(). -data_record_to_external_map_with_metrics(Data) -> - DataMap = data_record_to_external_map(Data), - DataMap#{metrics => get_metrics(Data#data.id)}. - -spec wait_for_ready(resource_id(), integer()) -> ok | timeout | {error, term()}. wait_for_ready(ResId, WaitTime) -> do_wait_for_ready(ResId, WaitTime div ?WAIT_FOR_RESOURCE_DELAY). diff --git a/changes/ce/feat-10359.en.md b/changes/ce/feat-10359.en.md index 21a253556..524b6dbdd 100644 --- a/changes/ce/feat-10359.en.md +++ b/changes/ce/feat-10359.en.md @@ -1 +1 @@ -Metrics now are not implicitly collected in places where API handlers don't make any use of them. Instead, a separate backplane RPC gathers cluster-wide metrics. \ No newline at end of file +Metrics now are not implicitly collected in places where API handlers don't make any use of them. Instead, a separate backplane RPC gathers cluster-wide metrics. From 1e3360019041a4c0937ffe5c7f2a098c4c863751 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 12 Apr 2023 17:33:53 +0200 Subject: [PATCH 059/279] chore: bump version to v5.0.22 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 84f4468e4..64fa12d3f 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.21"). +-define(EMQX_RELEASE_CE, "5.0.22"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.2"). diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index a0662c9cd..ecc211e35 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.21 +version: 5.0.22 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.21 +appVersion: 5.0.22 From 871ee90b3e3138f8e663df68b7d248ca18e70406 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 6 Apr 2023 15:04:44 -0300 Subject: [PATCH 060/279] refactor(kafka_bridge): move kafka bridge into its own app Fixes https://emqx.atlassian.net/browse/EMQX-9481 --- .github/workflows/elixir_deps_check.yaml | 11 +++ apps/emqx_bridge_kafka/BSL.txt | 94 +++++++++++++++++++ apps/emqx_bridge_kafka/README.md | 19 ++++ apps/emqx_bridge_kafka/docker-ct | 2 + .../etc/emqx_bridge_kafka.conf | 0 apps/emqx_bridge_kafka/rebar.config | 14 +++ .../src/emqx_bridge_kafka.app.src | 16 ++++ .../src/emqx_bridge_kafka.erl | 2 +- .../src/emqx_bridge_kafka_consumer_sup.erl | 2 +- .../src/emqx_bridge_kafka_impl.erl | 2 +- .../src/emqx_bridge_kafka_impl_consumer.erl | 27 +++--- .../src/emqx_bridge_kafka_impl_producer.erl | 8 +- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 34 ++++--- .../emqx_bridge_kafka_impl_producer_SUITE.erl | 10 +- .../test/emqx_bridge_kafka_tests.erl | 2 +- lib-ee/emqx_ee_bridge/docker-ct | 1 - lib-ee/emqx_ee_bridge/rebar.config | 6 +- .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 5 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 16 ++-- .../src/emqx_ee_connector.app.src | 2 - mix.exs | 25 ++++- rebar.config.erl | 14 ++- ...ge_kafka.hocon => emqx_bridge_kafka.hocon} | 2 +- scripts/ct/run.sh | 6 +- scripts/find-apps.sh | 3 + 25 files changed, 264 insertions(+), 59 deletions(-) create mode 100644 apps/emqx_bridge_kafka/BSL.txt create mode 100644 apps/emqx_bridge_kafka/README.md create mode 100644 apps/emqx_bridge_kafka/docker-ct create mode 100644 apps/emqx_bridge_kafka/etc/emqx_bridge_kafka.conf create mode 100644 apps/emqx_bridge_kafka/rebar.config create mode 100644 apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl => apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl (99%) rename lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl => apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_sup.erl (98%) rename lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl => apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl (97%) rename lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl => apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl (94%) rename lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl => apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl (98%) rename lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl => apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl (98%) rename lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl => apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl (98%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl => apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl (99%) rename rel/i18n/{emqx_ee_bridge_kafka.hocon => emqx_bridge_kafka.hocon} (99%) diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 511639a3c..d6449f563 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -23,7 +23,18 @@ jobs: mix local.hex --force mix local.rebar --force mix deps.get + # we check only enterprise because `rebar3 tree`, even if an + # enterprise app is excluded from `project_app_dirs` in + # `rebar.config.erl`, will still list dependencies from it. + # Since the enterprise profile is a superset of the + # community one and thus more complete, we use the former. + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise - name: check elixir deps run: ./scripts/check-elixir-deps-discrepancies.exs + env: + MIX_ENV: emqx-enterprise + PROFILE: emqx-enterprise ... diff --git a/apps/emqx_bridge_kafka/BSL.txt b/apps/emqx_bridge_kafka/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_kafka/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_kafka/README.md b/apps/emqx_bridge_kafka/README.md new file mode 100644 index 000000000..72cbeecc6 --- /dev/null +++ b/apps/emqx_bridge_kafka/README.md @@ -0,0 +1,19 @@ +# Kafka Data Integration Bridge + +This application houses the Kafka Producer and Consumer data +integration bridges for EMQX Enterprise Edition. It provides the +means to connect to Kafka and publish/consume messages to/from it. + +Currently, our Kafka Producer library (`wolff`) has its own `replayq` +buffering implementation, so this bridge does not require buffer +workers from `emqx_resource`. It implements the connection management +and interaction without need for a separate connector app, since it's +not used by authentication and authorization applications. + +## Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + +## License + +See [BSL](./BSL.txt). diff --git a/apps/emqx_bridge_kafka/docker-ct b/apps/emqx_bridge_kafka/docker-ct new file mode 100644 index 000000000..5288ee246 --- /dev/null +++ b/apps/emqx_bridge_kafka/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +kafka diff --git a/apps/emqx_bridge_kafka/etc/emqx_bridge_kafka.conf b/apps/emqx_bridge_kafka/etc/emqx_bridge_kafka.conf new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_bridge_kafka/rebar.config b/apps/emqx_bridge_kafka/rebar.config new file mode 100644 index 000000000..fd21fd15b --- /dev/null +++ b/apps/emqx_bridge_kafka/rebar.config @@ -0,0 +1,14 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.5"}}} + , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} + , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} + , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_kafka]} +]}. diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src new file mode 100644 index 000000000..a4fbe5673 --- /dev/null +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -0,0 +1,16 @@ +{application, emqx_bridge_kafka, [ + {description, "EMQX Enterprise Kafka Bridge"}, + {vsn, "0.1.0"}, + {registered, [emqx_bridge_kafka_consumer_sup]}, + {applications, [ + kernel, + stdlib, + telemetry, + wolff, + brod + ]}, + {env, []}, + {modules, []}, + + {links, []} +]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index f3dfa5964..30f6cd60d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_kafka). +-module(emqx_bridge_kafka). -include_lib("emqx_connector/include/emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_sup.erl similarity index 98% rename from lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_sup.erl index feec8c09b..638c1def6 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_ee_bridge_kafka_consumer_sup.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_sup.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_kafka_consumer_sup). +-module(emqx_bridge_kafka_consumer_sup). -behaviour(supervisor). diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl similarity index 97% rename from lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl index c9dcce9a2..22a67c551 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl @@ -3,7 +3,7 @@ %%-------------------------------------------------------------------- %% Kafka connection configuration --module(emqx_bridge_impl_kafka). +-module(emqx_bridge_kafka_impl). -export([ hosts/1, diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl similarity index 94% rename from lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index f4dc3456e..2dc43a130 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_impl_kafka_consumer). +-module(emqx_bridge_kafka_impl_consumer). -behaviour(emqx_resource). @@ -52,8 +52,9 @@ ssl := _, any() => term() }. --type subscriber_id() :: emqx_ee_bridge_kafka_consumer_sup:child_id(). +-type subscriber_id() :: emqx_bridge_kafka_consumer_sup:child_id(). -type kafka_topic() :: brod:topic(). +-type kafka_message() :: #kafka_message{}. -type state() :: #{ kafka_topics := nonempty_list(kafka_topic()), subscriber_id := subscriber_id(), @@ -129,14 +130,14 @@ on_start(InstanceId, Config) -> ssl := SSL, topic_mapping := _ } = Config, - BootstrapHosts = emqx_bridge_impl_kafka:hosts(BootstrapHosts0), + BootstrapHosts = emqx_bridge_kafka_impl:hosts(BootstrapHosts0), KafkaType = kafka_consumer, %% Note: this is distinct per node. ClientID = make_client_id(InstanceId, KafkaType, BridgeName), ClientOpts0 = case Auth of none -> []; - Auth -> [{sasl, emqx_bridge_impl_kafka:sasl(Auth)}] + Auth -> [{sasl, emqx_bridge_kafka_impl:sasl(Auth)}] end, ClientOpts = add_ssl_opts(ClientOpts0, SSL), case brod:start_client(BootstrapHosts, ClientID, ClientOpts) of @@ -191,7 +192,7 @@ init(GroupData, State0) -> State = State0#{kafka_topic => KafkaTopic}, {ok, State}. --spec handle_message(#kafka_message{}, consumer_state()) -> {ok, commit, consumer_state()}. +-spec handle_message(kafka_message(), consumer_state()) -> {ok, commit, consumer_state()}. handle_message(Message, State) -> ?tp_span( kafka_consumer_handle_message, @@ -240,13 +241,13 @@ add_ssl_opts(ClientOpts, #{enable := false}) -> add_ssl_opts(ClientOpts, SSL) -> [{ssl, emqx_tls_lib:to_client_opts(SSL)} | ClientOpts]. --spec make_subscriber_id(atom() | binary()) -> emqx_ee_bridge_kafka_consumer_sup:child_id(). +-spec make_subscriber_id(atom() | binary()) -> emqx_bridge_kafka_consumer_sup:child_id(). make_subscriber_id(BridgeName) -> BridgeNameBin = to_bin(BridgeName), <<"kafka_subscriber:", BridgeNameBin/binary>>. ensure_consumer_supervisor_started() -> - Mod = emqx_ee_bridge_kafka_consumer_sup, + Mod = emqx_bridge_kafka_consumer_sup, ChildSpec = #{ id => Mod, @@ -327,7 +328,7 @@ start_consumer(Config, InstanceId, ClientID) -> %% spawns one worker for each assigned topic-partition %% automatically, so we should not spawn duplicate workers. SubscriberId = make_subscriber_id(BridgeName), - case emqx_ee_bridge_kafka_consumer_sup:start_child(SubscriberId, GroupSubscriberConfig) of + case emqx_bridge_kafka_consumer_sup:start_child(SubscriberId, GroupSubscriberConfig) of {ok, _ConsumerPid} -> ?tp( kafka_consumer_subscriber_started, @@ -342,18 +343,18 @@ start_consumer(Config, InstanceId, ClientID) -> ?SLOG(error, #{ msg => "failed_to_start_kafka_consumer", instance_id => InstanceId, - kafka_hosts => emqx_bridge_impl_kafka:hosts(BootstrapHosts0), + kafka_hosts => emqx_bridge_kafka_impl:hosts(BootstrapHosts0), reason => emqx_misc:redact(Reason2) }), stop_client(ClientID), throw(failed_to_start_kafka_consumer) end. --spec stop_subscriber(emqx_ee_bridge_kafka_consumer_sup:child_id()) -> ok. +-spec stop_subscriber(emqx_bridge_kafka_consumer_sup:child_id()) -> ok. stop_subscriber(SubscriberId) -> _ = log_when_error( fun() -> - emqx_ee_bridge_kafka_consumer_sup:ensure_child_deleted(SubscriberId) + emqx_bridge_kafka_consumer_sup:ensure_child_deleted(SubscriberId) end, #{ msg => "failed_to_delete_kafka_subscriber", @@ -437,7 +438,7 @@ do_get_status1(ClientID, KafkaTopic, SubscriberId, NPartitions) -> end. are_subscriber_workers_alive(SubscriberId) -> - Children = supervisor:which_children(emqx_ee_bridge_kafka_consumer_sup), + Children = supervisor:which_children(emqx_bridge_kafka_consumer_sup), case lists:keyfind(SubscriberId, 1, Children) of false -> false; @@ -479,7 +480,7 @@ is_dry_run(InstanceId) -> make_client_id(InstanceId, KafkaType, KafkaName) -> case is_dry_run(InstanceId) of false -> - ClientID0 = emqx_bridge_impl_kafka:make_client_id(KafkaType, KafkaName), + ClientID0 = emqx_bridge_kafka_impl:make_client_id(KafkaType, KafkaName), binary_to_atom(ClientID0); true -> %% It is a dry run and we don't want to leak too many diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl similarity index 98% rename from lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl rename to apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 09713a431..7bee2c70d 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_impl_kafka_producer). +-module(emqx_bridge_kafka_impl_producer). -include_lib("emqx_resource/include/emqx_resource.hrl"). @@ -47,15 +47,15 @@ on_start(InstId, Config) -> BridgeType = ?BRIDGE_TYPE, ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName), _ = maybe_install_wolff_telemetry_handlers(ResourceId), - Hosts = emqx_bridge_impl_kafka:hosts(Hosts0), - ClientId = emqx_bridge_impl_kafka:make_client_id(BridgeType, BridgeName), + Hosts = emqx_bridge_kafka_impl:hosts(Hosts0), + ClientId = emqx_bridge_kafka_impl:make_client_id(BridgeType, BridgeName), ClientConfig = #{ min_metadata_refresh_interval => MinMetaRefreshInterval, connect_timeout => ConnTimeout, client_id => ClientId, request_timeout => MetaReqTimeout, extra_sock_opts => socket_opts(SocketOpts), - sasl => emqx_bridge_impl_kafka:sasl(Auth), + sasl => emqx_bridge_kafka_impl:sasl(Auth), ssl => ssl(SSL) }, case wolff:ensure_supervised_client(ClientId, Hosts, ClientConfig) of diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl similarity index 98% rename from lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl rename to apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 4019a9c42..fb7cf524c 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_impl_kafka_consumer_SUITE). +-module(emqx_bridge_kafka_impl_consumer_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -15,6 +15,7 @@ -import(emqx_common_test_helpers, [on_exit/1]). -define(BRIDGE_TYPE_BIN, <<"kafka_consumer">>). +-define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_bridge_kafka]). %%------------------------------------------------------------------------------ %% CT boilerplate @@ -67,7 +68,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]), + ok = emqx_connector_test_helpers:stop_apps(lists:reverse(?APPS)), _ = application:stop(emqx_connector), ok. @@ -228,7 +229,7 @@ common_init_per_group() -> emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), application:load(emqx_bridge), ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]), + ok = emqx_connector_test_helpers:start_apps(?APPS), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), UniqueNum = integer_to_binary(erlang:unique_integer()), @@ -408,7 +409,7 @@ start_producers(TestCase, Config) -> DirectKafkaPort = ?config(direct_kafka_port, Config), UseTLS = ?config(use_tls, Config), UseSASL = ?config(use_sasl, Config), - Hosts = emqx_bridge_impl_kafka:hosts( + Hosts = emqx_bridge_kafka_impl:hosts( DirectKafkaHost ++ ":" ++ integer_to_list(DirectKafkaPort) ), SSL = @@ -876,7 +877,7 @@ ensure_connected(Config) -> consumer_clientid(Config) -> KafkaName = ?config(kafka_name, Config), - binary_to_atom(emqx_bridge_impl_kafka:make_client_id(kafka_consumer, KafkaName)). + binary_to_atom(emqx_bridge_kafka_impl:make_client_id(kafka_consumer, KafkaName)). get_client_connection(Config) -> KafkaHost = ?config(kafka_host, Config), @@ -885,7 +886,7 @@ get_client_connection(Config) -> brod_client:get_connection(ClientID, KafkaHost, KafkaPort). get_subscriber_workers() -> - [{_, SubscriberPid, _, _}] = supervisor:which_children(emqx_ee_bridge_kafka_consumer_sup), + [{_, SubscriberPid, _, _}] = supervisor:which_children(emqx_bridge_kafka_consumer_sup), brod_group_subscriber_v2:get_workers(SubscriberPid). wait_downs(Refs, _Timeout) when map_size(Refs) =:= 0 -> @@ -1069,7 +1070,7 @@ cluster(Config) -> Cluster = emqx_common_test_helpers:emqx_cluster( [core, core], [ - {apps, [emqx_conf, emqx_bridge, emqx_rule_engine]}, + {apps, [emqx_conf, emqx_bridge, emqx_rule_engine, emqx_bridge_kafka]}, {listener_ports, []}, {peer_mod, PeerModule}, {priv_data_dir, PrivDataDir}, @@ -1504,7 +1505,7 @@ do_t_receive_after_recovery(Config) -> _Interval = 500, _NAttempts = 20, begin - GroupId = emqx_bridge_impl_kafka_consumer:consumer_group_id(KafkaNameA), + GroupId = emqx_bridge_kafka_impl_consumer:consumer_group_id(KafkaNameA), {ok, [#{partitions := Partitions}]} = brod:fetch_committed_offsets( KafkaClientId, GroupId ), @@ -1745,8 +1746,12 @@ t_node_joins_existing_cluster(Config) -> ?check_trace( begin [{Name1, Opts1}, {Name2, Opts2} | _] = Cluster, + ct:pal("starting ~p", [Name1]), N1 = emqx_common_test_helpers:start_slave(Name1, Opts1), - on_exit(fun() -> ok = emqx_common_test_helpers:stop_slave(N1) end), + on_exit(fun() -> + ct:pal("stopping ~p", [N1]), + ok = emqx_common_test_helpers:stop_slave(N1) + end), setup_group_subscriber_spy(N1), {{ok, _}, {ok, _}} = ?wait_async_action( @@ -1785,8 +1790,12 @@ t_node_joins_existing_cluster(Config) -> 1, 30_000 ), + ct:pal("starting ~p", [Name2]), N2 = emqx_common_test_helpers:start_slave(Name2, Opts2), - on_exit(fun() -> ok = emqx_common_test_helpers:stop_slave(N2) end), + on_exit(fun() -> + ct:pal("stopping ~p", [N2]), + ok = emqx_common_test_helpers:stop_slave(N2) + end), setup_group_subscriber_spy(N2), Nodes = [N1, N2], wait_for_cluster_rpc(N2), @@ -1873,7 +1882,10 @@ t_cluster_node_down(Config) -> Nodes = [N1, N2 | _] = lists:map( - fun({Name, Opts}) -> emqx_common_test_helpers:start_slave(Name, Opts) end, + fun({Name, Opts}) -> + ct:pal("starting ~p", [Name]), + emqx_common_test_helpers:start_slave(Name, Opts) + end, Cluster ), on_exit(fun() -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl similarity index 98% rename from lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl rename to apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index 9b6ac05a7..6e3ddf5bb 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_impl_kafka_producer_SUITE). +-module(emqx_bridge_kafka_impl_producer_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -12,7 +12,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("brod/include/brod.hrl"). --define(PRODUCER, emqx_bridge_impl_kafka_producer). +-define(PRODUCER, emqx_bridge_kafka_impl_producer). %%------------------------------------------------------------------------------ %% Things for REST API tests @@ -41,6 +41,8 @@ %% to hocon; keeping this as just `kafka' for backwards compatibility. -define(BRIDGE_TYPE, "kafka"). +-define(APPS, [emqx_resource, emqx_bridge, emqx_rule_engine, emqx_bridge_kafka]). + %%------------------------------------------------------------------------------ %% CT boilerplate %%------------------------------------------------------------------------------ @@ -76,7 +78,7 @@ init_per_suite(Config) -> _ = emqx_ee_bridge:module_info(), application:load(emqx_bridge), ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]), + ok = emqx_connector_test_helpers:start_apps(?APPS), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), wait_until_kafka_is_up(), @@ -96,7 +98,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]), + ok = emqx_connector_test_helpers:stop_apps(lists:reverse(?APPS)), _ = application:stop(emqx_connector), ok. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl rename to apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl index 1b32f856d..fa352ce89 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_kafka_tests.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_kafka_tests). +-module(emqx_bridge_kafka_tests). -include_lib("eunit/include/eunit.hrl"). diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index 963122082..0e947d89e 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -1,6 +1,5 @@ toxiproxy influxdb -kafka mongo mongo_rs_sharded mysql diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index 1c7d130ae..afd90f622 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,9 +1,5 @@ {erl_opts, [debug_info]}. -{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.5"}}} - , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}} - , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0-rc1"}}} - , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}} - , {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.1"}}} +{deps, [ {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.1"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 8316545a3..f1793d2e0 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,12 +1,13 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, {vsn, "0.1.9"}, - {registered, [emqx_ee_bridge_kafka_consumer_sup]}, + {registered, []}, {applications, [ kernel, stdlib, emqx_ee_connector, - telemetry + telemetry, + emqx_bridge_kafka ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 1ddc1a110..84b0b98b0 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -15,8 +15,8 @@ api_schemas(Method) -> [ ref(emqx_ee_bridge_gcp_pubsub, Method), - ref(emqx_ee_bridge_kafka, Method ++ "_consumer"), - ref(emqx_ee_bridge_kafka, Method ++ "_producer"), + ref(emqx_bridge_kafka, Method ++ "_consumer"), + ref(emqx_bridge_kafka, Method ++ "_producer"), ref(emqx_ee_bridge_mysql, Method), ref(emqx_ee_bridge_pgsql, Method), ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), @@ -39,7 +39,7 @@ api_schemas(Method) -> schema_modules() -> [ - emqx_ee_bridge_kafka, + emqx_bridge_kafka, emqx_ee_bridge_hstreamdb, emqx_ee_bridge_gcp_pubsub, emqx_ee_bridge_influxdb, @@ -69,10 +69,10 @@ examples(Method) -> lists:foldl(Fun, #{}, schema_modules()). resource_type(Type) when is_binary(Type) -> resource_type(binary_to_atom(Type, utf8)); -resource_type(kafka_consumer) -> emqx_bridge_impl_kafka_consumer; +resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer; %% TODO: rename this to `kafka_producer' after alias support is added %% to hocon; keeping this as just `kafka' for backwards compatibility. -resource_type(kafka) -> emqx_bridge_impl_kafka_producer; +resource_type(kafka) -> emqx_bridge_kafka_impl_producer; resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; resource_type(gcp_pubsub) -> emqx_ee_connector_gcp_pubsub; resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; @@ -174,16 +174,16 @@ kafka_structs() -> %% backwards compatibility. {kafka, mk( - hoconsc:map(name, ref(emqx_ee_bridge_kafka, kafka_producer)), + hoconsc:map(name, ref(emqx_bridge_kafka, kafka_producer)), #{ desc => <<"Kafka Producer Bridge Config">>, required => false, - converter => fun emqx_ee_bridge_kafka:kafka_producer_converter/2 + converter => fun emqx_bridge_kafka:kafka_producer_converter/2 } )}, {kafka_consumer, mk( - hoconsc:map(name, ref(emqx_ee_bridge_kafka, kafka_consumer)), + hoconsc:map(name, ref(emqx_bridge_kafka, kafka_consumer)), #{desc => <<"Kafka Consumer Bridge Config">>, required => false} )} ]. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index cd176081b..2e6406d70 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -8,8 +8,6 @@ hstreamdb_erl, influxdb, tdengine, - wolff, - brod, clickhouse, erlcloud, rocketmq, diff --git a/mix.exs b/mix.exs index 56840579d..6709f1774 100644 --- a/mix.exs +++ b/mix.exs @@ -107,6 +107,8 @@ defmodule EMQXUmbrella.MixProject do end defp umbrella_apps() do + enterprise_apps = enterprise_umbrella_apps() + "apps/*" |> Path.wildcard() |> Enum.map(fn path -> @@ -117,9 +119,20 @@ defmodule EMQXUmbrella.MixProject do {app, path: path, manager: :rebar3, override: true} end) + |> Enum.reject(fn dep_spec -> + dep_spec + |> elem(0) + |> then(&MapSet.member?(enterprise_apps, &1)) + end) end defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do + umbrella_apps = + Enum.map(enterprise_umbrella_apps(), fn app_name -> + path = "apps/#{app_name}" + {app_name, path: path, manager: :rebar3, override: true} + end) + "lib-ee/*" |> Path.wildcard() |> Enum.filter(&File.dir?/1) @@ -131,12 +144,20 @@ defmodule EMQXUmbrella.MixProject do {app, path: path, manager: :rebar3, override: true} end) + |> Enum.concat(umbrella_apps) end defp enterprise_apps(_profile_info) do [] end + # need to remove those when listing `/apps/`... + defp enterprise_umbrella_apps() do + MapSet.new([ + :emqx_bridge_kafka + ]) + end + defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do [ {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}, @@ -146,7 +167,8 @@ defmodule EMQXUmbrella.MixProject do {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0-rc1"}, {:brod, github: "kafka4beam/brod", tag: "3.16.8"}, {:snappyer, "1.2.8", override: true}, - {:supervisor3, "1.1.11", override: true} + {:crc32cer, "0.1.8", override: true}, + {:supervisor3, "1.1.12", override: true} ] end @@ -320,6 +342,7 @@ defmodule EMQXUmbrella.MixProject do emqx_ee_conf: :load, emqx_ee_connector: :permanent, emqx_ee_bridge: :permanent, + emqx_bridge_kafka: :permanent, emqx_ee_schema_registry: :permanent ], else: [] diff --git a/rebar.config.erl b/rebar.config.erl index cdd628664..39c336422 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -78,6 +78,9 @@ is_cover_enabled() -> is_enterprise(ce) -> false; is_enterprise(ee) -> true. +is_community_umbrella_app("apps/emqx_bridge_kafka") -> false; +is_community_umbrella_app(_) -> true. + is_jq_supported() -> not (false =/= os:getenv("BUILD_WITHOUT_JQ") orelse is_win32()) orelse @@ -122,8 +125,14 @@ project_app_dirs() -> project_app_dirs(get_edition_from_profile_env()). project_app_dirs(Edition) -> - ["apps/*"] ++ - case is_enterprise(Edition) of + IsEnterprise = is_enterprise(Edition), + UmbrellaApps = [ + Path + || Path <- filelib:wildcard("apps/*"), + is_community_umbrella_app(Path) orelse IsEnterprise + ], + UmbrellaApps ++ + case IsEnterprise of true -> ["lib-ee/*"]; false -> [] end. @@ -428,6 +437,7 @@ relx_apps_per_edition(ee) -> {emqx_ee_conf, load}, emqx_ee_connector, emqx_ee_bridge, + emqx_bridge_kafka, emqx_ee_schema_registry ]; relx_apps_per_edition(ce) -> diff --git a/rel/i18n/emqx_ee_bridge_kafka.hocon b/rel/i18n/emqx_bridge_kafka.hocon similarity index 99% rename from rel/i18n/emqx_ee_bridge_kafka.hocon rename to rel/i18n/emqx_bridge_kafka.hocon index 1638eb89f..2f1811269 100644 --- a/rel/i18n/emqx_ee_bridge_kafka.hocon +++ b/rel/i18n/emqx_bridge_kafka.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_kafka { +emqx_bridge_kafka { config_enable { desc { en: "Enable (true) or disable (false) this Kafka bridge." diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 82823720d..83e69d1ad 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -107,6 +107,10 @@ case "${WHICH_APP}" in ## ensure enterprise profile when testing lib-ee applications export PROFILE='emqx-enterprise' ;; + apps/emqx_bridge_kafka) + ## ensure enterprise profile when testing ee applications + export PROFILE='emqx-enterprise' + ;; *) export PROFILE="${PROFILE:-emqx}" ;; @@ -172,7 +176,7 @@ for dep in ${CT_DEPS}; do ;; rocketmq) FILES+=( '.ci/docker-compose-file/docker-compose-rocketmq.yaml' ) - ;; + ;; cassandra) FILES+=( '.ci/docker-compose-file/docker-compose-cassandra.yaml' ) ;; diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index f07cd2f7d..66990ae12 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -72,6 +72,9 @@ describe_app() { runner="docker" fi case "${app}" in + apps/emqx_bridge_kafka) + profile='emqx-enterprise' + ;; apps/*) profile='emqx' ;; From a70930fed04b721e2c1c5e1753de5ea319ff377d Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 13 Apr 2023 09:26:16 +0800 Subject: [PATCH 061/279] feat: hiden auto_subscribe conf --- .../src/emqx_auto_subscribe.app.src | 2 +- .../src/emqx_auto_subscribe_schema.erl | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src index a273face1..d6f6f4058 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auto_subscribe, [ {description, "Auto subscribe Application"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {mod, {emqx_auto_subscribe_app, []}}, {applications, [ diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl index a01e17c1f..e6eea2b74 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl @@ -31,14 +31,18 @@ namespace() -> "auto_subscribe". roots() -> - ["auto_subscribe"]. + [{"auto_subscribe", ?HOCON(?R_REF("auto_subscribe"), #{importance => ?IMPORTANCE_HIDDEN})}]. fields("auto_subscribe") -> [ {topics, ?HOCON( ?ARRAY(?R_REF("topic")), - #{desc => ?DESC(auto_subscribe), default => []} + #{ + desc => ?DESC(auto_subscribe), + default => [], + importance => ?IMPORTANCE_HIDDEN + } )} ]; fields("topic") -> @@ -47,26 +51,31 @@ fields("topic") -> ?HOCON(binary(), #{ required => true, example => topic_example(), + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC("topic") })}, {qos, ?HOCON(emqx_schema:qos(), #{ default => 0, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC("qos") })}, {rh, ?HOCON(range(0, 2), #{ default => 0, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC("rh") })}, {rap, ?HOCON(range(0, 1), #{ default => 0, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC("rap") })}, {nl, ?HOCON(range(0, 1), #{ default => 0, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(nl) })} ]. From 48600cbc8e502a7e363cf3b08409950cbb6ccc29 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 13 Apr 2023 09:50:11 +0800 Subject: [PATCH 062/279] chore: add auto_subscribe changelog --- changes/ce/feat-10381.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-10381.en.md diff --git a/changes/ce/feat-10381.en.md b/changes/ce/feat-10381.en.md new file mode 100644 index 000000000..355839db9 --- /dev/null +++ b/changes/ce/feat-10381.en.md @@ -0,0 +1 @@ +Hide the auto_subscribe configuration item so that it can be modified later only through the HTTP API. From 17b2a622462418b2068b52086b14a104cbaeb79a Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 13 Apr 2023 11:02:10 +0800 Subject: [PATCH 063/279] chore: update changes/ce/feat-10381.en.md Co-authored-by: JianBo He --- changes/ce/feat-10381.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/feat-10381.en.md b/changes/ce/feat-10381.en.md index 355839db9..3ea11188f 100644 --- a/changes/ce/feat-10381.en.md +++ b/changes/ce/feat-10381.en.md @@ -1 +1 @@ -Hide the auto_subscribe configuration item so that it can be modified later only through the HTTP API. +Hide the `auto_subscribe` configuration items so that they can be modified later only through the HTTP API. From b57b964083c54cc46e4f920f768dea67e2e157e6 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 13 Apr 2023 11:15:19 +0800 Subject: [PATCH 064/279] chore(i18n): improve config desc --- rel/i18n/emqx_conf_schema.hocon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index 2f0c6b938..e9f9c1211 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -1190,11 +1190,11 @@ To flush events, the handler discards the buffered log messages without logging. log_overload_kill_restart_after { desc { en: """The handler restarts automatically after a delay in the event of termination, unless the value `infinity` is set, which blocks any subsequent restarts.""" - zh: """如果处理进程终止，它会在以指定的时间后后自动重新启动。 `infinity` 不自动重启。""" + zh: """处理进程停止后，会在该延迟时间后自动重新启动。除非该值设置为 infinity，这表示永不重新启动。""" } label { en: "Handler Restart Timer" - zh: "处理进程重启机制" + zh: "处理进程重启延迟" } } From a4d9234b24d802f599f50612f7c2c6eaeecd93dc Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 13 Apr 2023 12:02:22 +0800 Subject: [PATCH 065/279] test(dynamo): remove the flaky test case --- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index c0d58c4f7..0c0dbbe04 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -40,7 +40,7 @@ groups() -> %% due to the poorly implemented driver or other reasons %% if we mix these cases with others, this suite will become flaky. - Flaky = [t_get_status, t_write_failure, t_write_timeout], + Flaky = [t_get_status, t_write_failure], TCs = TCs0 -- Flaky, [ @@ -383,25 +383,6 @@ t_write_failure(Config) -> end), ok. -t_write_timeout(Config) -> - ProxyName = ?config(proxy_name, Config), - ProxyPort = ?config(proxy_port, Config), - ProxyHost = ?config(proxy_host, Config), - {{ok, _}, {ok, _}} = - ?wait_async_action( - create_bridge(Config), - #{?snk_kind := resource_connected_enter}, - 20_000 - ), - SentData = #{id => emqx_misc:gen_id(), payload => ?PAYLOAD}, - emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - query_resource(Config, {send_message, SentData}) - ) - end), - ok. - t_simple_query(Config) -> ?assertMatch( {ok, _}, From 4ba4c6bf6219f419941b18961aebbcffca633946 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 13 Apr 2023 14:22:13 +0800 Subject: [PATCH 066/279] chore: only hiden root keys --- .../src/emqx_auto_subscribe_schema.erl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl index e6eea2b74..98f32c8a4 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl @@ -40,8 +40,7 @@ fields("auto_subscribe") -> ?ARRAY(?R_REF("topic")), #{ desc => ?DESC(auto_subscribe), - default => [], - importance => ?IMPORTANCE_HIDDEN + default => [] } )} ]; @@ -51,31 +50,26 @@ fields("topic") -> ?HOCON(binary(), #{ required => true, example => topic_example(), - importance => ?IMPORTANCE_HIDDEN, desc => ?DESC("topic") })}, {qos, ?HOCON(emqx_schema:qos(), #{ default => 0, - importance => ?IMPORTANCE_HIDDEN, desc => ?DESC("qos") })}, {rh, ?HOCON(range(0, 2), #{ default => 0, - importance => ?IMPORTANCE_HIDDEN, desc => ?DESC("rh") })}, {rap, ?HOCON(range(0, 1), #{ default => 0, - importance => ?IMPORTANCE_HIDDEN, desc => ?DESC("rap") })}, {nl, ?HOCON(range(0, 1), #{ default => 0, - importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(nl) })} ]. From cc2beda37a5d680e9bdee231686d48b157f27a8d Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 13 Apr 2023 14:42:21 +0800 Subject: [PATCH 067/279] feat: hiden rule_engine/bridge/authz/authn from doc/example --- apps/emqx/src/emqx_schema.erl | 8 ++++++-- apps/emqx_bridge/src/schema/emqx_bridge_schema.erl | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 5 ++++- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 20018b2d5..948c5d6d3 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -163,7 +163,7 @@ roots(high) -> {?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, sc( ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )} ]; roots(medium) -> @@ -2760,7 +2760,11 @@ authentication(Which) -> Module -> Module:root_type() end, - hoconsc:mk(Type, #{desc => Desc, converter => fun ensure_array/2}). + hoconsc:mk(Type, #{ + desc => Desc, + converter => fun ensure_array/2, + importance => ?IMPORTANCE_HIDDEN + }). %% the older version schema allows individual element (instead of a chain) in config ensure_array(undefined, _) -> undefined; diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 6c278a5ec..52000ba7f 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -137,7 +137,7 @@ namespace() -> "bridge". tags() -> [<<"Bridge">>]. -roots() -> [bridges]. +roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_HIDDEN})}]. fields(bridges) -> [ diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 58bcf9700..a24f43801 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1278,7 +1278,10 @@ emqx_schema_high_prio_roots() -> {"authorization", sc( ?R_REF("authorization"), - #{desc => ?DESC(authorization)} + #{ + desc => ?DESC(authorization), + importance => ?IMPORTANCE_HIDDEN + } )}, lists:keyreplace("authorization", 1, Roots, Authz). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 5b205f355..242c86c71 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -38,7 +38,7 @@ namespace() -> rule_engine. tags() -> [<<"Rule Engine">>]. -roots() -> ["rule_engine"]. +roots() -> [{"rule_engine", ?HOCON(?R_REF("rule_engine"), #{importance => ?IMPORTANCE_HIDDEN})}]. fields("rule_engine") -> rule_engine_settings() ++ From 0a62d6c55639f7ddaf5b4e9f8420b8a245739828 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 13 Apr 2023 14:53:47 +0800 Subject: [PATCH 068/279] refactor(gw): rename all gateway application name --- .../.gitignore | 0 .../README.md | 0 .../doc/flow.png | Bin .../doc/shared_state.png | Bin .../doc/transport.png | Bin .../include/emqx_coap.hrl | 0 .../rebar.config | 0 .../src/emqx_coap_api.erl | 0 .../src/emqx_coap_channel.erl | 0 .../src/emqx_coap_frame.erl | 0 .../src/emqx_coap_medium.erl | 0 .../src/emqx_coap_message.erl | 0 .../src/emqx_coap_mqtt_handler.erl | 0 .../src/emqx_coap_observe_res.erl | 0 .../src/emqx_coap_pubsub_handler.erl | 0 .../src/emqx_coap_schema.erl | 0 .../src/emqx_coap_session.erl | 0 .../src/emqx_coap_tm.erl | 0 .../src/emqx_coap_transport.erl | 0 .../src/emqx_gateway_coap.app.src} | 2 +- .../src/emqx_gateway_coap.erl} | 2 +- .../test/emqx_coap_SUITE.erl | 0 .../test/emqx_coap_api_SUITE.erl | 0 .../.gitignore | 0 .../README.md | 0 .../include/emqx_exproto.hrl | 0 .../priv/protos/exproto.proto | 0 .../rebar.config | 0 .../src/emqx_exproto_channel.erl | 0 .../src/emqx_exproto_frame.erl | 0 .../src/emqx_exproto_gcli.erl | 0 .../src/emqx_exproto_gsvr.erl | 0 .../src/emqx_exproto_schema.erl | 0 .../src/emqx_gateway_exproto.app.src} | 2 +- .../src/emqx_gateway_exproto.erl} | 2 +- .../test/emqx_exproto_SUITE.erl | 0 .../test/emqx_exproto_echo_svr.erl | 0 .../.gitignore | 0 .../README.md | 0 .../include/emqx_lwm2m.hrl | 0 .../lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml | 0 .../LWM2M_Connectivity_Statistics-v1_0_1.xml | 0 .../lwm2m_xml/LWM2M_Device-v1_0_1.xml | 0 .../LWM2M_Firmware_Update-v1_0_1.xml | 0 .../lwm2m_xml/LWM2M_Location-v1_0.xml | 0 .../lwm2m_xml/LWM2M_Security-v1_0.xml | 0 .../lwm2m_xml/LWM2M_Server-v1_0.xml | 0 .../rebar.config | 0 .../src/binary_util.erl | 0 .../src/emqx_gateway_lwm2m.app.src} | 4 ++-- .../src/emqx_gateway_lwm2m.erl} | 2 +- .../src/emqx_lwm2m_api.erl | 0 .../src/emqx_lwm2m_channel.erl | 2 +- .../src/emqx_lwm2m_cmd.erl | 2 +- .../src/emqx_lwm2m_message.erl | 0 .../src/emqx_lwm2m_schema.erl | 0 .../src/emqx_lwm2m_session.erl | 2 +- .../src/emqx_lwm2m_tlv.erl | 0 .../src/emqx_lwm2m_xml_object.erl | 0 .../src/emqx_lwm2m_xml_object_db.erl | 0 .../test/emqx_lwm2m_SUITE.erl | 2 +- .../test/emqx_lwm2m_api_SUITE.erl | 2 +- .../test/emqx_tlv_SUITE.erl | 2 +- .../.gitignore | 0 .../README.md | 0 .../include/emqx_mqttsn.hrl | 0 .../rebar.config | 0 .../src/emqx_gateway_mqttsn.app.src} | 2 +- .../src/emqx_gateway_mqttsn.erl} | 2 +- .../src/emqx_mqttsn_broadcast.erl | 0 .../src/emqx_mqttsn_channel.erl | 0 .../src/emqx_mqttsn_frame.erl | 0 .../src/emqx_mqttsn_registry.erl | 0 .../src/emqx_mqttsn_schema.erl | 0 .../test/broadcast_test.py | 0 .../test/emqx_sn_frame_SUITE.erl | 0 .../test/emqx_sn_protocol_SUITE.erl | 0 .../test/emqx_sn_registry_SUITE.erl | 0 .../test/intergration_test/Makefile | 0 .../test/intergration_test/README.md | 0 .../add_emqx_sn_to_project.py | 0 .../intergration_test/client/case1_qos0pub.c | 0 .../intergration_test/client/case1_qos0sub.c | 0 .../intergration_test/client/case2_qos0pub.c | 0 .../intergration_test/client/case2_qos0sub.c | 0 .../intergration_test/client/case3_qos0pub.c | 0 .../intergration_test/client/case3_qos0sub.c | 0 .../intergration_test/client/case4_qos3pub.c | 0 .../intergration_test/client/case4_qos3sub.c | 0 .../intergration_test/client/case5_qos3pub.c | 0 .../intergration_test/client/case5_qos3sub.c | 0 .../intergration_test/client/case6_sleep.c | 0 .../client/case7_double_connect.c | 0 .../client/int_test_result.c | 0 .../client/int_test_result.h | 0 .../test/intergration_test/disable_qos3.py | 0 .../test/intergration_test/enable_qos3.py | 0 .../test/props/emqx_sn_proper_types.erl | 0 .../test/props/prop_emqx_sn_frame.erl | 0 .../.gitignore | 0 .../README.md | 0 .../include/emqx_stomp.hrl | 0 .../rebar.config | 0 .../src/emqx_gateway_stomp.app.src} | 2 +- .../src/emqx_gateway_stomp.erl} | 2 +- .../src/emqx_stomp_channel.erl | 0 .../src/emqx_stomp_frame.erl | 0 .../src/emqx_stomp_heartbeat.erl | 0 .../src/emqx_stomp_schema.erl | 0 .../test/emqx_stomp_SUITE.erl | 0 .../test/emqx_stomp_heartbeat_SUITE.erl | 0 mix.exs | 20 +++++++++--------- rebar.config.erl | 12 +++++------ 113 files changed, 33 insertions(+), 33 deletions(-) rename apps/{emqx_coap => emqx_gateway_coap}/.gitignore (100%) rename apps/{emqx_coap => emqx_gateway_coap}/README.md (100%) rename apps/{emqx_coap => emqx_gateway_coap}/doc/flow.png (100%) rename apps/{emqx_coap => emqx_gateway_coap}/doc/shared_state.png (100%) rename apps/{emqx_coap => emqx_gateway_coap}/doc/transport.png (100%) rename apps/{emqx_coap => emqx_gateway_coap}/include/emqx_coap.hrl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/rebar.config (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_api.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_channel.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_frame.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_medium.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_message.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_mqtt_handler.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_observe_res.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_pubsub_handler.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_schema.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_session.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_tm.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/src/emqx_coap_transport.erl (100%) rename apps/{emqx_coap/src/emqx_coap.app.src => emqx_gateway_coap/src/emqx_gateway_coap.app.src} (86%) rename apps/{emqx_coap/src/emqx_coap.erl => emqx_gateway_coap/src/emqx_gateway_coap.erl} (99%) rename apps/{emqx_coap => emqx_gateway_coap}/test/emqx_coap_SUITE.erl (100%) rename apps/{emqx_coap => emqx_gateway_coap}/test/emqx_coap_api_SUITE.erl (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/.gitignore (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/README.md (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/include/emqx_exproto.hrl (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/priv/protos/exproto.proto (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/rebar.config (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/src/emqx_exproto_channel.erl (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/src/emqx_exproto_frame.erl (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/src/emqx_exproto_gcli.erl (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/src/emqx_exproto_gsvr.erl (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/src/emqx_exproto_schema.erl (100%) rename apps/{emqx_exproto/src/emqx_exproto.app.src => emqx_gateway_exproto/src/emqx_gateway_exproto.app.src} (85%) rename apps/{emqx_exproto/src/emqx_exproto.erl => emqx_gateway_exproto/src/emqx_gateway_exproto.erl} (99%) rename apps/{emqx_exproto => emqx_gateway_exproto}/test/emqx_exproto_SUITE.erl (100%) rename apps/{emqx_exproto => emqx_gateway_exproto}/test/emqx_exproto_echo_svr.erl (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/.gitignore (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/README.md (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/include/emqx_lwm2m.hrl (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/lwm2m_xml/LWM2M_Device-v1_0_1.xml (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/lwm2m_xml/LWM2M_Location-v1_0.xml (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/lwm2m_xml/LWM2M_Security-v1_0.xml (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/lwm2m_xml/LWM2M_Server-v1_0.xml (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/rebar.config (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/src/binary_util.erl (100%) rename apps/{emqx_lwm2m/src/emqx_lwm2m.app.src => emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src} (59%) rename apps/{emqx_lwm2m/src/emqx_lwm2m.erl => emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.erl} (99%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/src/emqx_lwm2m_api.erl (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/src/emqx_lwm2m_channel.erl (99%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/src/emqx_lwm2m_cmd.erl (99%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/src/emqx_lwm2m_message.erl (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/src/emqx_lwm2m_schema.erl (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/src/emqx_lwm2m_session.erl (99%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/src/emqx_lwm2m_tlv.erl (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/src/emqx_lwm2m_xml_object.erl (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/src/emqx_lwm2m_xml_object_db.erl (100%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/test/emqx_lwm2m_SUITE.erl (99%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/test/emqx_lwm2m_api_SUITE.erl (99%) rename apps/{emqx_lwm2m => emqx_gateway_lwm2m}/test/emqx_tlv_SUITE.erl (99%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/.gitignore (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/README.md (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/include/emqx_mqttsn.hrl (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/rebar.config (100%) rename apps/{emqx_mqttsn/src/emqx_mqttsn.app.src => emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src} (85%) rename apps/{emqx_mqttsn/src/emqx_mqttsn.erl => emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.erl} (99%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/src/emqx_mqttsn_broadcast.erl (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/src/emqx_mqttsn_channel.erl (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/src/emqx_mqttsn_frame.erl (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/src/emqx_mqttsn_registry.erl (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/src/emqx_mqttsn_schema.erl (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/broadcast_test.py (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/emqx_sn_frame_SUITE.erl (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/emqx_sn_protocol_SUITE.erl (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/emqx_sn_registry_SUITE.erl (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/Makefile (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/README.md (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/add_emqx_sn_to_project.py (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case1_qos0pub.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case1_qos0sub.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case2_qos0pub.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case2_qos0sub.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case3_qos0pub.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case3_qos0sub.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case4_qos3pub.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case4_qos3sub.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case5_qos3pub.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case5_qos3sub.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case6_sleep.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/case7_double_connect.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/int_test_result.c (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/client/int_test_result.h (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/disable_qos3.py (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/intergration_test/enable_qos3.py (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/props/emqx_sn_proper_types.erl (100%) rename apps/{emqx_mqttsn => emqx_gateway_mqttsn}/test/props/prop_emqx_sn_frame.erl (100%) rename apps/{emqx_stomp => emqx_gateway_stomp}/.gitignore (100%) rename apps/{emqx_stomp => emqx_gateway_stomp}/README.md (100%) rename apps/{emqx_stomp => emqx_gateway_stomp}/include/emqx_stomp.hrl (100%) rename apps/{emqx_stomp => emqx_gateway_stomp}/rebar.config (100%) rename apps/{emqx_stomp/src/emqx_stomp.app.src => emqx_gateway_stomp/src/emqx_gateway_stomp.app.src} (86%) rename apps/{emqx_stomp/src/emqx_stomp.erl => emqx_gateway_stomp/src/emqx_gateway_stomp.erl} (99%) rename apps/{emqx_stomp => emqx_gateway_stomp}/src/emqx_stomp_channel.erl (100%) rename apps/{emqx_stomp => emqx_gateway_stomp}/src/emqx_stomp_frame.erl (100%) rename apps/{emqx_stomp => emqx_gateway_stomp}/src/emqx_stomp_heartbeat.erl (100%) rename apps/{emqx_stomp => emqx_gateway_stomp}/src/emqx_stomp_schema.erl (100%) rename apps/{emqx_stomp => emqx_gateway_stomp}/test/emqx_stomp_SUITE.erl (100%) rename apps/{emqx_stomp => emqx_gateway_stomp}/test/emqx_stomp_heartbeat_SUITE.erl (100%) diff --git a/apps/emqx_coap/.gitignore b/apps/emqx_gateway_coap/.gitignore similarity index 100% rename from apps/emqx_coap/.gitignore rename to apps/emqx_gateway_coap/.gitignore diff --git a/apps/emqx_coap/README.md b/apps/emqx_gateway_coap/README.md similarity index 100% rename from apps/emqx_coap/README.md rename to apps/emqx_gateway_coap/README.md diff --git a/apps/emqx_coap/doc/flow.png b/apps/emqx_gateway_coap/doc/flow.png similarity index 100% rename from apps/emqx_coap/doc/flow.png rename to apps/emqx_gateway_coap/doc/flow.png diff --git a/apps/emqx_coap/doc/shared_state.png b/apps/emqx_gateway_coap/doc/shared_state.png similarity index 100% rename from apps/emqx_coap/doc/shared_state.png rename to apps/emqx_gateway_coap/doc/shared_state.png diff --git a/apps/emqx_coap/doc/transport.png b/apps/emqx_gateway_coap/doc/transport.png similarity index 100% rename from apps/emqx_coap/doc/transport.png rename to apps/emqx_gateway_coap/doc/transport.png diff --git a/apps/emqx_coap/include/emqx_coap.hrl b/apps/emqx_gateway_coap/include/emqx_coap.hrl similarity index 100% rename from apps/emqx_coap/include/emqx_coap.hrl rename to apps/emqx_gateway_coap/include/emqx_coap.hrl diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_gateway_coap/rebar.config similarity index 100% rename from apps/emqx_coap/rebar.config rename to apps/emqx_gateway_coap/rebar.config diff --git a/apps/emqx_coap/src/emqx_coap_api.erl b/apps/emqx_gateway_coap/src/emqx_coap_api.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_api.erl rename to apps/emqx_gateway_coap/src/emqx_coap_api.erl diff --git a/apps/emqx_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_channel.erl rename to apps/emqx_gateway_coap/src/emqx_coap_channel.erl diff --git a/apps/emqx_coap/src/emqx_coap_frame.erl b/apps/emqx_gateway_coap/src/emqx_coap_frame.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_frame.erl rename to apps/emqx_gateway_coap/src/emqx_coap_frame.erl diff --git a/apps/emqx_coap/src/emqx_coap_medium.erl b/apps/emqx_gateway_coap/src/emqx_coap_medium.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_medium.erl rename to apps/emqx_gateway_coap/src/emqx_coap_medium.erl diff --git a/apps/emqx_coap/src/emqx_coap_message.erl b/apps/emqx_gateway_coap/src/emqx_coap_message.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_message.erl rename to apps/emqx_gateway_coap/src/emqx_coap_message.erl diff --git a/apps/emqx_coap/src/emqx_coap_mqtt_handler.erl b/apps/emqx_gateway_coap/src/emqx_coap_mqtt_handler.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_mqtt_handler.erl rename to apps/emqx_gateway_coap/src/emqx_coap_mqtt_handler.erl diff --git a/apps/emqx_coap/src/emqx_coap_observe_res.erl b/apps/emqx_gateway_coap/src/emqx_coap_observe_res.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_observe_res.erl rename to apps/emqx_gateway_coap/src/emqx_coap_observe_res.erl diff --git a/apps/emqx_coap/src/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_pubsub_handler.erl rename to apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl diff --git a/apps/emqx_coap/src/emqx_coap_schema.erl b/apps/emqx_gateway_coap/src/emqx_coap_schema.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_schema.erl rename to apps/emqx_gateway_coap/src/emqx_coap_schema.erl diff --git a/apps/emqx_coap/src/emqx_coap_session.erl b/apps/emqx_gateway_coap/src/emqx_coap_session.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_session.erl rename to apps/emqx_gateway_coap/src/emqx_coap_session.erl diff --git a/apps/emqx_coap/src/emqx_coap_tm.erl b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_tm.erl rename to apps/emqx_gateway_coap/src/emqx_coap_tm.erl diff --git a/apps/emqx_coap/src/emqx_coap_transport.erl b/apps/emqx_gateway_coap/src/emqx_coap_transport.erl similarity index 100% rename from apps/emqx_coap/src/emqx_coap_transport.erl rename to apps/emqx_gateway_coap/src/emqx_coap_transport.erl diff --git a/apps/emqx_coap/src/emqx_coap.app.src b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src similarity index 86% rename from apps/emqx_coap/src/emqx_coap.app.src rename to apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src index c0f3f23da..decd13bef 100644 --- a/apps/emqx_coap/src/emqx_coap.app.src +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src @@ -1,4 +1,4 @@ -{application, emqx_coap, [ +{application, emqx_gateway_coap, [ {description, "CoAP Gateway"}, {vsn, "0.1.0"}, {registered, []}, diff --git a/apps/emqx_coap/src/emqx_coap.erl b/apps/emqx_gateway_coap/src/emqx_gateway_coap.erl similarity index 99% rename from apps/emqx_coap/src/emqx_coap.erl rename to apps/emqx_gateway_coap/src/emqx_gateway_coap.erl index d553349a4..6c495fbdb 100644 --- a/apps/emqx_coap/src/emqx_coap.erl +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- %% @doc The CoAP Gateway implement --module(emqx_coap). +-module(emqx_gateway_coap). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). diff --git a/apps/emqx_coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl similarity index 100% rename from apps/emqx_coap/test/emqx_coap_SUITE.erl rename to apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl diff --git a/apps/emqx_coap/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl similarity index 100% rename from apps/emqx_coap/test/emqx_coap_api_SUITE.erl rename to apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl diff --git a/apps/emqx_exproto/.gitignore b/apps/emqx_gateway_exproto/.gitignore similarity index 100% rename from apps/emqx_exproto/.gitignore rename to apps/emqx_gateway_exproto/.gitignore diff --git a/apps/emqx_exproto/README.md b/apps/emqx_gateway_exproto/README.md similarity index 100% rename from apps/emqx_exproto/README.md rename to apps/emqx_gateway_exproto/README.md diff --git a/apps/emqx_exproto/include/emqx_exproto.hrl b/apps/emqx_gateway_exproto/include/emqx_exproto.hrl similarity index 100% rename from apps/emqx_exproto/include/emqx_exproto.hrl rename to apps/emqx_gateway_exproto/include/emqx_exproto.hrl diff --git a/apps/emqx_exproto/priv/protos/exproto.proto b/apps/emqx_gateway_exproto/priv/protos/exproto.proto similarity index 100% rename from apps/emqx_exproto/priv/protos/exproto.proto rename to apps/emqx_gateway_exproto/priv/protos/exproto.proto diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_gateway_exproto/rebar.config similarity index 100% rename from apps/emqx_exproto/rebar.config rename to apps/emqx_gateway_exproto/rebar.config diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl similarity index 100% rename from apps/emqx_exproto/src/emqx_exproto_channel.erl rename to apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl diff --git a/apps/emqx_exproto/src/emqx_exproto_frame.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_frame.erl similarity index 100% rename from apps/emqx_exproto/src/emqx_exproto_frame.erl rename to apps/emqx_gateway_exproto/src/emqx_exproto_frame.erl diff --git a/apps/emqx_exproto/src/emqx_exproto_gcli.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl similarity index 100% rename from apps/emqx_exproto/src/emqx_exproto_gcli.erl rename to apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl diff --git a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_gsvr.erl similarity index 100% rename from apps/emqx_exproto/src/emqx_exproto_gsvr.erl rename to apps/emqx_gateway_exproto/src/emqx_exproto_gsvr.erl diff --git a/apps/emqx_exproto/src/emqx_exproto_schema.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_schema.erl similarity index 100% rename from apps/emqx_exproto/src/emqx_exproto_schema.erl rename to apps/emqx_gateway_exproto/src/emqx_exproto_schema.erl diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src similarity index 85% rename from apps/emqx_exproto/src/emqx_exproto.app.src rename to apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src index aa586a4fd..09cf58338 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src @@ -1,4 +1,4 @@ -{application, emqx_exproto, [ +{application, emqx_gateway_exproto, [ {description, "ExProto Gateway"}, {vsn, "0.1.0"}, {registered, []}, diff --git a/apps/emqx_exproto/src/emqx_exproto.erl b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl similarity index 99% rename from apps/emqx_exproto/src/emqx_exproto.erl rename to apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl index 1e6e0e6de..7860f1220 100644 --- a/apps/emqx_exproto/src/emqx_exproto.erl +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- %% @doc The ExProto Gateway implement --module(emqx_exproto). +-module(emqx_gateway_exproto). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). diff --git a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl similarity index 100% rename from apps/emqx_exproto/test/emqx_exproto_SUITE.erl rename to apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl diff --git a/apps/emqx_exproto/test/emqx_exproto_echo_svr.erl b/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl similarity index 100% rename from apps/emqx_exproto/test/emqx_exproto_echo_svr.erl rename to apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl diff --git a/apps/emqx_lwm2m/.gitignore b/apps/emqx_gateway_lwm2m/.gitignore similarity index 100% rename from apps/emqx_lwm2m/.gitignore rename to apps/emqx_gateway_lwm2m/.gitignore diff --git a/apps/emqx_lwm2m/README.md b/apps/emqx_gateway_lwm2m/README.md similarity index 100% rename from apps/emqx_lwm2m/README.md rename to apps/emqx_gateway_lwm2m/README.md diff --git a/apps/emqx_lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_gateway_lwm2m/include/emqx_lwm2m.hrl similarity index 100% rename from apps/emqx_lwm2m/include/emqx_lwm2m.hrl rename to apps/emqx_gateway_lwm2m/include/emqx_lwm2m.hrl diff --git a/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml similarity index 100% rename from apps/emqx_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml diff --git a/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml similarity index 100% rename from apps/emqx_lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Connectivity_Statistics-v1_0_1.xml diff --git a/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml similarity index 100% rename from apps/emqx_lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Device-v1_0_1.xml diff --git a/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml similarity index 100% rename from apps/emqx_lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Firmware_Update-v1_0_1.xml diff --git a/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml similarity index 100% rename from apps/emqx_lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Location-v1_0.xml diff --git a/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml similarity index 100% rename from apps/emqx_lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Security-v1_0.xml diff --git a/apps/emqx_lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml b/apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml similarity index 100% rename from apps/emqx_lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml rename to apps/emqx_gateway_lwm2m/lwm2m_xml/LWM2M_Server-v1_0.xml diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_gateway_lwm2m/rebar.config similarity index 100% rename from apps/emqx_lwm2m/rebar.config rename to apps/emqx_gateway_lwm2m/rebar.config diff --git a/apps/emqx_lwm2m/src/binary_util.erl b/apps/emqx_gateway_lwm2m/src/binary_util.erl similarity index 100% rename from apps/emqx_lwm2m/src/binary_util.erl rename to apps/emqx_gateway_lwm2m/src/binary_util.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src similarity index 59% rename from apps/emqx_lwm2m/src/emqx_lwm2m.app.src rename to apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src index 6338fa9d3..83a707395 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src +++ b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src @@ -1,8 +1,8 @@ -{application, emqx_lwm2m, [ +{application, emqx_gateway_lwm2m, [ {description, "LwM2M Gateway"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_coap]}, + {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.erl b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.erl similarity index 99% rename from apps/emqx_lwm2m/src/emqx_lwm2m.erl rename to apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.erl index 222d1076e..1c8f67863 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- %% @doc The LwM2M Gateway implement --module(emqx_lwm2m). +-module(emqx_gateway_lwm2m). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl similarity index 100% rename from apps/emqx_lwm2m/src/emqx_lwm2m_api.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl similarity index 99% rename from apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl index 276b4f19d..a424375dd 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl @@ -18,7 +18,7 @@ -include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_coap/include/emqx_coap.hrl"). +-include_lib("emqx_gateway_coap/include/emqx_coap.hrl"). %% API -export([ diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl similarity index 99% rename from apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl index 9ef3fb10d..53995c97c 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl @@ -18,7 +18,7 @@ -include("emqx_lwm2m.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx_coap/include/emqx_coap.hrl"). +-include_lib("emqx_gateway_coap/include/emqx_coap.hrl"). -export([ mqtt_to_coap/2, diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl similarity index 100% rename from apps/emqx_lwm2m/src/emqx_lwm2m_message.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_schema.erl similarity index 100% rename from apps/emqx_lwm2m/src/emqx_lwm2m_schema.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_schema.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl similarity index 99% rename from apps/emqx_lwm2m/src/emqx_lwm2m_session.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl index 6c8b419ee..2752804dc 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl @@ -20,7 +20,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --include_lib("emqx_coap/include/emqx_coap.hrl"). +-include_lib("emqx_gateway_coap/include/emqx_coap.hrl"). %% API -export([ diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_tlv.erl similarity index 100% rename from apps/emqx_lwm2m/src/emqx_lwm2m_tlv.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_tlv.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object.erl similarity index 100% rename from apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object.erl diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object_db.erl similarity index 100% rename from apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl rename to apps/emqx_gateway_lwm2m/src/emqx_lwm2m_xml_object_db.erl diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl similarity index 99% rename from apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl rename to apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl index dd2e3bbfd..b090faba7 100644 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -32,7 +32,7 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). -include("emqx_lwm2m.hrl"). --include_lib("emqx_coap/include/emqx_coap.hrl"). +-include_lib("emqx_gateway_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl similarity index 99% rename from apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl rename to apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl index a1d048d76..8608b543d 100644 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl @@ -24,7 +24,7 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). -include("emqx_lwm2m.hrl"). --include("emqx_coap/include/emqx_coap.hrl"). +-include("emqx_gateway_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_tlv_SUITE.erl similarity index 99% rename from apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl rename to apps/emqx_gateway_lwm2m/test/emqx_tlv_SUITE.erl index da1e3a9c4..c413469ea 100644 --- a/apps/emqx_lwm2m/test/emqx_tlv_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_tlv_SUITE.erl @@ -22,7 +22,7 @@ -define(LOGT(Format, Args), logger:debug("TEST_SUITE: " ++ Format, Args)). -include("emqx_lwm2m.hrl"). --include("emqx_coap/include/emqx_coap.hrl"). +-include("emqx_gateway_coap/include/emqx_coap.hrl"). -include_lib("eunit/include/eunit.hrl"). %%-------------------------------------------------------------------- diff --git a/apps/emqx_mqttsn/.gitignore b/apps/emqx_gateway_mqttsn/.gitignore similarity index 100% rename from apps/emqx_mqttsn/.gitignore rename to apps/emqx_gateway_mqttsn/.gitignore diff --git a/apps/emqx_mqttsn/README.md b/apps/emqx_gateway_mqttsn/README.md similarity index 100% rename from apps/emqx_mqttsn/README.md rename to apps/emqx_gateway_mqttsn/README.md diff --git a/apps/emqx_mqttsn/include/emqx_mqttsn.hrl b/apps/emqx_gateway_mqttsn/include/emqx_mqttsn.hrl similarity index 100% rename from apps/emqx_mqttsn/include/emqx_mqttsn.hrl rename to apps/emqx_gateway_mqttsn/include/emqx_mqttsn.hrl diff --git a/apps/emqx_mqttsn/rebar.config b/apps/emqx_gateway_mqttsn/rebar.config similarity index 100% rename from apps/emqx_mqttsn/rebar.config rename to apps/emqx_gateway_mqttsn/rebar.config diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src similarity index 85% rename from apps/emqx_mqttsn/src/emqx_mqttsn.app.src rename to apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src index 55e18e800..dd48b2723 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn.app.src +++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src @@ -1,4 +1,4 @@ -{application, emqx_mqttsn, [ +{application, emqx_gateway_mqttsn, [ {description, "MQTT-SN Gateway"}, {vsn, "0.1.0"}, {registered, []}, diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn.erl b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.erl similarity index 99% rename from apps/emqx_mqttsn/src/emqx_mqttsn.erl rename to apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.erl index 5d6a94df4..167ee465c 100644 --- a/apps/emqx_mqttsn/src/emqx_mqttsn.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- %% @doc The MQTT-SN Gateway implement interface --module(emqx_mqttsn). +-module(emqx_gateway_mqttsn). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_broadcast.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_broadcast.erl similarity index 100% rename from apps/emqx_mqttsn/src/emqx_mqttsn_broadcast.erl rename to apps/emqx_gateway_mqttsn/src/emqx_mqttsn_broadcast.erl diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl similarity index 100% rename from apps/emqx_mqttsn/src/emqx_mqttsn_channel.erl rename to apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_frame.erl similarity index 100% rename from apps/emqx_mqttsn/src/emqx_mqttsn_frame.erl rename to apps/emqx_gateway_mqttsn/src/emqx_mqttsn_frame.erl diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_registry.erl similarity index 100% rename from apps/emqx_mqttsn/src/emqx_mqttsn_registry.erl rename to apps/emqx_gateway_mqttsn/src/emqx_mqttsn_registry.erl diff --git a/apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_schema.erl similarity index 100% rename from apps/emqx_mqttsn/src/emqx_mqttsn_schema.erl rename to apps/emqx_gateway_mqttsn/src/emqx_mqttsn_schema.erl diff --git a/apps/emqx_mqttsn/test/broadcast_test.py b/apps/emqx_gateway_mqttsn/test/broadcast_test.py similarity index 100% rename from apps/emqx_mqttsn/test/broadcast_test.py rename to apps/emqx_gateway_mqttsn/test/broadcast_test.py diff --git a/apps/emqx_mqttsn/test/emqx_sn_frame_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_frame_SUITE.erl similarity index 100% rename from apps/emqx_mqttsn/test/emqx_sn_frame_SUITE.erl rename to apps/emqx_gateway_mqttsn/test/emqx_sn_frame_SUITE.erl diff --git a/apps/emqx_mqttsn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl similarity index 100% rename from apps/emqx_mqttsn/test/emqx_sn_protocol_SUITE.erl rename to apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl diff --git a/apps/emqx_mqttsn/test/emqx_sn_registry_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_registry_SUITE.erl similarity index 100% rename from apps/emqx_mqttsn/test/emqx_sn_registry_SUITE.erl rename to apps/emqx_gateway_mqttsn/test/emqx_sn_registry_SUITE.erl diff --git a/apps/emqx_mqttsn/test/intergration_test/Makefile b/apps/emqx_gateway_mqttsn/test/intergration_test/Makefile similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/Makefile rename to apps/emqx_gateway_mqttsn/test/intergration_test/Makefile diff --git a/apps/emqx_mqttsn/test/intergration_test/README.md b/apps/emqx_gateway_mqttsn/test/intergration_test/README.md similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/README.md rename to apps/emqx_gateway_mqttsn/test/intergration_test/README.md diff --git a/apps/emqx_mqttsn/test/intergration_test/add_emqx_sn_to_project.py b/apps/emqx_gateway_mqttsn/test/intergration_test/add_emqx_sn_to_project.py similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/add_emqx_sn_to_project.py rename to apps/emqx_gateway_mqttsn/test/intergration_test/add_emqx_sn_to_project.py diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case1_qos0pub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case1_qos0pub.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case1_qos0pub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case1_qos0pub.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case1_qos0sub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case1_qos0sub.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case1_qos0sub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case1_qos0sub.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case2_qos0pub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case2_qos0pub.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case2_qos0pub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case2_qos0pub.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case2_qos0sub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case2_qos0sub.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case2_qos0sub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case2_qos0sub.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case3_qos0pub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case3_qos0pub.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case3_qos0pub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case3_qos0pub.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case3_qos0sub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case3_qos0sub.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case3_qos0sub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case3_qos0sub.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case4_qos3pub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case4_qos3pub.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case4_qos3pub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case4_qos3pub.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case4_qos3sub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case4_qos3sub.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case4_qos3sub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case4_qos3sub.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case5_qos3pub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case5_qos3pub.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case5_qos3pub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case5_qos3pub.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case5_qos3sub.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case5_qos3sub.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case5_qos3sub.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case5_qos3sub.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case6_sleep.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case6_sleep.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case6_sleep.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case6_sleep.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/case7_double_connect.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/case7_double_connect.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/case7_double_connect.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/case7_double_connect.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/int_test_result.c b/apps/emqx_gateway_mqttsn/test/intergration_test/client/int_test_result.c similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/int_test_result.c rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/int_test_result.c diff --git a/apps/emqx_mqttsn/test/intergration_test/client/int_test_result.h b/apps/emqx_gateway_mqttsn/test/intergration_test/client/int_test_result.h similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/client/int_test_result.h rename to apps/emqx_gateway_mqttsn/test/intergration_test/client/int_test_result.h diff --git a/apps/emqx_mqttsn/test/intergration_test/disable_qos3.py b/apps/emqx_gateway_mqttsn/test/intergration_test/disable_qos3.py similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/disable_qos3.py rename to apps/emqx_gateway_mqttsn/test/intergration_test/disable_qos3.py diff --git a/apps/emqx_mqttsn/test/intergration_test/enable_qos3.py b/apps/emqx_gateway_mqttsn/test/intergration_test/enable_qos3.py similarity index 100% rename from apps/emqx_mqttsn/test/intergration_test/enable_qos3.py rename to apps/emqx_gateway_mqttsn/test/intergration_test/enable_qos3.py diff --git a/apps/emqx_mqttsn/test/props/emqx_sn_proper_types.erl b/apps/emqx_gateway_mqttsn/test/props/emqx_sn_proper_types.erl similarity index 100% rename from apps/emqx_mqttsn/test/props/emqx_sn_proper_types.erl rename to apps/emqx_gateway_mqttsn/test/props/emqx_sn_proper_types.erl diff --git a/apps/emqx_mqttsn/test/props/prop_emqx_sn_frame.erl b/apps/emqx_gateway_mqttsn/test/props/prop_emqx_sn_frame.erl similarity index 100% rename from apps/emqx_mqttsn/test/props/prop_emqx_sn_frame.erl rename to apps/emqx_gateway_mqttsn/test/props/prop_emqx_sn_frame.erl diff --git a/apps/emqx_stomp/.gitignore b/apps/emqx_gateway_stomp/.gitignore similarity index 100% rename from apps/emqx_stomp/.gitignore rename to apps/emqx_gateway_stomp/.gitignore diff --git a/apps/emqx_stomp/README.md b/apps/emqx_gateway_stomp/README.md similarity index 100% rename from apps/emqx_stomp/README.md rename to apps/emqx_gateway_stomp/README.md diff --git a/apps/emqx_stomp/include/emqx_stomp.hrl b/apps/emqx_gateway_stomp/include/emqx_stomp.hrl similarity index 100% rename from apps/emqx_stomp/include/emqx_stomp.hrl rename to apps/emqx_gateway_stomp/include/emqx_stomp.hrl diff --git a/apps/emqx_stomp/rebar.config b/apps/emqx_gateway_stomp/rebar.config similarity index 100% rename from apps/emqx_stomp/rebar.config rename to apps/emqx_gateway_stomp/rebar.config diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src similarity index 86% rename from apps/emqx_stomp/src/emqx_stomp.app.src rename to apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src index e118f8370..38da1e18b 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src @@ -1,4 +1,4 @@ -{application, emqx_stomp, [ +{application, emqx_gateway_stomp, [ {description, "Stomp Gateway"}, {vsn, "0.1.0"}, {registered, []}, diff --git a/apps/emqx_stomp/src/emqx_stomp.erl b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.erl similarity index 99% rename from apps/emqx_stomp/src/emqx_stomp.erl rename to apps/emqx_gateway_stomp/src/emqx_gateway_stomp.erl index dbfdfdce5..b8c2f0166 100644 --- a/apps/emqx_stomp/src/emqx_stomp.erl +++ b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- %% @doc The Stomp Gateway implement --module(emqx_stomp). +-module(emqx_gateway_stomp). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_gateway/include/emqx_gateway.hrl"). diff --git a/apps/emqx_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl similarity index 100% rename from apps/emqx_stomp/src/emqx_stomp_channel.erl rename to apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl diff --git a/apps/emqx_stomp/src/emqx_stomp_frame.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_frame.erl similarity index 100% rename from apps/emqx_stomp/src/emqx_stomp_frame.erl rename to apps/emqx_gateway_stomp/src/emqx_stomp_frame.erl diff --git a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_heartbeat.erl similarity index 100% rename from apps/emqx_stomp/src/emqx_stomp_heartbeat.erl rename to apps/emqx_gateway_stomp/src/emqx_stomp_heartbeat.erl diff --git a/apps/emqx_stomp/src/emqx_stomp_schema.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_schema.erl similarity index 100% rename from apps/emqx_stomp/src/emqx_stomp_schema.erl rename to apps/emqx_gateway_stomp/src/emqx_stomp_schema.erl diff --git a/apps/emqx_stomp/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl similarity index 100% rename from apps/emqx_stomp/test/emqx_stomp_SUITE.erl rename to apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl diff --git a/apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl b/apps/emqx_gateway_stomp/test/emqx_stomp_heartbeat_SUITE.erl similarity index 100% rename from apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl rename to apps/emqx_gateway_stomp/test/emqx_stomp_heartbeat_SUITE.erl diff --git a/mix.exs b/mix.exs index 56840579d..63e2de03f 100644 --- a/mix.exs +++ b/mix.exs @@ -223,11 +223,11 @@ defmodule EMQXUmbrella.MixProject do applications: applications(edition_type), skip_mode_validation_for: [ :emqx_gateway, - :emqx_stomp, - :emqx_mqttsn, - :emqx_coap, - :emqx_lwm2m, - :emqx_exproto, + :emqx_gateway_stomp, + :emqx_gateway_mqttsn, + :emqx_gateway_coap, + :emqx_gateway_lwm2m, + :emqx_gateway_exproto, :emqx_dashboard, :emqx_resource, :emqx_connector, @@ -288,11 +288,11 @@ defmodule EMQXUmbrella.MixProject do emqx_authz: :permanent, emqx_auto_subscribe: :permanent, emqx_gateway: :permanent, - emqx_stomp: :permanent, - emqx_mqttsn: :permanent, - emqx_coap: :permanent, - emqx_lwm2m: :permanent, - emqx_exproto: :permanent, + emqx_gateway_stomp: :permanent, + emqx_gateway_mqttsn: :permanent, + emqx_gateway_coap: :permanent, + emqx_gateway_lwm2m: :permanent, + emqx_gateway_exproto: :permanent, emqx_exhook: :permanent, emqx_bridge: :permanent, emqx_rule_engine: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index cdd628664..8f0cf4292 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -389,11 +389,11 @@ relx_apps(ReleaseType, Edition) -> emqx_authz, emqx_auto_subscribe, emqx_gateway, - emqx_stomp, - emqx_mqttsn, - emqx_coap, - emqx_lwm2m, - emqx_exproto, + emqx_gateway_stomp, + emqx_gateway_mqttsn, + emqx_gateway_coap, + emqx_gateway_lwm2m, + emqx_gateway_exproto, emqx_exhook, emqx_bridge, emqx_rule_engine, @@ -455,7 +455,7 @@ relx_overlay(ReleaseType, Edition) -> {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"}, %% for relup {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"}, - {copy, "apps/emqx_lwm2m/lwm2m_xml", "etc/lwm2m_xml"}, + {copy, "apps/emqx_gateway_lwm2m/lwm2m_xml", "etc/lwm2m_xml"}, {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"}, {template, "bin/emqx.cmd", "bin/emqx.cmd"}, {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}, From 0cec52d5e4da6f790916f09aa468e75f6408a674 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 4 Apr 2023 12:01:41 +0200 Subject: [PATCH 069/279] fix: check Clickhouse connection after creation When pressing reconnect on a badly configured Clickhouse bridge in the dashboard, no error message was shown. This commit fixes this issue by testing the connection after creation and returning an error tuple if the connection is not working. Fixes: https://emqx.atlassian.net/browse/EMQX-9374 --- changes/ee/fix-10324.en.md | 1 + changes/ee/fix-10324.zh.md | 1 + .../src/emqx_ee_connector_clickhouse.erl | 11 +++++++++-- 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 changes/ee/fix-10324.en.md create mode 100644 changes/ee/fix-10324.zh.md diff --git a/changes/ee/fix-10324.en.md b/changes/ee/fix-10324.en.md new file mode 100644 index 000000000..2d4c323da --- /dev/null +++ b/changes/ee/fix-10324.en.md @@ -0,0 +1 @@ +Previously, when attempting to reconnect to a misconfigured Clickhouse bridge through the dashboard, users would not receive an error message. This issue is now resolved, and error messages will now be displayed diff --git a/changes/ee/fix-10324.zh.md b/changes/ee/fix-10324.zh.md new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/changes/ee/fix-10324.zh.md @@ -0,0 +1 @@ + diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl index 8f2fdc042..5d68687dc 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl @@ -270,8 +270,15 @@ connect(Options) -> {pool_size, PoolSize} ], case clickhouse:start_link(FixedOptions) of - {ok, _Conn} = Ok -> - Ok; + {ok, Connection} -> + %% Check if we can connect and send a query + case clickhouse:detailed_status(Connection) of + ok -> + {ok, Connection}; + Error -> + ok = clickhouse:stop(Connection), + Error + end; {error, Reason} -> {error, Reason} end. From aa9292e37ecbba51170347e3d0f00a40221f538a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 13 Apr 2023 16:18:34 +0800 Subject: [PATCH 070/279] chore: update apps/emqx_gateway/README.md Co-authored-by: Stefan Strigler --- apps/emqx_gateway/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index d1f327db2..5dedea418 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -1,6 +1,6 @@ # Gateway -EMQX Gateway is an application that managing all gateways in EMQX. +EMQX Gateway is an application framework that manages all gateways within EMQX. It provides a set of standards to define how to implement a certain type of protocol access on EMQX. For example: From 40e4419a62d0a06cff19987aa3668a53c301c00a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 13 Apr 2023 16:18:46 +0800 Subject: [PATCH 071/279] chore: update apps/emqx_gateway/README.md Co-authored-by: Stefan Strigler --- apps/emqx_gateway/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index 5dedea418..8ef0fa30a 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -43,7 +43,7 @@ gateway.stomp { ## How to develop your Gateway application -There are three ways to develop Gateway application to accept your private protocol +There are three ways to develop a Gateway application to accept your private protocol clients. ### Raw Erlang Application From 4dd0080e821682dda28036cd65b68d8eda9e51d2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 13 Apr 2023 17:19:47 +0800 Subject: [PATCH 072/279] chore: fix unreachable links --- apps/emqx_gateway/README.md | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md index 8ef0fa30a..ebab3a7a9 100644 --- a/apps/emqx_gateway/README.md +++ b/apps/emqx_gateway/README.md @@ -49,13 +49,11 @@ clients. ### Raw Erlang Application This approach is the same as in EMQX 4.x. You need to implement an Erlang application, -which is packaged in EMQX as a [Plugin](todo) or as a source code dependency. +which is packaged in EMQX as a Plugin or as a source code dependency. In this approach, you do not need to respect any specifications of emqx_gateway, and you can freely implement the features you need. -Steps guide: [Implement Gateway via Raw Application](doc/implement_gateway_via_raw_appliction.md) - ### Respect emqx_gateway framework Similar to the first approach, you still need to implement an application using Erlang @@ -67,8 +65,6 @@ This is the approach we recommend. In this approach, your implementation can be by the emqx_gateway framework, even if it may require you to understand more details about it. -Steps guide: [Implement Gateway via Gateway framework](doc/implement_gateway_via_gateway_framekwork.md) - ### Use ExProto Gateway (Non-Erlang developers) If you want to implement your gateway using other programming languages such as @@ -77,11 +73,6 @@ Java, Python, Go, etc. You need to implement a gRPC service in the other programming language to parse your device protocol and integrate it with EMQX. -Refer to: [ExProto Gateway](../emqx_exproto/README.md) - -## Understand emqx_gateway framework - -*WIP* ## Contributing @@ -89,4 +80,4 @@ Please see our [contributing.md](../../CONTRIBUTING.md). ## License -See [LICENSE](../../LICENSE) +See [LICENSE](../../APL.txt) From c7054886b1e5ae569824688ab3198ef12593d17e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 13 Apr 2023 17:44:02 +0800 Subject: [PATCH 073/279] chore: refine test cases --- apps/emqx_gateway/test/emqx_gateway_test_utils.erl | 10 +++++----- apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl | 2 +- apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl | 2 +- apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl | 2 +- apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl | 4 ++-- apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl | 2 +- .../test/emqx_sn_protocol_SUITE.erl | 2 +- apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index 56a2fe7f9..7ec0c9538 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -102,11 +102,11 @@ assert_fields_exist(Ks, Map) -> Ks ). load_all_gateway_apps() -> - application:load(emqx_stomp), - application:load(emqx_mqttsn), - application:load(emqx_coap), - application:load(emqx_lwm2m), - application:load(emqx_exproto). + application:load(emqx_gateway_stomp), + application:load(emqx_gateway_mqttsn), + application:load(emqx_gateway_coap), + application:load(emqx_gateway_lwm2m), + application:load(emqx_gateway_exproto). %%-------------------------------------------------------------------- %% http diff --git a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl index 1d33e042a..9b6f7ce1f 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl @@ -56,7 +56,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - application:load(emqx_coap), + application:load(emqx_gateway_coap), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), diff --git a/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl index 9c418ab57..a0d6dbaaa 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl @@ -56,7 +56,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - application:load(emqx_coap), + application:load(emqx_gateway_coap), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), Config. diff --git a/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl index a8ce41f44..264f6af95 100644 --- a/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl @@ -76,7 +76,7 @@ metrics() -> [tcp, ssl, udp, dtls]. init_per_group(GrpName, Cfg) -> - application:load(emqx_exproto), + application:load(emqx_gateway_exproto), put(grpname, GrpName), Svrs = emqx_exproto_echo_svr:start(), emqx_common_test_helpers:start_apps([emqx_authn, emqx_gateway], fun set_special_cfg/1), diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl index b090faba7..33ccec2c7 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -134,7 +134,7 @@ groups() -> init_per_suite(Config) -> %% load application first for minirest api searching application:load(emqx_gateway), - application:load(emqx_lwm2m), + application:load(emqx_gateway_lwm2m), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]), Config. @@ -181,7 +181,7 @@ default_config(Overrides) -> [ emqx_common_test_helpers:proj_root(), "apps", - "emqx_lwm2m", + "emqx_gateway_lwm2m", "lwm2m_xml" ] ), diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl index 8608b543d..b3c059b31 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl @@ -59,7 +59,7 @@ all() -> init_per_suite(Config) -> application:load(emqx_gateway), - application:load(emqx_lwm2m), + application:load(emqx_gateway_lwm2m), DefaultConfig = emqx_lwm2m_SUITE:default_config(), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, DefaultConfig), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]), diff --git a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl index 0e04ec67a..04b1b5fb2 100644 --- a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl @@ -97,7 +97,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - application:load(emqx_mqttsn), + application:load(emqx_gateway_mqttsn), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), Config. diff --git a/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl index fed7f5163..4323cf32f 100644 --- a/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl @@ -53,7 +53,7 @@ all() -> emqx_common_test_helpers:all(?MODULE). %%-------------------------------------------------------------------- init_per_suite(Cfg) -> - application:load(emqx_stomp), + application:load(emqx_gateway_stomp), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_authn, emqx_gateway]), Cfg. From 993be31998615da3ef005809b347f48fae6bb4ee Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 13 Apr 2023 18:12:17 +0800 Subject: [PATCH 074/279] chore: update rel/i18n/emqx_conf_schema.hocon Co-authored-by: LenaLenaPan <120552185+LenaLenaPan@users.noreply.github.com> --- rel/i18n/emqx_conf_schema.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index e9f9c1211..b252353f8 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -1190,7 +1190,7 @@ To flush events, the handler discards the buffered log messages without logging. log_overload_kill_restart_after { desc { en: """The handler restarts automatically after a delay in the event of termination, unless the value `infinity` is set, which blocks any subsequent restarts.""" - zh: """处理进程停止后，会在该延迟时间后自动重新启动。除非该值设置为 infinity，这表示永不重新启动。""" + zh: """处理进程停止后，会在该延迟时间后自动重新启动。除非该值设置为 infinity，这会阻止任何后续的重启。""" } label { en: "Handler Restart Timer" From 3605e4209a59fee5771549477fea293576bdbc39 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 13 Apr 2023 13:22:13 +0200 Subject: [PATCH 075/279] ci(cut): Don't generate Chinese changelog --- scripts/rel/cut.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh index 71033035a..4a0a7ab6c 100755 --- a/scripts/rel/cut.sh +++ b/scripts/rel/cut.sh @@ -233,19 +233,19 @@ if [ -d "${CHECKS_DIR}" ]; then fi generate_changelog () { - local from_tag num_en num_zh + local from_tag from_tag="${PREV_TAG:-}" if [[ -z $from_tag ]]; then from_tag="$(./scripts/find-prev-rel-tag.sh "$PROFILE")" fi - num_en=$(git diff --name-only -a "${from_tag}...HEAD" "changes" | grep -c '.en.md') - num_zh=$(git diff --name-only -a "${from_tag}...HEAD" "changes" | grep -c '.zh.md') - if [ "$num_en" -ne "$num_zh" ]; then - echo "Number of English and Chinese changelog files added since ${from_tag} do not match." - exit 1 - fi + # num_en=$(git diff --name-only -a "${from_tag}...HEAD" "changes" | grep -c '.en.md') + # num_zh=$(git diff --name-only -a "${from_tag}...HEAD" "changes" | grep -c '.zh.md') + # if [ "$num_en" -ne "$num_zh" ]; then + # echo "Number of English and Chinese changelog files added since ${from_tag} do not match." + # exit 1 + # fi ./scripts/rel/format-changelog.sh -b "${from_tag}" -l 'en' -v "$TAG" > "changes/${TAG}.en.md" - ./scripts/rel/format-changelog.sh -b "${from_tag}" -l 'zh' -v "$TAG" > "changes/${TAG}.zh.md" + # ./scripts/rel/format-changelog.sh -b "${from_tag}" -l 'zh' -v "$TAG" > "changes/${TAG}.zh.md" git add changes/"${TAG}".*.md if [ -n "$(git diff --staged --stat)" ]; then git commit -m "docs: Generate changelog for ${TAG}" From d1089fa4603fddd7035a8487515fcabe496acdfa Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 13 Apr 2023 13:29:32 +0200 Subject: [PATCH 076/279] docs: Generate changelog for v5.0.22 --- changes/v5.0.22.en.md | 131 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 changes/v5.0.22.en.md diff --git a/changes/v5.0.22.en.md b/changes/v5.0.22.en.md new file mode 100644 index 000000000..853f781cc --- /dev/null +++ b/changes/v5.0.22.en.md @@ -0,0 +1,131 @@ +# v5.0.22 + +## Enhancements + +- [#10077](https://github.com/emqx/emqx/pull/10077) Add support for QUIC TLS password protected certificate file. + + +- [#10128](https://github.com/emqx/emqx/pull/10128) Add support for OCSP stapling for SSL MQTT listeners. + +- [#10164](https://github.com/emqx/emqx/pull/10164) Add CRL check support for TLS MQTT listeners. + +- [#10206](https://github.com/emqx/emqx/pull/10206) Decouple the query mode from the underlying call mode for buffer + workers. + + Prior to this change, setting the query mode of a resource + such as a bridge to `sync` would force the buffer to call the + underlying connector in a synchronous way, even if it supports async + calls. + +- [#10207](https://github.com/emqx/emqx/pull/10207) Use 'label' from i18n file as 'summary' in OpenAPI spec. + +- [#10210](https://github.com/emqx/emqx/pull/10210) Unregister Mnesia post commit hook when Mria is being stopped. + This fixes hook failures occasionally occurring on stopping/restarting Mria. + + [Mria PR](https://github.com/emqx/mria/pull/133) + +- [#10224](https://github.com/emqx/emqx/pull/10224) Add the option to customize `clusterIP` in Helm chart, so that a user may set it to a fixed IP. + +- [#10263](https://github.com/emqx/emqx/pull/10263) Add command 'eval-ex' for Elixir expression evaluation. + +- [#10278](https://github.com/emqx/emqx/pull/10278) Refactor the directory structure of all gateways. + +- [#10306](https://github.com/emqx/emqx/pull/10306) Add support for `async` query mode for most bridges. + + Before this change, some bridges (Cassandra, MongoDB, MySQL, Postgres, Redis, RocketMQ, TDengine) were only allowed to be created with a `sync` query mode. + +- [#10318](https://github.com/emqx/emqx/pull/10318) Now, the rule engine language's FROM clause supports both strings enclosed in double quotes (") and single quotes ('). + +- [#10336](https://github.com/emqx/emqx/pull/10336) Add `/rule_engine` API endpoint to manage configuration of rule engine. + + + +## Bug Fixes + +- [#10145](https://github.com/emqx/emqx/pull/10145) Fix `bridges` API to report error conditions for a failing bridge as + `status_reason`. Also when creating an alarm for a failing resource we include + this error condition with the alarm's message. + +- [#10154](https://github.com/emqx/emqx/pull/10154) Change the default `resume_interval` for bridges and connectors to be + the minimum of `health_check_interval` and `request_timeout / 3`. + Also exposes it as a hidden configuration to allow fine tuning. + + Before this change, the default values for `resume_interval` meant + that, if a buffer ever got blocked due to resource errors or high + message volumes, then, by the time the buffer would try to resume its + normal operations, almost all requests would have timed out. + +- [#10172](https://github.com/emqx/emqx/pull/10172) Fix the incorrect default ACL rule, which was: + ``` + {allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}. + ``` + + However, it should use `{re, "^dashboard$"}` to perform a regular expression match: + ``` + {allow, {username, {re,"^dashboard$"}}, subscribe, ["$SYS/#"]}. + ``` + +- [#10174](https://github.com/emqx/emqx/pull/10174) Upgrade library `esockd` from 5.9.4 to 5.9.6. + Fix an unnecessary error level logging when a connection is closed before proxy protocol header is sent by the proxy. + +- [#10195](https://github.com/emqx/emqx/pull/10195) Add labels to API schemas where description contains HTML and breaks formatting of generated documentation otherwise. + +- [#10196](https://github.com/emqx/emqx/pull/10196) Use lower-case for schema summaries and descritptions to be used in menu of generated online documentation. + +- [#10209](https://github.com/emqx/emqx/pull/10209) Fix bug where a last will testament (LWT) message could be published + when kicking out a banned client. + +- [#10211](https://github.com/emqx/emqx/pull/10211) Hide `broker.broker_perf` config and API documents. + The two configs `route_lock_type` and `trie_compaction` are rarely used and requires a full cluster restart to take effect. They are not suitable for being exposed to users. + Detailed changes can be found here: https://gist.github.com/zmstone/01ad5754b9beaeaf3f5b86d14d49a0b7/revisions + +- [#10225](https://github.com/emqx/emqx/pull/10225) Allow installing a plugin if its name matches the beginning of another (already installed) plugin name. + For example: if plugin "emqx_plugin_template_a" is installed, it must not block installing plugin "emqx_plugin_template". + +- [#10226](https://github.com/emqx/emqx/pull/10226) Don't crash on validation error in `/bridges` API, return `400` instead. + +- [#10237](https://github.com/emqx/emqx/pull/10237) Ensure we return `404` status code for unknown node names in `/nodes/:node[/metrics|/stats]` API. + +- [#10242](https://github.com/emqx/emqx/pull/10242) Fixed a log data field name clash. + Piror to this fix, some debug logs may report a wrong Erlang PID which may affect troubleshooting session takeover issues. + +- [#10251](https://github.com/emqx/emqx/pull/10251) Consider bridges referenced in `FROM` rule clauses as dependencies. + + Before this fix, when one tried to delete an ingress rule referenced in an action like `select * from "$bridges/mqtt:ingress"`, the UI would not trigger a warning about dependent rule actions. + +- [#10257](https://github.com/emqx/emqx/pull/10257) Fixed the issue where `auto_observe` was not working in LwM2M Gateway. + + Before the fix, OBSERVE requests were sent without a token, causing failures + that LwM2M clients could not handle. + + After the fix, LwM2M Gateway can correctly observe the resource list carried by + client, furthermore, unknown resources will be ignored and printing the following + warning log: + ``` + 2023-03-28T18:50:27.771123+08:00 [warning] msg: ignore_observer_resource, mfa: emqx_lwm2m_session:observe_object_list/3, line: 522, peername: 127.0.0.1:56830, clientid: testlwm2mclient, object_id: 31024, reason: no_xml_definition + ``` + +- [#10286](https://github.com/emqx/emqx/pull/10286) Enhance logging behaviour during boot failure. + When EMQX fails to start due to corrupted configuration files, excessive logging is eliminated and no crash dump file is generated. + +- [#10297](https://github.com/emqx/emqx/pull/10297) Keeps `eval` command backward compatible with v4 by evaluating only Erlang expressions, even on Elixir node. For Elixir expressions, use `eval-ex` command. + +- [#10300](https://github.com/emqx/emqx/pull/10300) Fixed an issue where a build made with Elixir could not receive uploaded plugins until the `plugins` folder was created manually to receive the uploaded files. + +- [#10313](https://github.com/emqx/emqx/pull/10313) Ensure that when the core or replicant node starting, the `cluster-override.conf` file is only copied from the core node. + Previously, when sorting nodes by startup time, the core node may have copied this file from the replicant node. + +- [#10314](https://github.com/emqx/emqx/pull/10314) Fix /monitor_current API so that it only looks at the current node. + Fix /stats API to not crash when one or more nodes in the cluster are down. + +- [#10315](https://github.com/emqx/emqx/pull/10315) Fix crash checking `limit` and `page` parameters in `/mqtt/delayed/messages` API call. + +- [#10317](https://github.com/emqx/emqx/pull/10317) Do not expose listener level authentications before extensive verification. + +- [#10323](https://github.com/emqx/emqx/pull/10323) For security reasons, the value of the `password` field in the API examples is replaced with `******`. + + +- [#10327](https://github.com/emqx/emqx/pull/10327) Don't increment 'actions.failed.unknown' rule metrics counter upon receiving unrecoverable bridge errors. + This counter is displayed on the dashboard's rule overview tab ('Action statistics' - 'Unknown'). + The fix is only applicable for synchronous bridges, as all rule actions for asynchronous bridges + are counted as successful (they increment 'actions.success' which is displayed as 'Action statistics' - 'Success'). From cbd2d90bb356cee15ebd80f785be972e201b1a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 13 Apr 2023 19:45:49 +0800 Subject: [PATCH 077/279] chore: add changelog for hide data items --- changes/ce/feat-10385.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-10385.en.md diff --git a/changes/ce/feat-10385.en.md b/changes/ce/feat-10385.en.md new file mode 100644 index 000000000..667e01890 --- /dev/null +++ b/changes/ce/feat-10385.en.md @@ -0,0 +1 @@ +Hide data items(rule_engine/bridge/authz/authn) from configuration files and documentation. From dd7dcfe3739f2686721d907f04804b674a8cc67a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 13 Apr 2023 20:04:03 +0800 Subject: [PATCH 078/279] fix: fix dialyzer warning --- apps/emqx/src/emqx_trace/emqx_trace.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index a185d1910..533e2e822 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -148,7 +148,11 @@ list(Enable) -> -spec create([{Key :: binary(), Value :: binary()}] | #{atom() => binary()}) -> {ok, #?TRACE{}} - | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} | iodata()}. + | {error, + {duplicate_condition, iodata()} + | {already_existed, iodata()} + | {error, {bad_type, any()}} + | iodata()}. create(Trace) -> case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of true -> From dc71a97d26532a7438a967aea40d9f2fbfc4a72a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 13 Apr 2023 20:35:15 +0800 Subject: [PATCH 079/279] chore: add changelog for max_heap_size --- changes/ce/feat-10354.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/feat-10354.en.md diff --git a/changes/ce/feat-10354.en.md b/changes/ce/feat-10354.en.md new file mode 100644 index 000000000..d728a1b20 --- /dev/null +++ b/changes/ce/feat-10354.en.md @@ -0,0 +1,2 @@ +More specific error messages when configure with bad max_heap_size value. +Log current value and the max value when the `message_queue_too_long` error is thrown. From c1163d1952dc35e038125167cc5b1655f3589c6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 13 Apr 2023 20:51:14 +0800 Subject: [PATCH 080/279] chore: share emqx_trace.hrl between code and test --- .../{src/emqx_trace => include}/emqx_trace.hrl | 0 apps/emqx/src/emqx_trace/emqx_trace.erl | 16 +++++++--------- apps/emqx/test/emqx_trace_SUITE.erl | 5 +---- 3 files changed, 8 insertions(+), 13 deletions(-) rename apps/emqx/{src/emqx_trace => include}/emqx_trace.hrl (100%) diff --git a/apps/emqx/src/emqx_trace/emqx_trace.hrl b/apps/emqx/include/emqx_trace.hrl similarity index 100% rename from apps/emqx/src/emqx_trace/emqx_trace.hrl rename to apps/emqx/include/emqx_trace.hrl diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 533e2e822..f14dc0c15 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -21,6 +21,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("kernel/include/file.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -export([ publish/1, @@ -54,13 +55,10 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --include("emqx_trace.hrl"). - -ifdef(TEST). -export([ log_file/2, - find_closest_time/2, - migrate_trace/0 + find_closest_time/2 ]). -endif. @@ -151,7 +149,7 @@ list(Enable) -> | {error, {duplicate_condition, iodata()} | {already_existed, iodata()} - | {error, {bad_type, any()}} + | {bad_type, any()} | iodata()}. create(Trace) -> case mnesia:table_info(?TRACE, size) < ?MAX_SIZE of @@ -227,15 +225,16 @@ format(Traces) -> init([]) -> erlang:process_flag(trap_exit, true), + Fields = record_info(fields, ?TRACE), ok = mria:create_table(?TRACE, [ {type, set}, {rlog_shard, ?SHARD}, {storage, disc_copies}, {record_name, ?TRACE}, - {attributes, record_info(fields, ?TRACE)} + {attributes, Fields} ]), ok = mria:wait_for_tables([?TRACE]), - migrate_trace(), + maybe_migrate_trace(Fields), {ok, _} = mnesia:subscribe({table, ?TRACE, simple}), ok = filelib:ensure_dir(filename:join([trace_dir(), dummy])), ok = filelib:ensure_dir(filename:join([zip_dir(), dummy])), @@ -583,8 +582,7 @@ filter_cli_handler(Names) -> now_second() -> os:system_time(second). -migrate_trace() -> - Fields = record_info(fields, ?TRACE), +maybe_migrate_trace(Fields) -> case mnesia:table_info(emqx_trace, attributes) =:= Fields of true -> ok; diff --git a/apps/emqx/test/emqx_trace_SUITE.erl b/apps/emqx/test/emqx_trace_SUITE.erl index 3594f0651..140ec79ff 100644 --- a/apps/emqx/test/emqx_trace_SUITE.erl +++ b/apps/emqx/test/emqx_trace_SUITE.erl @@ -22,12 +22,9 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_trace.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --record(emqx_trace, { - name, type, filter, enable = true, payload_encode = text, extra = #{}, start_at, end_at -}). - %%-------------------------------------------------------------------- %% Setups %%-------------------------------------------------------------------- From 58e31d5efd8dba6adad48359f1feb088ac14aa30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 13 Apr 2023 21:33:48 +0800 Subject: [PATCH 081/279] feat: hide ex_hook/rewrite/topic_metric/persistent_session_store/overload_protection --- apps/emqx/src/emqx_schema.erl | 4 ++-- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook_schema.erl | 3 ++- apps/emqx_modules/src/emqx_modules_schema.erl | 10 ++++++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 20018b2d5..48b0b4fed 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -186,7 +186,7 @@ roots(medium) -> {"overload_protection", sc( ref("overload_protection"), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )} ]; roots(low) -> @@ -224,7 +224,7 @@ roots(low) -> {"persistent_session_store", sc( ref("persistent_session_store"), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )}, {"trace", sc( diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 8ca15a907..194c91206 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 07373288d..708e164fc 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -31,7 +31,8 @@ namespace() -> exhook. -roots() -> [exhook]. +roots() -> + [{exhook, ?HOCON(?R_REF(exhook), #{importance => ?IMPORTANCE_HIDDEN})}]. fields(exhook) -> [ diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index ddb8f37a3..36a08de60 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -34,8 +34,14 @@ roots() -> [ "delayed", "telemetry", - array("rewrite", #{desc => "List of topic rewrite rules."}), - array("topic_metrics", #{desc => "List of topics whose metrics are reported."}) + array("rewrite", #{ + desc => "List of topic rewrite rules.", + importance => ?IMPORTANCE_HIDDEN + }), + array("topic_metrics", #{ + desc => "List of topics whose metrics are reported.", + importance => ?IMPORTANCE_HIDDEN + }) ]. fields("telemetry") -> From 3431a1b12385d004afa87428150c91f23dd2fba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 13 Apr 2023 21:55:01 +0800 Subject: [PATCH 082/279] chore: add changelog --- changes/ce/feat-10391.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-10391.en.md diff --git a/changes/ce/feat-10391.en.md b/changes/ce/feat-10391.en.md new file mode 100644 index 000000000..a64b01221 --- /dev/null +++ b/changes/ce/feat-10391.en.md @@ -0,0 +1 @@ +hide exhook/rewrite/topic_metric/persistent_session_store/overload_protection from the docs and configuration file. From 2334917e353d45688df9b4d973840fd7150c2f00 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 13 Apr 2023 21:59:26 +0800 Subject: [PATCH 083/279] chore: apply suggestions from code review Co-authored-by: Thales Macedo Garitezi --- apps/emqx_management/src/emqx_mgmt_api_trace.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 619da92f0..a69837eb3 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -277,8 +277,7 @@ fields(trace) -> "It is recommended when payload is a custom binary protocol.
\n" "`hidden`: payload is obfuscated as `******`" "", - default => text, - required => false + default => text })}, {start_at, hoconsc:mk( From d3ccd8a65dcbcdf8f5d4845bcec78d4636f64303 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Thu, 13 Apr 2023 15:20:04 +0200 Subject: [PATCH 084/279] feat: add date_to_unix_ts/3 function to the rule engine Fixes: https://emqx.atlassian.net/browse/EMQX-9245 --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 9 +++++++++ .../test/emqx_rule_funcs_SUITE.erl | 18 ++++++++++++++++++ changes/ce/feat-10392.en.md | 1 + 3 files changed, 28 insertions(+) create mode 100644 changes/ce/feat-10392.en.md diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index b8bfeb84c..7f7662b1b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -227,6 +227,7 @@ now_timestamp/1, format_date/3, format_date/4, + date_to_unix_ts/3, date_to_unix_ts/4 ]). @@ -1085,6 +1086,14 @@ format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> ) ). +date_to_unix_ts(TimeUnit, FormatString, InputString) -> + emqx_rule_date:parse_date( + time_unit(TimeUnit), + "Z", + emqx_plugin_libs_rule:str(FormatString), + emqx_plugin_libs_rule:str(InputString) + ). + date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> emqx_rule_date:parse_date( time_unit(TimeUnit), diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 94adb3506..2bce5a1b4 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -1003,6 +1003,24 @@ prop_format_date_fun() -> ) ] ) + ), + %% When no offset is specified, the offset should be taken from the formatted time string + ArgsNoOffset = [<<"second">>, <<"%y-%m-%d-%H:%M:%S%Z">>], + ArgsOffset = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], + ?FORALL( + S, + erlang:system_time(second), + S == + apply_func( + date_to_unix_ts, + ArgsNoOffset ++ + [ + apply_func( + format_date, + ArgsOffset ++ [S] + ) + ] + ) ). %%------------------------------------------------------------------------------ diff --git a/changes/ce/feat-10392.en.md b/changes/ce/feat-10392.en.md new file mode 100644 index 000000000..04c6c85cc --- /dev/null +++ b/changes/ce/feat-10392.en.md @@ -0,0 +1 @@ +A new function to convert a formatted date to an integer timestamp has been added: date_to_unix_ts/3 From 9f99a725940a0dace605495ff946b09cf0b66e66 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 14 Apr 2023 00:45:22 +0800 Subject: [PATCH 085/279] chore: refine tests --- apps/emqx_gateway/test/emqx_gateway_SUITE.erl | 12 ++++++------ apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl | 2 +- apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl index 5120e096e..9f8c7911c 100644 --- a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl @@ -68,11 +68,11 @@ end_per_testcase(_TestCase, _Config) -> t_registered_gateway(_) -> [ - {coap, #{cbkmod := emqx_coap}}, - {exproto, #{cbkmod := emqx_exproto}}, - {lwm2m, #{cbkmod := emqx_lwm2m}}, - {mqttsn, #{cbkmod := emqx_mqttsn}}, - {stomp, #{cbkmod := emqx_stomp}} + {coap, #{cbkmod := emqx_gateway_coap}}, + {exproto, #{cbkmod := emqx_gateway_exproto}}, + {lwm2m, #{cbkmod := emqx_gateway_lwm2m}}, + {mqttsn, #{cbkmod := emqx_gateway_mqttsn}}, + {stomp, #{cbkmod := emqx_gateway_stomp}} ] = emqx_gateway:registered_gateway(). t_load_unload_list_lookup(_) -> @@ -192,7 +192,7 @@ setup_fake_usage_data(Lwm2mDataDir) -> [ emqx_common_test_helpers:proj_root(), "apps", - "emqx_lwm2m", + "emqx_gateway_lwm2m", "lwm2m_xml" ] ), diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index c5fabf2fd..10adcc428 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -219,7 +219,7 @@ t_gateway_lwm2m(_) -> [ emqx_common_test_helpers:proj_root(), "apps", - "emqx_lwm2m", + "emqx_gateway_lwm2m", "lwm2m_xml" ] ), diff --git a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl index a234dd126..641528eda 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl @@ -117,11 +117,11 @@ t_gateway_registry_usage(_) -> t_gateway_registry_list(_) -> emqx_gateway_cli:'gateway-registry'(["list"]), ?assertEqual( - "Registered Name: coap, Callback Module: emqx_coap\n" - "Registered Name: exproto, Callback Module: emqx_exproto\n" - "Registered Name: lwm2m, Callback Module: emqx_lwm2m\n" - "Registered Name: mqttsn, Callback Module: emqx_mqttsn\n" - "Registered Name: stomp, Callback Module: emqx_stomp\n", + "Registered Name: coap, Callback Module: emqx_gateway_coap\n" + "Registered Name: exproto, Callback Module: emqx_gateway_exproto\n" + "Registered Name: lwm2m, Callback Module: emqx_gateway_lwm2m\n" + "Registered Name: mqttsn, Callback Module: emqx_gateway_mqttsn\n" + "Registered Name: stomp, Callback Module: emqx_gateway_stomp\n", acc_print() ). From c5ad3c8f41f05308ed983093efece207623c487a Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Thu, 13 Apr 2023 22:39:49 +0200 Subject: [PATCH 086/279] docs(emqx_machine): Improve README --- apps/emqx_machine/README.md | 45 ++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/apps/emqx_machine/README.md b/apps/emqx_machine/README.md index 9ff33a5e5..8c2bb6391 100644 --- a/apps/emqx_machine/README.md +++ b/apps/emqx_machine/README.md @@ -1,5 +1,48 @@ -# EMQX Machine +# EMQX machine This application manages other OTP applications in EMQX and serves as the entry point when BEAM VM starts up. It prepares the node before starting mnesia/mria, as well as EMQX business logic. It keeps track of the business applications storing data in Mnesia, which need to be restarted when the node joins the cluster by registering `ekka` callbacks. +Also it kicks off autoclustering (EMQX cluster discovery) on the core nodes. + +`emqx_machine` application doesn't do much on its own, but it facilitates the environment for running other EMQX applications. + +# Features + +## Global GC + +`emqx_global_gc` is a gen_server that forces garbage collection of all Erlang processes running in the BEAM VM. +This is meant to save the RAM. + +## Restricted shell + +`emqx_restricted_shell` module prevents user from accidentally issuing Erlang shell commands that can stop the remote node. + +## Signal handler + +`emqx_machine_signal_handler` handles POSIX signals sent to BEAM VM process. +It helps to shut down EMQX broker gracefully when it receives `SIGTERM` signal. + +## Cover + +`emqx_cover` is a helper module that helps to collect coverage reports during testing. + +# Limitation + +Currently `emqx_machine` boots the business apps before starting autocluster, so a fresh node joining the cluster actually starts business application twice: first in the singleton mode, and then in clustered mode. + +# Documention links + +Configuration: [node.global_gc_interval](https://www.emqx.io/docs/en/v5.0/configuration/configuration-manual.html#node-and-cookie) + +# Configurations + +The following application environment variables are used: + +- `emqx_machine.global_gc_interval`: interval at which global GC is run +- `emqx_machine.custom_shard_transports`: contains a map that allows to fine tune transport (`rpc` or `gen_rpc`) used to send Mria transactions from the core node to the replicant +- `emqx_machine.backtrace_depth`: set maximum depth of Erlang stacktraces in crash reports + + +# Contributing +Please see our [contributing.md](../../CONTRIBUTING.md). From 8ceeafb0de8f49e02afdf9d63ec402ad6a8a2b42 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 22 Mar 2023 10:28:37 +0800 Subject: [PATCH 087/279] chore: fix file license in lib-ee --- .../emqx_connector/src/emqx_connector_mysql.erl | 2 +- .../src/emqx_ee_connector_cassa.erl | 13 +------------ .../src/emqx_ee_connector_clickhouse.erl | 14 +------------- .../test/ee_connector_clickhouse_SUITE.erl | 17 +++-------------- .../test/emqx_ee_connector_cassa_SUITE.erl | 13 +------------ 5 files changed, 7 insertions(+), 52 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index fe495252a..c69cf5ca0 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -437,7 +437,7 @@ do_sql_query(SQLFunc, Conn, SQLOrKey, Params, Timeout, LogMeta) -> error, LogMeta#{msg => "mysql_connector_do_sql_query_failed", reason => disconnected} ), - %% kill the poll worker to trigger reconnection + %% kill the pool worker to trigger reconnection _ = exit(Conn, restart), {error, {recoverable_error, disconnected}}; {error, not_prepared} = Error -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index 1e1882a1f..ce964dd17 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -1,18 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. %%-------------------------------------------------------------------- + -module(emqx_ee_connector_cassa). -behaviour(emqx_resource). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl index 5d68687dc..3749ecc6e 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl @@ -1,17 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_connector_clickhouse). diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl b/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl index 111deba06..c608588a1 100644 --- a/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl @@ -1,17 +1,6 @@ -% %%-------------------------------------------------------------------- -% %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -% %% -% %% Licensed under the Apache License, Version 2.0 (the "License"); -% %% you may not use this file except in compliance with the License. -% %% You may obtain a copy of the License at -% %% http://www.apache.org/licenses/LICENSE-2.0 -% %% -% %% Unless required by applicable law or agreed to in writing, software -% %% distributed under the License is distributed on an "AS IS" BASIS, -% %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% %% See the License for the specific language governing permissions and -% %% limitations under the License. -% %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- -module(ee_connector_clickhouse_SUITE). diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl index 95b4407cf..52ed03a62 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl @@ -1,16 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -module(emqx_ee_connector_cassa_SUITE). From 8e62afecdd31cb905426bae1c90e55c767db9dfe Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 27 Mar 2023 13:55:16 +0800 Subject: [PATCH 088/279] chore: suite module rename --- ...ckhouse_SUITE.erl => emqx_ee_connector_clickhouse_SUITE.erl} | 2 +- ...streamdb_SUITE.erl => emqx_ee_connector_hstreamdb_SUITE.erl} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename lib-ee/emqx_ee_connector/test/{ee_connector_clickhouse_SUITE.erl => emqx_ee_connector_clickhouse_SUITE.erl} (99%) rename lib-ee/emqx_ee_connector/test/{ee_connector_hstreamdb_SUITE.erl => emqx_ee_connector_hstreamdb_SUITE.erl} (90%) diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl similarity index 99% rename from lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl rename to lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl index c608588a1..73018e14f 100644 --- a/lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(ee_connector_clickhouse_SUITE). +-module(emqx_ee_connector_clickhouse_SUITE). -compile(nowarn_export_all). -compile(export_all). diff --git a/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl similarity index 90% rename from lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl rename to lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl index 8acabbef4..ad49d9f62 100644 --- a/lib-ee/emqx_ee_connector/test/ee_connector_hstreamdb_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(ee_connector_hstreamdb_SUITE). +-module(emqx_ee_connector_hstreamdb_SUITE). -compile(nowarn_export_all). -compile(export_all). From 58419698775275da71fa16229ebb94aa21b4e3ec Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 20 Mar 2023 10:28:34 +0800 Subject: [PATCH 089/279] feat: implement Microsoft SQL Server bridge (e5.0) --- apps/emqx_bridge/src/emqx_bridge.erl | 3 +- changes/ee/feat-10363.en.md | 1 + lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 23 +- .../src/emqx_ee_bridge_sqlserver.erl | 128 +++++ .../src/emqx_ee_connector.app.src | 4 +- .../src/emqx_ee_connector_sqlserver.erl | 531 ++++++++++++++++++ rel/i18n/emqx_ee_bridge_sqlserver.hocon | 87 +++ rel/i18n/emqx_ee_connector_sqlserver.hocon | 22 + 8 files changed, 793 insertions(+), 6 deletions(-) create mode 100644 changes/ee/feat-10363.en.md create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl create mode 100644 rel/i18n/emqx_ee_bridge_sqlserver.hocon create mode 100644 rel/i18n/emqx_ee_connector_sqlserver.hocon diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index bf91d20f7..5d1850ebd 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -69,7 +69,8 @@ T == tdengine; T == dynamo; T == rocketmq; - T == cassandra + T == cassandra; + T == sqlserver ). load() -> diff --git a/changes/ee/feat-10363.en.md b/changes/ee/feat-10363.en.md new file mode 100644 index 000000000..c3b53a538 --- /dev/null +++ b/changes/ee/feat-10363.en.md @@ -0,0 +1 @@ +Add MicroSoft SQL Server data Bridge support. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 84b0b98b0..3ad5cbbb4 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -34,7 +34,8 @@ api_schemas(Method) -> ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_ee_bridge_dynamo, Method), ref(emqx_ee_bridge_rocketmq, Method), - ref(emqx_ee_bridge_cassa, Method) + ref(emqx_ee_bridge_cassa, Method), + ref(emqx_ee_bridge_sqlserver, Method) ]. schema_modules() -> @@ -53,7 +54,8 @@ schema_modules() -> emqx_ee_bridge_clickhouse, emqx_ee_bridge_dynamo, emqx_ee_bridge_rocketmq, - emqx_ee_bridge_cassa + emqx_ee_bridge_cassa, + emqx_ee_bridge_sqlserver ]. examples(Method) -> @@ -91,7 +93,8 @@ resource_type(tdengine) -> emqx_ee_connector_tdengine; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; resource_type(dynamo) -> emqx_ee_connector_dynamo; resource_type(rocketmq) -> emqx_ee_connector_rocketmq; -resource_type(cassandra) -> emqx_ee_connector_cassa. +resource_type(cassandra) -> emqx_ee_connector_cassa; +resource_type(sqlserver) -> emqx_ee_connector_sqlserver. fields(bridges) -> [ @@ -152,7 +155,7 @@ fields(bridges) -> } )} ] ++ kafka_structs() ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ - pgsql_structs() ++ clickhouse_structs(). + pgsql_structs() ++ clickhouse_structs() ++ sqlserver_structs(). mongodb_structs() -> [ @@ -249,3 +252,15 @@ clickhouse_structs() -> } )} ]. + +sqlserver_structs() -> + [ + {sqlserver, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_sqlserver, "config")), + #{ + desc => <<"Microsoft SQL Server Bridge Config">>, + required => false + } + )} + ]. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl new file mode 100644 index 000000000..49a5ed0ce --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl @@ -0,0 +1,128 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_sqlserver). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_examples/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-define(DEFAULT_SQL, << + "insert into t_mqtt_msg(msgid, topic, qos, payload)" + "values (${id}, ${topic}, ${qos}, ${payload})" +>>). + +-define(DEFAULT_DRIVER, <<"ms-sqlserver-18">>). + +conn_bridge_examples(Method) -> + [ + #{ + <<"sqlserver">> => #{ + summary => <<"Microsoft SQL Server Bridge">>, + value => values(Method) + } + } + ]. + +values(get) -> + values(post); +values(post) -> + #{ + enable => true, + type => sqlserver, + name => <<"bar">>, + server => <<"127.0.0.1:1433">>, + database => <<"test">>, + pool_size => 8, + username => <<"sa">>, + password => <<"******">>, + sql => ?DEFAULT_SQL, + driver => ?DEFAULT_DRIVER, + local_topic => <<"local/topic/#">>, + resource_opts => #{ + worker_pool_size => 1, + health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, + auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, + batch_size => ?DEFAULT_BATCH_SIZE, + batch_time => ?DEFAULT_BATCH_TIME, + query_mode => async, + max_queue_bytes => ?DEFAULT_QUEUE_SIZE + } + }; +values(put) -> + values(post). + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge_sqlserver". + +roots() -> []. + +fields("config") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {sql, + mk( + binary(), + #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} + )}, + {driver, mk(binary(), #{desc => ?DESC("driver"), default => ?DEFAULT_DRIVER})}, + {local_topic, + mk( + binary(), + #{desc => ?DESC("local_topic"), default => undefined} + )}, + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{ + required => false, + default => #{}, + desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) + } + )} + ] ++ + (emqx_ee_connector_sqlserver:fields(config) -- + emqx_connector_schema_lib:prepare_statement_fields()); +fields("creation_opts") -> + emqx_resource_schema:fields("creation_opts"); +fields("post") -> + fields("post", sqlserver); +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +fields("post", Type) -> + [type_field(Type), name_field() | fields("config")]. + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for Microsoft SQL Server using `", string:to_upper(Method), "` method."]; +desc("creation_opts" = Name) -> + emqx_resource_schema:desc(Name); +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- + +type_field(Type) -> + {type, mk(enum([Type]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 2e6406d70..2e2ff2a62 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -5,13 +5,15 @@ {applications, [ kernel, stdlib, + ecpool, hstreamdb_erl, influxdb, tdengine, clickhouse, erlcloud, rocketmq, - ecql + ecql, + odbc ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl new file mode 100644 index 000000000..4cc92e80a --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -0,0 +1,531 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_sqlserver). + +-behaviour(emqx_resource). + +-include_lib("kernel/include/file.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("emqx_ee_connector/include/emqx_ee_connector.hrl"). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%%==================================================================== +%% Exports +%%==================================================================== + +%% Hocon config schema exports +-export([ + roots/0, + fields/1 +]). + +%% callbacks for behaviour emqx_resource +-export([ + callback_mode/0, + is_buffer_supported/0, + on_start/2, + on_stop/2, + on_query/3, + on_batch_query/3, + on_query_async/4, + on_batch_query_async/4, + on_get_status/2 +]). + +%% callbacks for ecpool +-export([connect/1]). + +%% Internal exports used to execute code with ecpool worker +-export([do_get_status/2, worker_do_insert/3, do_async_reply/2]). + +-import(emqx_plugin_libs_rule, [str/1]). +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-define(ACTION_SEND_MESSAGE, send_message). + +-define(SINGLE_INSERT, single_insert). + +-define(SYNC_QUERY_MODE, handover). +-define(ASYNC_QUERY_MODE(REPLY), {handover_async, {?MODULE, do_async_reply, [REPLY]}}). + +-define(SQLSERVER_HOST_OPTIONS, #{ + default_port => 1433 +}). + +-define(REQUEST_TIMEOUT(RESOURCE_OPTS), + maps:get(request_timeout, RESOURCE_OPTS, ?DEFAULT_REQUEST_TIMEOUT) +). + +-define(SINGLE_INSERT_TEMP, single_insert_temp). +-define(BATCH_INSERT_TEMP, batch_insert_temp). + +-define(BATCH_INSERT_PART, batch_insert_part). +-define(BATCH_PARAMS_TOKENS, batch_insert_tks). + +%% Copied from odbc reference page +%% https://www.erlang.org/doc/man/odbc.html + +-type bridge_id() :: binary(). +%% as returned by connect/2 +-type connection_reference() :: pid(). +-type time_out() :: milliseconds() | infinity. +-type sql() :: string() | binary(). +-type milliseconds() :: pos_integer(). +%% Tuple of column values e.g. one row of the result set. +%% it's a variable size tuple of column values. +-type row() :: tuple(). +%% Some kind of explanation of what went wrong +-type common_reason() :: connection_closed | extended_error() | term(). +%% extended error type with ODBC +%% and native database error codes, as well as the base reason that would have been +%% returned had extended_errors not been enabled. +-type extended_error() :: {string(), integer(), _Reason :: term()}. +%% Name of column in the result set +-type col_name() :: string(). +%% e.g. a list of the names of the selected columns in the result set. +-type col_names() :: [col_name()]. +%% A list of rows from the result set. +-type rows() :: list(row()). + +%% -type result_tuple() :: {updated, n_rows()} | {selected, col_names(), rows()}. +-type updated_tuple() :: {updated, n_rows()}. +-type selected_tuple() :: {selected, col_names(), rows()}. +%% The number of affected rows for UPDATE, +%% INSERT, or DELETE queries. For other query types the value +%% is driver defined, and hence should be ignored. +-type n_rows() :: integer(). + +%% These type was not used in this module, but we may use it later +%% -type odbc_data_type() :: +%% sql_integer +%% | sql_smallint +%% | sql_tinyint +%% | {sql_decimal, precision(), scale()} +%% | {sql_numeric, precision(), scale()} +%% | {sql_char, size()} +%% | {sql_wchar, size()} +%% | {sql_varchar, size()} +%% | {sql_wvarchar, size()} +%% | {sql_float, precision()} +%% | {sql_wlongvarchar, size()} +%% | {sql_float, precision()} +%% | sql_real +%% | sql_double +%% | sql_bit +%% | atom(). +%% -type precision() :: integer(). +%% -type scale() :: integer(). +%% -type size() :: integer(). + +-type state() :: #{ + poolname := binary(), + resource_opts := map(), + sql_templates := map() +}. + +%%==================================================================== +%% Configuration and default values +%%==================================================================== + +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]. + +fields(config) -> + [ + {server, server()} + | add_default_username(emqx_connector_schema_lib:relational_db_fields()) + ]. + +add_default_username(Fields) -> + lists:map( + fun + ({username, OrigUsernameFn}) -> + {username, add_default_fn(OrigUsernameFn, <<"sa">>)}; + (Field) -> + Field + end, + Fields + ). + +add_default_fn(OrigFn, Default) -> + fun + (default) -> Default; + (Field) -> OrigFn(Field) + end. + +server() -> + Meta = #{desc => ?DESC("server")}, + emqx_schema:servers_sc(Meta, ?SQLSERVER_HOST_OPTIONS). + +%%==================================================================== +%% Callbacks defined in emqx_resource +%%==================================================================== + +callback_mode() -> async_if_possible. + +is_buffer_supported() -> false. + +on_start( + BridgeId = PoolName, + #{ + server := Server, + username := Username, + password := Password, + driver := Driver, + database := Database, + pool_size := PoolSize, + resource_opts := ResourceOpts + } = Config +) -> + ?SLOG(info, #{ + msg => "starting_sqlserver_connector", + connector => BridgeId, + config => emqx_misc:redact(Config) + }), + + ODBCDir = code:priv_dir(odbc), + OdbcserverDir = filename:join(ODBCDir, "bin/odbcserver"), + {ok, Info = #file_info{mode = Mode}} = file:read_file_info(OdbcserverDir), + case 33261 =:= Mode of + true -> + ok; + false -> + _ = file:write_file_info(OdbcserverDir, Info#file_info{mode = 33261}), + ok + end, + + Options = [ + {server, to_bin(Server)}, + {username, Username}, + {password, Password}, + {driver, Driver}, + {database, Database}, + {pool_size, PoolSize}, + {poolname, PoolName} + ], + + State = #{ + %% also BridgeId + poolname => PoolName, + sql_templates => parse_sql_template(Config), + resource_opts => ResourceOpts + }, + case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options) of + ok -> + {ok, State}; + {error, Reason} -> + ?tp( + sqlserver_connector_start_failed, + #{error => Reason} + ), + {error, Reason} + end. + +on_stop(InstanceId, #{poolname := PoolName} = _State) -> + ?SLOG(info, #{ + msg => "stopping_sqlserver_connector", + connector => InstanceId + }), + emqx_plugin_libs_pool:stop_pool(PoolName). + +-spec on_query( + bridge_id(), + {?ACTION_SEND_MESSAGE, map()}, + state() +) -> + ok + | {ok, list()} + | {error, {recoverable_error, term()}} + | {error, term()}. +on_query(BridgeId, {?ACTION_SEND_MESSAGE, _Msg} = Query, State) -> + ?TRACE( + "SINGLE_QUERY_SYNC", + "bridge_sqlserver_received", + #{requests => Query, connector => BridgeId, state => State} + ), + do_query(BridgeId, Query, ?SYNC_QUERY_MODE, State). + +-spec on_query_async( + bridge_id(), + {?ACTION_SEND_MESSAGE, map()}, + {ReplyFun :: function(), Args :: list()}, + state() +) -> + {ok, any()} + | {error, term()}. +on_query_async( + BridgeId, + {?ACTION_SEND_MESSAGE, _Msg} = Query, + ReplyFunAndArgs, + %% #{poolname := PoolName, sql_templates := Templates} = State + State +) -> + ?TRACE( + "SINGLE_QUERY_ASYNC", + "bridge_sqlserver_received", + #{requests => Query, connector => BridgeId, state => State} + ), + do_query(BridgeId, Query, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). + +-spec on_batch_query( + bridge_id(), + [{?ACTION_SEND_MESSAGE, map()}], + state() +) -> + ok + | {ok, list()} + | {error, {recoverable_error, term()}} + | {error, term()}. +on_batch_query(BridgeId, BatchRequests, State) -> + ?TRACE( + "BATCH_QUERY_SYNC", + "bridge_sqlserver_received", + #{requests => BatchRequests, connector => BridgeId, state => State} + ), + do_query(BridgeId, BatchRequests, ?SYNC_QUERY_MODE, State). + +-spec on_batch_query_async( + bridge_id(), + [{?ACTION_SEND_MESSAGE, map()}], + {ReplyFun :: function(), Args :: list()}, + state() +) -> {ok, any()}. +on_batch_query_async(BridgeId, Requests, ReplyFunAndArgs, State) -> + ?TRACE( + "BATCH_QUERY_ASYNC", + "bridge_sqlserver_received", + #{requests => Requests, connector => BridgeId, state => State} + ), + do_query(BridgeId, Requests, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). + +on_get_status(_InstanceId, #{poolname := Pool, resource_opts := ResourceOpts} = _State) -> + RequestTimeout = ?REQUEST_TIMEOUT(ResourceOpts), + Health = emqx_plugin_libs_pool:health_check_ecpool_workers( + Pool, {?MODULE, do_get_status, [RequestTimeout]}, RequestTimeout + ), + status_result(Health). + +status_result(_Status = true) -> connected; +status_result(_Status = false) -> connecting. +%% TODO: +%% case for disconnected + +%%==================================================================== +%% ecpool callback fns +%%==================================================================== + +-spec connect(Options :: list()) -> {ok, connection_reference()} | {error, term()}. +connect(Options) -> + ConnectStr = lists:concat(conn_str(Options, [])), + Opts = proplists:get_value(options, Options, []), + odbc:connect(ConnectStr, Opts). + +-spec do_get_status(connection_reference(), time_out()) -> Result :: boolean(). +do_get_status(Conn, RequestTimeout) -> + case execute(Conn, <<"SELECT 1">>, RequestTimeout) of + {selected, [[]], [{1}]} -> true; + _ -> false + end. + +%%==================================================================== +%% Internal Helper fns +%%==================================================================== + +%% TODO && FIXME: +%% About the connection string attribute `Encrypt`: +%% The default value is `yes` in odbc version 18.0+ and `no` in previous versions. +%% And encrypted connections always verify the server's certificate. +%% So `Encrypt=YES;TrustServerCertificate=YES` must be set in the connection string when connecting to a server that has a self-signed certificate. +%% See also: +%% https://learn.microsoft.com/en-us/sql/connect/odbc/dsn-connection-string-attribute?source=recommendations&view=sql-server-ver16#encrypt +conn_str([], Acc) -> + %% we should use this for msodbcsql 18+ + %% lists:join(";", ["Encrypt=YES", "TrustServerCertificate=YES" | Acc]); + lists:join(";", Acc); +conn_str([{driver, Driver} | Opts], Acc) -> + conn_str(Opts, ["Driver=" ++ str(Driver) | Acc]); +conn_str([{server, Server} | Opts], Acc) -> + {Host, Port} = emqx_schema:parse_server(Server, ?SQLSERVER_HOST_OPTIONS), + conn_str(Opts, ["Server=" ++ str(Host) ++ "," ++ str(Port) | Acc]); +conn_str([{database, Database} | Opts], Acc) -> + conn_str(Opts, ["Database=" ++ str(Database) | Acc]); +conn_str([{username, Username} | Opts], Acc) -> + conn_str(Opts, ["UID=" ++ str(Username) | Acc]); +conn_str([{password, Password} | Opts], Acc) -> + conn_str(Opts, ["PWD=" ++ str(Password) | Acc]); +conn_str([{_, _} | Opts], Acc) -> + conn_str(Opts, Acc). + +%% Sync & Async query with singe & batch sql statement +-spec do_query( + bridge_id(), + Query :: {?ACTION_SEND_MESSAGE, map()} | [{?ACTION_SEND_MESSAGE, map()}], + ApplyMode :: + handover + | {handover_async, {?MODULE, do_async_reply, [{ReplyFun :: function(), Args :: list()}]}}, + state() +) -> + {ok, list()} + | {error, {recoverable_error, term()}} + | {error, term()}. +do_query( + BridgeId, + Query, + ApplyMode, + #{poolname := PoolName, sql_templates := Templates} = State +) -> + ?TRACE( + "SINGLE_QUERY_SYNC", + "sqlserver_connector_received", + #{query => Query, connector => BridgeId, state => State} + ), + + %% only insert sql statement for single query and batch query + case apply_template(Query, Templates) of + {?ACTION_SEND_MESSAGE, SQL} -> + Result = ecpool:pick_and_do( + PoolName, + {?MODULE, worker_do_insert, [SQL, State]}, + ApplyMode + ); + Query -> + Result = {error, {unrecoverable_error, invalid_query}}; + _ -> + Result = {error, {unrecoverable_error, failed_to_apply_sql_template}} + end, + case Result of + {error, Reason} -> + ?tp( + sqlserver_connector_query_return, + #{error => Reason} + ), + ?SLOG(error, #{ + msg => "sqlserver_connector_do_query_failed", + connector => BridgeId, + query => Query, + reason => Reason + }), + Result; + _ -> + ?tp( + sqlserver_connector_query_return, + #{result => Result} + ), + Result + end. + +worker_do_insert( + Conn, SQL, #{resource_opts := ResourceOpts, poolname := BridgeId} = State +) -> + LogMeta = #{connector => BridgeId, state => State}, + try + case execute(Conn, SQL, ?REQUEST_TIMEOUT(ResourceOpts)) of + {selected, Rows, _} -> + {ok, Rows}; + {updated, _} -> + ok; + {error, ErrStr} -> + ?SLOG(error, LogMeta#{msg => "invalid_request", reason => ErrStr}), + {error, {unrecoverable_error, {invalid_request, ErrStr}}} + end + catch + _Type:Reason -> + ?SLOG(error, LogMeta#{msg => "invalid_request", reason => Reason}), + {error, {unrecoverable_error, {invalid_request, Reason}}} + end. + +-spec execute(pid(), sql(), time_out()) -> + updated_tuple() + | selected_tuple() + | [updated_tuple()] + | [selected_tuple()] + | {error, common_reason()}. +execute(Conn, SQL, Timeout) -> + odbc:sql_query(Conn, str(SQL), Timeout). + +to_bin(List) when is_list(List) -> + unicode:characters_to_binary(List, utf8). + +%% for bridge data to sql server +parse_sql_template(Config) -> + RawSQLTemplates = + case maps:get(sql, Config, undefined) of + undefined -> #{}; + <<>> -> #{}; + SQLTemplate -> #{?ACTION_SEND_MESSAGE => SQLTemplate} + end, + + SingleInsertTks = #{}, + BatchInsertTks = #{}, + parse_sql_template(maps:to_list(RawSQLTemplates), SingleInsertTks, BatchInsertTks). + +parse_sql_template([{Key, H} | T], SingleInsertTks, BatchInsertTks) -> + case emqx_plugin_libs_rule:detect_sql_type(H) of + {ok, select} -> + parse_sql_template(T, SingleInsertTks, BatchInsertTks); + {ok, insert} -> + case emqx_plugin_libs_rule:split_insert_sql(H) of + {ok, {InsertSQL, Params}} -> + parse_sql_template( + T, + SingleInsertTks#{Key => emqx_plugin_libs_rule:preproc_tmpl(H)}, + BatchInsertTks#{ + Key => + #{ + ?BATCH_INSERT_PART => InsertSQL, + ?BATCH_PARAMS_TOKENS => emqx_plugin_libs_rule:preproc_tmpl( + Params + ) + } + } + ); + {error, Reason} -> + ?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}), + parse_sql_template(T, SingleInsertTks, BatchInsertTks) + end; + {error, Reason} -> + ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}), + parse_sql_template(T, SingleInsertTks, BatchInsertTks) + end; +parse_sql_template([], SingleInsertTks, BatchInsertTks) -> + #{ + ?SINGLE_INSERT_TEMP => SingleInsertTks, + ?BATCH_INSERT_TEMP => BatchInsertTks + }. + +%% single insert +apply_template( + {?ACTION_SEND_MESSAGE = Key, Msg} = Query, + #{?SINGLE_INSERT_TEMP := SingleInsertTks} = _Templates +) -> + case maps:get(Key, SingleInsertTks, undefined) of + undefined -> + Query; + Template -> + {Key, emqx_plugin_libs_rule:proc_tmpl(Template, Msg)} + end; +%% batch inserts +apply_template( + [{?ACTION_SEND_MESSAGE = Key, _Msg} | _T] = BatchReqs, + #{?BATCH_INSERT_TEMP := BatchInsertsTks} = _Templates +) -> + case maps:get(Key, BatchInsertsTks, undefined) of + undefined -> + BatchReqs; + #{?BATCH_INSERT_PART := BatchInserts, ?BATCH_PARAMS_TOKENS := BatchParamsTks} -> + SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, BatchInserts, BatchParamsTks), + {Key, SQL} + end; +apply_template(Query, Templates) -> + %% TODO: more detail infomatoin + ?SLOG(error, #{msg => "apply sql template failed", query => Query, templates => Templates}). + +do_async_reply(Result, {ReplyFun, Args}) -> + erlang:apply(ReplyFun, Args ++ Result). diff --git a/rel/i18n/emqx_ee_bridge_sqlserver.hocon b/rel/i18n/emqx_ee_bridge_sqlserver.hocon new file mode 100644 index 000000000..bb82c82f8 --- /dev/null +++ b/rel/i18n/emqx_ee_bridge_sqlserver.hocon @@ -0,0 +1,87 @@ +emqx_ee_bridge_sqlserver { + + local_topic { + desc { + en: """The MQTT topic filter to be forwarded to MicroSoft SQL Server. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded. +""" + zh: """发送到 'local_topic' 的消息都会转发到 MicroSoft SQL Server。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。 +""" + } + label { + en: """Local Topic""" + zh: """本地 Topic""" + } + } + + sql_template { + desc { + en: """SQL Template""" + zh: """SQL 模板""" + } + label { + en: """SQL Template""" + zh: """SQL 模板""" + } + } + + driver { + desc { + en: """SQL Server Driver Name""" + zh: """SQL Server Driver 名称""" + } + desc { + en: """SQL Server Driver Name""" + zh: """SQL Server Driver 名称""" + } + } + + config_enable { + desc { + en: """Enable or disable this bridge""" + zh: """启用/禁用桥接""" + } + label { + en: """Enable Or Disable Bridge""" + zh: """启用/禁用桥接""" + } + } + + desc_config { + desc { + en: """Configuration for an MicroSoft SQL Server bridge.""" + zh: """MicroSoft SQL Server 桥接配置""" + } + label: { + en: """MicroSoft SQL Server Bridge Configuration""" + zh: """MicroSoft SQL Server 桥接配置""" + } + } + + desc_type { + desc { + en: """The Bridge Type""" + zh: """Bridge 类型""" + } + label { + en: """Bridge Type""" + zh: """桥接类型""" + } + } + + desc_name { + desc { + en: """Bridge name.""" + zh: """桥接名字""" + } + label { + en: """Bridge Name""" + zh: """桥接名字""" + } + } + +} diff --git a/rel/i18n/emqx_ee_connector_sqlserver.hocon b/rel/i18n/emqx_ee_connector_sqlserver.hocon new file mode 100644 index 000000000..9ff24c1c1 --- /dev/null +++ b/rel/i18n/emqx_ee_connector_sqlserver.hocon @@ -0,0 +1,22 @@ +emqx_ee_connector_sqlserver { + + server { + desc { + en: """ +The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
+The SQL Server default port 1433 is used if `[:Port]` is not specified. +""" + zh: """ +将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 SQL Server 默认端口 1433。 +""" + } + label: { + en: "Server Host" + zh: "服务器地址" + } + } + +} From 611edf422719069297219370fe120968737fc0cf Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 10 Apr 2023 14:18:25 +0800 Subject: [PATCH 090/279] build: docker file add unixodbc and msodbcsql17 --- .gitignore | 1 + deploy/docker/Dockerfile | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 62e8ddc81..3f7e268a2 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,4 @@ apps/emqx/test/emqx_static_checks_data/master.bpapi lux_logs/ /.prepare bom.json +.dockerignore diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 8f04c433c..dcebe9b1a 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -29,9 +29,19 @@ COPY --from=builder /emqx-rel/emqx /opt/emqx RUN ln -s /opt/emqx/bin/* /usr/local/bin/ -RUN apt-get update; \ - apt-get install -y --no-install-recommends ca-certificates procps; \ - rm -rf /var/lib/apt/lists/* +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates procps + +RUN apt-get update \ + && apt-get install -y gnupg2 curl apt-utils \ + && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ + && curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list \ + && apt-get update \ + && ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev \ + && sed -i 's/ODBC Driver 17 for SQL Server/ms-sql/g' /etc/odbcinst.ini + +RUN apt-get clean && \ + rm -rf /var/lib/apt/lists/*; WORKDIR /opt/emqx From c366267b0faf7290ce5479690acfe2fefb0a2a9b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 7 Apr 2023 09:59:07 +0800 Subject: [PATCH 091/279] test: MS SQL Server data bridge --- .ci/docker-compose-file/.env | 3 + .../docker-compose-sqlserver.yaml | 19 + .../docker-compose-toxiproxy.yaml | 1 + .ci/docker-compose-file/docker-compose.yaml | 1 + .ci/docker-compose-file/odbc/odbcinst.ini | 10 + .ci/docker-compose-file/toxiproxy.json | 6 + lib-ee/emqx_ee_bridge/docker-ct | 1 + .../test/emqx_ee_bridge_sqlserver_SUITE.erl | 654 ++++++++++++++++++ lib-ee/emqx_ee_connector/docker-ct | 1 + scripts/ct/run.sh | 3 + 10 files changed, 699 insertions(+) create mode 100644 .ci/docker-compose-file/docker-compose-sqlserver.yaml create mode 100644 .ci/docker-compose-file/odbc/odbcinst.ini create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index 956750e00..b394d6bb1 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -8,4 +8,7 @@ TDENGINE_TAG=3.0.2.4 DYNAMO_TAG=1.21.0 CASSANDRA_TAG=3.11.6 +MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server +SQLSERVER_TAG=2019-CU11-ubuntu-20.04 + TARGET=emqx/emqx diff --git a/.ci/docker-compose-file/docker-compose-sqlserver.yaml b/.ci/docker-compose-file/docker-compose-sqlserver.yaml new file mode 100644 index 000000000..37799e20b --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-sqlserver.yaml @@ -0,0 +1,19 @@ +version: '3.9' + +services: + sql_server: + container_name: sqlserver + # See also: + # https://mcr.microsoft.com/en-us/product/mssql/server/about + # https://hub.docker.com/_/microsoft-mssql-server + image: ${MS_IMAGE_ADDR}:${SQLSERVER_TAG} + environment: + # See also: + # https://learn.microsoft.com/en-us/sql/linux/sql-server-linux-configure-environment-variables + ACCEPT_EULA: "Y" + SA_PASSWORD: "mqtt_public" + restart: always + # ports: + # - "1433:1433" + networks: + - emqx_bridge diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index 9a1d08ba6..ba5e831a5 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -16,6 +16,7 @@ services: - 8474:8474 - 8086:8086 - 8087:8087 + - 11433:1433 - 13306:3306 - 13307:3307 - 15432:5432 diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 48d900400..8f97fa9d6 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -24,6 +24,7 @@ services: - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kerberos/krb5.conf:/etc/kdc/krb5.conf - ./kerberos/krb5.conf:/etc/krb5.conf + - ./odbc/odbcinst.ini:/etc/odbcinst.ini working_dir: /emqx tty: true user: "${DOCKER_USER:-root}" diff --git a/.ci/docker-compose-file/odbc/odbcinst.ini b/.ci/docker-compose-file/odbc/odbcinst.ini new file mode 100644 index 000000000..16701cfd2 --- /dev/null +++ b/.ci/docker-compose-file/odbc/odbcinst.ini @@ -0,0 +1,10 @@ +[ms-sql] +Description=Microsoft ODBC Driver 17 for SQL Server +Driver=/opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.10.so.2.1 +UsageCount=1 + +[ODBC Driver 17 for SQL Server] +Description=Microsoft ODBC Driver 17 for SQL Server +Driver=/opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.10.so.2.1 +UsageCount=1 + diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 708cbf1ef..da2dff763 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -95,5 +95,11 @@ "listen": "0.0.0.0:9142", "upstream": "cassandra:9142", "enabled": true + }, + { + "name": "sqlserver", + "listen": "0.0.0.0:1433", + "upstream": "sqlserver:1433", + "enabled": true } ] diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index 0e947d89e..35d6b9d5b 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -11,3 +11,4 @@ clickhouse dynamo rocketmq cassandra +sqlserver diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl new file mode 100644 index 000000000..8b0cc53ce --- /dev/null +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -0,0 +1,654 @@ +%%-------------------------------------------------------------------- +% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_bridge_sqlserver_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +% SQL definitions +-define(SQL_BRIDGE, + "insert into t_mqtt_msg(msgid, topic, qos, payload) values ( ${id}, ${topic}, ${qos}, ${payload})" +). +-define(SQL_SERVER_DRIVER, "ms-sql"). + +-define(SQL_CREATE_DATABASE_IF_NOT_EXISTS, + " IF NOT EXISTS(SELECT name FROM sys.databases WHERE name = 'mqtt')" + " BEGIN" + " CREATE DATABASE mqtt;" + " END" +). + +-define(SQL_CREATE_TABLE_IN_DB_MQTT, + " CREATE TABLE mqtt.dbo.t_mqtt_msg" + " (id int PRIMARY KEY IDENTITY(1000000001,1) NOT NULL," + " msgid VARCHAR(64) NULL," + " topic VARCHAR(100) NULL," + " qos tinyint NOT NULL DEFAULT 0," + " payload NVARCHAR(100) NULL," + " arrived DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP)" +). + +-define(SQL_DROP_DB_MQTT, "DROP DATABASE mqtt"). +-define(SQL_DROP_TABLE, "DROP TABLE mqtt.dbo.t_mqtt_msg"). +-define(SQL_DELETE, "DELETE from mqtt.dbo.t_mqtt_msg"). +-define(SQL_SELECT, "SELECT payload FROM mqtt.dbo.t_mqtt_msg"). +-define(SQL_SELECT_COUNT, "SELECT COUNT(*) FROM mqtt.dbo.t_mqtt_msg"). +% DB defaults +-define(SQL_SERVER_DATABASE, "mqtt"). +-define(SQL_SERVER_USERNAME, "sa"). +-define(SQL_SERVER_PASSWORD, "mqtt_public"). +-define(BATCH_SIZE, 10). +-define(REQUEST_TIMEOUT_MS, 500). + +-define(WORKER_POOL_SIZE, 4). + +-define(WITH_CON(Process), + Con = connect_direct_sqlserver(Config), + Process, + ok = disconnect(Con) +). + +%% How to run it locally: +%% A: run ct on host +%% 1. Start all deps services +%% sudo docker compose -f .ci/docker-compose-file/docker-compose.yaml \ +%% -f .ci/docker-compose-file/docker-compose-sqlserver.yaml \ +%% -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ +%% up --build +%% +%% 2. Run use cases with special environment variables +%% 11433 is toxiproxy exported port +%% Local: +%% ``` +%% SQLSERVER_HOST=toxiproxy SQLSERVER_PORT=11433 \ +%% PROXY_HOST=toxiproxy PROXY_PORT=1433 \ +%% ./rebar3 as test ct -c -v --readable true --name ct@127.0.0.1 --suite lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +%% ``` +%% +%% B: run ct in docker container +%% run script: +%% ./scripts/ct/run.sh --ci --app lib-ee/emqx_ee_bridge/ \ +%% -- --name 'test@127.0.0.1' -c -v --readable true --suite lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, async}, + {group, sync} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + NonBatchCases = [t_write_timeout], + BatchingGroups = [{group, with_batch}, {group, without_batch}], + [ + {async, BatchingGroups}, + {sync, BatchingGroups}, + {with_batch, TCs -- NonBatchCases}, + {without_batch, TCs} + ]. + +init_per_group(async, Config) -> + [{query_mode, async} | Config]; +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; +init_per_group(with_batch, Config0) -> + Config = [{enable_batch, true} | Config0], + common_init(Config); +init_per_group(without_batch, Config0) -> + Config = [{enable_batch, false} | Config0], + common_init(Config); +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch -> + connect_and_drop_table(Config), + connect_and_drop_db(Config), + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok. + +init_per_testcase(_Testcase, Config) -> + %% drop database and table + %% connect_and_clear_table(Config), + %% create a new one + %% TODO: create a new database for each test case + delete_bridge(Config), + snabbkaffe:start_trace(), + Config. + +end_per_testcase(_Testcase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + connect_and_clear_table(Config), + ok = snabbkaffe:stop(), + delete_bridge(Config), + ok. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_setup_via_config_and_publish(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Val = str(erlang:unique_integer()), + SentData = sent_data(Val), + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, SentData)), + #{?snk_kind := sqlserver_connector_query_return}, + 10_000 + ), + ?assertMatch( + [{Val}], + connect_and_get_payload(Config) + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(sqlserver_connector_query_return, Trace0), + ?assertMatch([#{result := ok}], Trace), + ok + end + ), + ok. + +t_setup_via_http_api_and_publish(Config) -> + BridgeType = ?config(sqlserver_bridge_type, Config), + Name = ?config(sqlserver_name, Config), + SQLServerConfig0 = ?config(sqlserver_config, Config), + SQLServerConfig = SQLServerConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType + }, + ?assertMatch( + {ok, _}, + create_bridge_http(SQLServerConfig) + ), + Val = str(erlang:unique_integer()), + SentData = sent_data(Val), + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, SentData)), + #{?snk_kind := sqlserver_connector_query_return}, + 10_000 + ), + ?assertMatch( + [{Val}], + connect_and_get_payload(Config) + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(sqlserver_connector_query_return, Trace0), + ?assertMatch([#{result := ok}], Trace), + ok + end + ), + ok. + +t_get_status(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + + Name = ?config(sqlserver_name, Config), + BridgeType = ?config(sqlserver_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + case emqx_resource_manager:health_check(ResourceID) of + {ok, Status} when Status =:= disconnected orelse Status =:= connecting -> + ok; + {error, timeout} -> + ok; + Other -> + ?assert( + false, lists:flatten(io_lib:format("invalid health check result:~p~n", [Other])) + ) + end + end), + ok. + +t_create_disconnected(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + ?check_trace( + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch({ok, _}, create_bridge(Config)) + end), + fun(Trace) -> + ?assertMatch( + [#{error := {start_pool_failed, _, _}}], + ?of_kind(sqlserver_connector_start_failed, Trace) + ), + ok + end + ), + ok. + +t_write_failure(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + QueryMode = ?config(query_mode, Config), + Val = str(erlang:unique_integer()), + SentData = sent_data(Val), + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := resource_connected_enter}, + 20_000 + ), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + case QueryMode of + sync -> + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + send_message(Config, SentData) + ); + async -> + ?assertMatch( + ok, send_message(Config, SentData) + ) + end + end), + ok. + +t_write_timeout(_Config) -> + %% msodbc driver handled all connection exceptions + %% the case is same as t_write_failure/1 + ok. + +t_simple_query(Config) -> + BatchSize = batch_size(Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + {Requests, Vals} = gen_batch_req(BatchSize), + ?check_trace( + begin + ?wait_async_action( + begin + [?assertEqual(ok, query_resource(Config, Request)) || Request <- Requests] + end, + #{?snk_kind := sqlserver_connector_query_return}, + 10_000 + ), + %% just assert the data count is correct + ?assertMatch( + BatchSize, + connect_and_get_count(Config) + ), + %% assert the data order is correct + ?assertMatch( + Vals, + connect_and_get_payload(Config) + ) + end, + fun(Trace0) -> + Trace = ?of_kind(sqlserver_connector_query_return, Trace0), + case BatchSize of + 1 -> + ?assertMatch([#{result := ok}], Trace); + _ -> + [?assertMatch(#{result := ok}, Trace1) || Trace1 <- Trace] + end, + ok + end + ), + ok. + +-define(MISSING_TINYINT_ERROR, + "[Microsoft][ODBC Driver 17 for SQL Server][SQL Server]" + "Conversion failed when converting the varchar value 'undefined' to data type tinyint. SQLSTATE IS: 22018" +). + +t_missing_data(Config) -> + QueryMode = ?config(query_mode, Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Result = send_message(Config, #{}), + case QueryMode of + sync -> + ?assertMatch( + {error, {unrecoverable_error, {invalid_request, ?MISSING_TINYINT_ERROR}}}, + Result + ); + async -> + ?assertMatch( + ok, send_message(Config, #{}) + ) + end, + ok. + +t_bad_parameter(Config) -> + QueryMode = ?config(query_mode, Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Result = send_message(Config, #{}), + case QueryMode of + sync -> + ?assertMatch( + {error, {unrecoverable_error, {invalid_request, ?MISSING_TINYINT_ERROR}}}, + Result + ); + async -> + ?assertMatch( + ok, send_message(Config, #{}) + ) + end, + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +common_init(ConfigT) -> + Host = os:getenv("SQLSERVER_HOST", "toxiproxy"), + Port = list_to_integer(os:getenv("SQLSERVER_PORT", "1433")), + + Config0 = [ + {sqlserver_host, Host}, + {sqlserver_port, Port}, + %% see also for `proxy_name` : $PROJ_ROOT/.ci/docker-compose-file/toxiproxy.json + {proxy_name, "sqlserver"}, + {batch_size, ?BATCH_SIZE} + | ConfigT + ], + + BridgeType = proplists:get_value(bridge_type, Config0, <<"sqlserver">>), + case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of + true -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + % Ensure EE bridge module is loaded + _ = application:load(emqx_ee_bridge), + _ = emqx_ee_bridge:module_info(), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + emqx_mgmt_api_test_util:init_suite(), + % Connect to sqlserver directly + % drop old db and table, and then create new ones + connect_and_create_db_and_table(Config0), + {Name, SQLServerConf} = sqlserver_config(BridgeType, Config0), + Config = + [ + {sqlserver_config, SQLServerConf}, + {sqlserver_bridge_type, BridgeType}, + {sqlserver_name, Name}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort} + | Config0 + ], + Config; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_sqlserver); + _ -> + {skip, no_sqlserver} + end + end. + +sqlserver_config(BridgeType, Config) -> + Port = integer_to_list(?config(sqlserver_port, Config)), + Server = ?config(sqlserver_host, Config) ++ ":" ++ Port, + Name = atom_to_binary(?MODULE), + BatchSize = batch_size(Config), + QueryMode = ?config(query_mode, Config), + ConfigString = + io_lib:format( + "bridges.~s.~s {\n" + " enable = true\n" + " server = ~p\n" + " database = ~p\n" + " username = ~p\n" + " password = ~p\n" + " sql = ~p\n" + " driver = ~p\n" + " resource_opts = {\n" + " request_timeout = 500ms\n" + " batch_size = ~b\n" + " query_mode = ~s\n" + " worker_pool_size = ~b\n" + " }\n" + "}", + [ + BridgeType, + Name, + Server, + ?SQL_SERVER_DATABASE, + ?SQL_SERVER_USERNAME, + ?SQL_SERVER_PASSWORD, + ?SQL_BRIDGE, + ?SQL_SERVER_DRIVER, + BatchSize, + QueryMode, + ?WORKER_POOL_SIZE + ] + ), + {Name, parse_and_check(ConfigString, BridgeType, Name)}. + +parse_and_check(ConfigString, BridgeType, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf, + Config. + +create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> + BridgeType = ?config(sqlserver_bridge_type, Config), + Name = ?config(sqlserver_name, Config), + SSConfig0 = ?config(sqlserver_config, Config), + SSConfig = emqx_map_lib:deep_merge(SSConfig0, Overrides), + emqx_bridge:create(BridgeType, Name, SSConfig). + +delete_bridge(Config) -> + BridgeType = ?config(sqlserver_bridge_type, Config), + Name = ?config(sqlserver_name, Config), + emqx_bridge:remove(BridgeType, Name). + +create_bridge_http(Params) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + Error -> Error + end. + +send_message(Config, Payload) -> + Name = ?config(sqlserver_name, Config), + BridgeType = ?config(sqlserver_bridge_type, Config), + BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), + emqx_bridge:send_message(BridgeID, Payload). + +query_resource(Config, Request) -> + Name = ?config(sqlserver_name, Config), + BridgeType = ?config(sqlserver_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). + +query_resource_async(Config, Request) -> + Name = ?config(sqlserver_name, Config), + BridgeType = ?config(sqlserver_bridge_type, Config), + Ref = alias([reply]), + AsyncReplyFun = fun(Result) -> Ref ! {result, Ref, Result} end, + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + Return = emqx_resource:query(ResourceID, Request, #{ + timeout => 500, async_reply_fun => {AsyncReplyFun, []} + }), + {Return, Ref}. + +receive_result(Ref, Timeout) -> + receive + {result, Ref, Result} -> + {ok, Result}; + {Ref, Result} -> + {ok, Result} + after Timeout -> + timeout + end. + +connect_direct_sqlserver(Config) -> + Opts = [ + {host, ?config(sqlserver_host, Config)}, + {port, ?config(sqlserver_port, Config)}, + {username, ?SQL_SERVER_USERNAME}, + {password, ?SQL_SERVER_PASSWORD}, + {driver, ?SQL_SERVER_DRIVER}, + {pool_size, 8} + ], + {ok, Con} = connect(Opts), + Con. + +connect(Options) -> + ConnectStr = lists:concat(conn_str(Options, [])), + Opts = proplists:get_value(options, Options, []), + odbc:connect(ConnectStr, Opts). + +disconnect(Ref) -> + odbc:disconnect(Ref). + +% These funs connect and then stop the sqlserver connection +connect_and_create_db_and_table(Config) -> + ?WITH_CON(begin + {updated, undefined} = directly_query(Con, ?SQL_CREATE_DATABASE_IF_NOT_EXISTS), + {updated, undefined} = directly_query(Con, ?SQL_CREATE_TABLE_IN_DB_MQTT) + end). + +connect_and_drop_db(Config) -> + ?WITH_CON({updated, undefined} = directly_query(Con, ?SQL_DROP_DB_MQTT)). + +connect_and_drop_table(Config) -> + ?WITH_CON({updated, undefined} = directly_query(Con, ?SQL_DROP_TABLE)). + +connect_and_clear_table(Config) -> + ?WITH_CON({updated, _} = directly_query(Con, ?SQL_DELETE)). + +connect_and_get_payload(Config) -> + ?WITH_CON( + {selected, ["payload"], Rows} = directly_query(Con, ?SQL_SELECT) + ), + Rows. + +connect_and_get_count(Config) -> + ?WITH_CON( + {selected, [[]], [{Count}]} = directly_query(Con, ?SQL_SELECT_COUNT) + ), + Count. + +directly_query(Con, Query) -> + directly_query(Con, Query, ?REQUEST_TIMEOUT_MS). + +directly_query(Con, Query, Timeout) -> + odbc:sql_query(Con, Query, Timeout). + +%%-------------------------------------------------------------------- +%% help functions +%%-------------------------------------------------------------------- + +batch_size(Config) -> + case ?config(enable_batch, Config) of + true -> ?BATCH_SIZE; + false -> 1 + end. + +conn_str([], Acc) -> + %% TODO: for msodbc 18+, we need to add "Encrypt=YES;TrustServerCertificate=YES" + %% but havn't tested now + %% we should use this for msodbcsql 18+ + %% lists:join(";", ["Encrypt=YES", "TrustServerCertificate=YES" | Acc]); + lists:join(";", Acc); +conn_str([{driver, Driver} | Opts], Acc) -> + conn_str(Opts, ["Driver=" ++ str(Driver) | Acc]); +conn_str([{host, Host} | Opts], Acc) -> + Port = proplists:get_value(port, Opts, "1433"), + NOpts = proplists:delete(port, Opts), + conn_str(NOpts, ["Server=" ++ str(Host) ++ "," ++ str(Port) | Acc]); +conn_str([{port, Port} | Opts], Acc) -> + Host = proplists:get_value(host, Opts, "localhost"), + NOpts = proplists:delete(host, Opts), + conn_str(NOpts, ["Server=" ++ str(Host) ++ "," ++ str(Port) | Acc]); +conn_str([{database, Database} | Opts], Acc) -> + conn_str(Opts, ["Database=" ++ str(Database) | Acc]); +conn_str([{username, Username} | Opts], Acc) -> + conn_str(Opts, ["UID=" ++ str(Username) | Acc]); +conn_str([{password, Password} | Opts], Acc) -> + conn_str(Opts, ["PWD=" ++ str(Password) | Acc]); +conn_str([{_, _} | Opts], Acc) -> + conn_str(Opts, Acc). + +sent_data(Payload) -> + #{ + payload => to_bin(Payload), + id => <<"0005F8F84FFFAFB9F44200000D810002">>, + topic => <<"test/topic">>, + qos => 0 + }. + +gen_batch_req(Count) when + is_integer(Count) andalso Count > 0 +-> + Vals = [{str(erlang:unique_integer())} || _Seq <- lists:seq(1, Count)], + Requests = [{send_message, sent_data(Payload)} || {Payload} <- Vals], + {Requests, Vals}; +gen_batch_req(Count) -> + ct:pal("Gen batch requests failed with unexpected Count: ~p", [Count]). + +str(List) when is_list(List) -> + unicode:characters_to_list(List, utf8); +str(Bin) when is_binary(Bin) -> + unicode:characters_to_list(Bin, utf8); +str(Num) when is_number(Num) -> + number_to_list(Num). + +number_to_list(Int) when is_integer(Int) -> + integer_to_list(Int); +number_to_list(Float) when is_float(Float) -> + float_to_list(Float, [{decimals, 10}, compact]). + +to_bin(List) when is_list(List) -> + unicode:characters_to_binary(List, utf8); +to_bin(Bin) when is_binary(Bin) -> + Bin. diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct index 446c8ee4e..fc8e75e68 100644 --- a/lib-ee/emqx_ee_connector/docker-ct +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -2,3 +2,4 @@ toxiproxy influxdb clickhouse cassandra +sqlserver diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 83e69d1ad..17a30e08b 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -180,6 +180,9 @@ for dep in ${CT_DEPS}; do cassandra) FILES+=( '.ci/docker-compose-file/docker-compose-cassandra.yaml' ) ;; + sqlserver) + FILES+=( '.ci/docker-compose-file/docker-compose-sqlserver.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From a379d909bfb6101ccc55f32e04ffa9dea3ac3b89 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 10 Apr 2023 16:44:21 +0800 Subject: [PATCH 092/279] test: use type VARCHAR to use utf8 encoding in sqlserver --- lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl index 8b0cc53ce..27a85d10f 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -30,7 +30,9 @@ " msgid VARCHAR(64) NULL," " topic VARCHAR(100) NULL," " qos tinyint NOT NULL DEFAULT 0," - " payload NVARCHAR(100) NULL," + %% use VARCHAR to use utf8 encoding + %% for default, sqlserver use utf16 encoding NVARCHAR() + " payload VARCHAR(100) NULL," " arrived DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP)" ). From 27fdf644aa5fa75906b5791c6d8cba08f96ae9fd Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 10 Apr 2023 16:18:29 +0800 Subject: [PATCH 093/279] fix(sqlserver): let a single query be a length 1 batch Need to be fixed: `emqx_plugin_libs_rule:proc_tmpl/2` won't add single quotes for string --- .../src/emqx_ee_connector_sqlserver.erl | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index 4cc92e80a..214663b22 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -502,15 +502,18 @@ parse_sql_template([], SingleInsertTks, BatchInsertTks) -> %% single insert apply_template( - {?ACTION_SEND_MESSAGE = Key, Msg} = Query, - #{?SINGLE_INSERT_TEMP := SingleInsertTks} = _Templates + {?ACTION_SEND_MESSAGE = _Key, _Msg} = Query, + #{?SINGLE_INSERT_TEMP := _SingleInsertTks} = Templates ) -> - case maps:get(Key, SingleInsertTks, undefined) of - undefined -> - Query; - Template -> - {Key, emqx_plugin_libs_rule:proc_tmpl(Template, Msg)} - end; + %% TODO: fix emqx_plugin_libs_rule:proc_tmpl/2 + %% it won't add single quotes for string + apply_template([Query], Templates); +%% case maps:get(Key, SingleInsertTks, undefined) of +%% undefined -> +%% Query; +%% Template -> +%% {Key, emqx_plugin_libs_rule:proc_tmpl(Template, Msg)} +%% end; %% batch inserts apply_template( [{?ACTION_SEND_MESSAGE = Key, _Msg} | _T] = BatchReqs, From 59433b8b33288bf8b9abcb0a2b708350efaa9eaa Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Apr 2023 03:41:40 +0800 Subject: [PATCH 094/279] ci: install msodbcsql17 drier data bridge --- scripts/install-odbc-driver.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 scripts/install-odbc-driver.sh diff --git a/scripts/install-odbc-driver.sh b/scripts/install-odbc-driver.sh new file mode 100755 index 000000000..c7b94cf13 --- /dev/null +++ b/scripts/install-odbc-driver.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +## this script install msodbcsql17 and unixodbc-dev on ci environment +set -euo pipefail + +# install msodbcsql17 +curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ +curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list && \ +ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev && \ +## and not needed to modify /etc/odbcinst.ini +## docker-compose will mount one in .ci/docker-compose-file/odbc +sed -i 's/ODBC Driver 17 for SQL Server/ms-sql/g' /etc/odbcinst.ini From d7ad07f1dc27dca9f20fc980163d036d203bc754 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Apr 2023 04:17:04 +0800 Subject: [PATCH 095/279] ci: install odbc driver for ms sqlserver bridge --- scripts/ct/run.sh | 9 +++++++++ scripts/install-odbc-driver.sh | 2 ++ 2 files changed, 11 insertions(+) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 17a30e08b..4e79476e0 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -39,6 +39,7 @@ ONLY_UP='no' ATTACH='no' STOP='no' IS_CI='no' +ODBC_REQUEST='no' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -181,6 +182,7 @@ for dep in ${CT_DEPS}; do FILES+=( '.ci/docker-compose-file/docker-compose-cassandra.yaml' ) ;; sqlserver) + ODBC_REQUEST='yes' FILES+=( '.ci/docker-compose-file/docker-compose-sqlserver.yaml' ) ;; *) @@ -190,6 +192,12 @@ for dep in ${CT_DEPS}; do esac done +if [ "$ODBC_REQUEST" = 'yes' ]; then + INSTALL_ODBC="./scripts/install-odbc-driver.sh" +else + INSTALL_ODBC="echo 'Driver msodbcsql driver not requested'" +fi + F_OPTIONS="" for file in "${FILES[@]}"; do @@ -232,6 +240,7 @@ docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base # the user must exist inside the container for `whoami` to work docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "useradd --uid $DOCKER_USER -M -d / emqx" || true docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "chown -R $DOCKER_USER /var/lib/secret" || true +docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "$INSTALL_ODBC" || true if [ "$ONLY_UP" = 'yes' ]; then exit 0 diff --git a/scripts/install-odbc-driver.sh b/scripts/install-odbc-driver.sh index c7b94cf13..b1fd894a0 100755 --- a/scripts/install-odbc-driver.sh +++ b/scripts/install-odbc-driver.sh @@ -6,6 +6,8 @@ set -euo pipefail # install msodbcsql17 curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list && \ +## TODO: upgrade builder image +apt-get update && \ ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev && \ ## and not needed to modify /etc/odbcinst.ini ## docker-compose will mount one in .ci/docker-compose-file/odbc From 437096985b7c43d02ae2cccebc19166c284a689b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Apr 2023 04:39:15 +0800 Subject: [PATCH 096/279] test: fix SUITE case `t_create_disconnected/1` --- .../test/emqx_ee_bridge_sqlserver_SUITE.erl | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl index 27a85d10f..7a9c1baf9 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -224,22 +224,9 @@ t_get_status(Config) -> ProxyHost = ?config(proxy_host, Config), ProxyName = ?config(proxy_name, Config), - Name = ?config(sqlserver_name, Config), - BridgeType = ?config(sqlserver_bridge_type, Config), - ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - - ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)), + health_check_resource_ok(Config), emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> - case emqx_resource_manager:health_check(ResourceID) of - {ok, Status} when Status =:= disconnected orelse Status =:= connecting -> - ok; - {error, timeout} -> - ok; - Other -> - ?assert( - false, lists:flatten(io_lib:format("invalid health check result:~p~n", [Other])) - ) - end + health_check_resource_down(Config) end), ok. @@ -247,18 +234,11 @@ t_create_disconnected(Config) -> ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), ProxyName = ?config(proxy_name, Config), - ?check_trace( - emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> - ?assertMatch({ok, _}, create_bridge(Config)) - end), - fun(Trace) -> - ?assertMatch( - [#{error := {start_pool_failed, _, _}}], - ?of_kind(sqlserver_connector_start_failed, Trace) - ), - ok - end - ), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch({ok, _}, create_bridge(Config)), + health_check_resource_down(Config) + end), + health_check_resource_ok(Config), ok. t_write_failure(Config) -> @@ -522,6 +502,26 @@ query_resource_async(Config, Request) -> }), {Return, Ref}. +resource_id(Config) -> + Name = ?config(sqlserver_name, Config), + BridgeType = ?config(sqlserver_bridge_type, Config), + _ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name). + +health_check_resource_ok(Config) -> + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(resource_id(Config))). + +health_check_resource_down(Config) -> + case emqx_resource_manager:health_check(resource_id(Config)) of + {ok, Status} when Status =:= disconnected orelse Status =:= connecting -> + ok; + {error, timeout} -> + ok; + Other -> + ?assert( + false, lists:flatten(io_lib:format("invalid health check result:~p~n", [Other])) + ) + end. + receive_result(Ref, Timeout) -> receive {result, Ref, Result} -> From 57505cdfb3fc84c559e47912e3cff9e016eed214 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Apr 2023 05:15:24 +0800 Subject: [PATCH 097/279] ci: unmount odbcinst.ini file --- .ci/docker-compose-file/docker-compose.yaml | 2 +- .ci/docker-compose-file/odbc/odbcinst.ini | 1 - scripts/install-odbc-driver.sh | 11 +++++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 8f97fa9d6..6f4b7c04b 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -24,7 +24,7 @@ services: - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret - ./kerberos/krb5.conf:/etc/kdc/krb5.conf - ./kerberos/krb5.conf:/etc/krb5.conf - - ./odbc/odbcinst.ini:/etc/odbcinst.ini + # - ./odbc/odbcinst.ini:/etc/odbcinst.ini working_dir: /emqx tty: true user: "${DOCKER_USER:-root}" diff --git a/.ci/docker-compose-file/odbc/odbcinst.ini b/.ci/docker-compose-file/odbc/odbcinst.ini index 16701cfd2..dd0241543 100644 --- a/.ci/docker-compose-file/odbc/odbcinst.ini +++ b/.ci/docker-compose-file/odbc/odbcinst.ini @@ -7,4 +7,3 @@ UsageCount=1 Description=Microsoft ODBC Driver 17 for SQL Server Driver=/opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.10.so.2.1 UsageCount=1 - diff --git a/scripts/install-odbc-driver.sh b/scripts/install-odbc-driver.sh index b1fd894a0..4ca71d88d 100755 --- a/scripts/install-odbc-driver.sh +++ b/scripts/install-odbc-driver.sh @@ -1,14 +1,21 @@ #!/usr/bin/env bash ## this script install msodbcsql17 and unixodbc-dev on ci environment +## specific to ubuntu 16.04, 18.04, 20.04, 22.04 set -euo pipefail # install msodbcsql17 +if ! [[ "16.04 18.04 20.04 22.04" == *"$(lsb_release -rs)"* ]]; +then + echo "Ubuntu $(lsb_release -rs) is not currently supported."; + exit 1; +fi + curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ -curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list && \ +curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list > /etc/apt/sources.list.d/mssql-release.list && \ ## TODO: upgrade builder image apt-get update && \ -ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev && \ +ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev mssql-tools && \ ## and not needed to modify /etc/odbcinst.ini ## docker-compose will mount one in .ci/docker-compose-file/odbc sed -i 's/ODBC Driver 17 for SQL Server/ms-sql/g' /etc/odbcinst.ini From f8c0aa49a240b65114edb99be2f1e68401cebc56 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Apr 2023 09:54:58 +0800 Subject: [PATCH 098/279] style: make elvis and shellcheck happy --- .../emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl | 6 ++++-- scripts/install-odbc-driver.sh | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index 214663b22..55a5d488f 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -342,9 +342,11 @@ do_get_status(Conn, RequestTimeout) -> %% About the connection string attribute `Encrypt`: %% The default value is `yes` in odbc version 18.0+ and `no` in previous versions. %% And encrypted connections always verify the server's certificate. -%% So `Encrypt=YES;TrustServerCertificate=YES` must be set in the connection string when connecting to a server that has a self-signed certificate. +%% So `Encrypt=YES;TrustServerCertificate=YES` must be set in the connection string +%% when connecting to a server that has a self-signed certificate. %% See also: -%% https://learn.microsoft.com/en-us/sql/connect/odbc/dsn-connection-string-attribute?source=recommendations&view=sql-server-ver16#encrypt +%% 'https://learn.microsoft.com/en-us/sql/connect/odbc/ +%% dsn-connection-string-attribute?source=recommendations&view=sql-server-ver16#encrypt' conn_str([], Acc) -> %% we should use this for msodbcsql 18+ %% lists:join(";", ["Encrypt=YES", "TrustServerCertificate=YES" | Acc]); diff --git a/scripts/install-odbc-driver.sh b/scripts/install-odbc-driver.sh index 4ca71d88d..7ceab95a0 100755 --- a/scripts/install-odbc-driver.sh +++ b/scripts/install-odbc-driver.sh @@ -5,14 +5,15 @@ set -euo pipefail # install msodbcsql17 -if ! [[ "16.04 18.04 20.04 22.04" == *"$(lsb_release -rs)"* ]]; +VERSION=$(lsb_release -rs) +if ! [[ "16.04 18.04 20.04 22.04" == *"$VERSION"* ]]; then - echo "Ubuntu $(lsb_release -rs) is not currently supported."; + echo "Ubuntu $VERSION is not currently supported."; exit 1; fi curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ -curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list > /etc/apt/sources.list.d/mssql-release.list && \ +curl https://packages.microsoft.com/config/ubuntu/"$VERSION"/prod.list > /etc/apt/sources.list.d/mssql-release.list && \ ## TODO: upgrade builder image apt-get update && \ ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev mssql-tools && \ From 675f9894ab5335bdd867ae6803bf47852faf9bf3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Apr 2023 10:43:35 +0800 Subject: [PATCH 099/279] fix: fix single quotes not applied for single query - alwasy use `emqx_plugin_libs_rule:proc_batch_sql/3` to add single quotes for string --- .../src/emqx_ee_connector_sqlserver.erl | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index 55a5d488f..99b5c2809 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -50,8 +50,6 @@ -define(ACTION_SEND_MESSAGE, send_message). --define(SINGLE_INSERT, single_insert). - -define(SYNC_QUERY_MODE, handover). -define(ASYNC_QUERY_MODE(REPLY), {handover_async, {?MODULE, do_async_reply, [REPLY]}}). @@ -63,7 +61,6 @@ maps:get(request_timeout, RESOURCE_OPTS, ?DEFAULT_REQUEST_TIMEOUT) ). --define(SINGLE_INSERT_TEMP, single_insert_temp). -define(BATCH_INSERT_TEMP, batch_insert_temp). -define(BATCH_INSERT_PART, batch_insert_part). @@ -464,20 +461,18 @@ parse_sql_template(Config) -> SQLTemplate -> #{?ACTION_SEND_MESSAGE => SQLTemplate} end, - SingleInsertTks = #{}, BatchInsertTks = #{}, - parse_sql_template(maps:to_list(RawSQLTemplates), SingleInsertTks, BatchInsertTks). + parse_sql_template(maps:to_list(RawSQLTemplates), BatchInsertTks). -parse_sql_template([{Key, H} | T], SingleInsertTks, BatchInsertTks) -> +parse_sql_template([{Key, H} | T], BatchInsertTks) -> case emqx_plugin_libs_rule:detect_sql_type(H) of {ok, select} -> - parse_sql_template(T, SingleInsertTks, BatchInsertTks); + parse_sql_template(T, BatchInsertTks); {ok, insert} -> case emqx_plugin_libs_rule:split_insert_sql(H) of {ok, {InsertSQL, Params}} -> parse_sql_template( T, - SingleInsertTks#{Key => emqx_plugin_libs_rule:preproc_tmpl(H)}, BatchInsertTks#{ Key => #{ @@ -490,32 +485,24 @@ parse_sql_template([{Key, H} | T], SingleInsertTks, BatchInsertTks) -> ); {error, Reason} -> ?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}), - parse_sql_template(T, SingleInsertTks, BatchInsertTks) + parse_sql_template(T, BatchInsertTks) end; {error, Reason} -> ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}), - parse_sql_template(T, SingleInsertTks, BatchInsertTks) + parse_sql_template(T, BatchInsertTks) end; -parse_sql_template([], SingleInsertTks, BatchInsertTks) -> +parse_sql_template([], BatchInsertTks) -> #{ - ?SINGLE_INSERT_TEMP => SingleInsertTks, ?BATCH_INSERT_TEMP => BatchInsertTks }. %% single insert apply_template( - {?ACTION_SEND_MESSAGE = _Key, _Msg} = Query, - #{?SINGLE_INSERT_TEMP := _SingleInsertTks} = Templates + {?ACTION_SEND_MESSAGE = _Key, _Msg} = Query, Templates ) -> %% TODO: fix emqx_plugin_libs_rule:proc_tmpl/2 %% it won't add single quotes for string apply_template([Query], Templates); -%% case maps:get(Key, SingleInsertTks, undefined) of -%% undefined -> -%% Query; -%% Template -> -%% {Key, emqx_plugin_libs_rule:proc_tmpl(Template, Msg)} -%% end; %% batch inserts apply_template( [{?ACTION_SEND_MESSAGE = Key, _Msg} | _T] = BatchReqs, @@ -530,7 +517,8 @@ apply_template( end; apply_template(Query, Templates) -> %% TODO: more detail infomatoin - ?SLOG(error, #{msg => "apply sql template failed", query => Query, templates => Templates}). + ?SLOG(error, #{msg => "apply sql template failed", query => Query, templates => Templates}), + {error, failed_to_apply_sql_template}. do_async_reply(Result, {ReplyFun, Args}) -> erlang:apply(ReplyFun, Args ++ Result). From 35ab1c01900ada9d1b1610c0e1d325babc5a664d Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Apr 2023 11:11:09 +0800 Subject: [PATCH 100/279] style: make hocon style check and spellcheck happy - with more human readable i18n style check format --- changes/ee/feat-10363.en.md | 2 +- rel/i18n/emqx_ee_bridge_dynamo.hocon | 2 +- rel/i18n/emqx_ee_bridge_pgsql.hocon | 2 +- rel/i18n/emqx_ee_bridge_sqlserver.hocon | 18 ++++++++---------- rel/i18n/emqx_ee_bridge_tdengine.hocon | 2 +- rel/i18n/emqx_ee_connector_sqlserver.hocon | 12 ++++-------- scripts/check-i18n-style.escript | 6 +++--- 7 files changed, 19 insertions(+), 25 deletions(-) diff --git a/changes/ee/feat-10363.en.md b/changes/ee/feat-10363.en.md index c3b53a538..84816031d 100644 --- a/changes/ee/feat-10363.en.md +++ b/changes/ee/feat-10363.en.md @@ -1 +1 @@ -Add MicroSoft SQL Server data Bridge support. +Implement Microsoft SQL Server bridge. diff --git a/rel/i18n/emqx_ee_bridge_dynamo.hocon b/rel/i18n/emqx_ee_bridge_dynamo.hocon index 664b13174..b93b12166 100644 --- a/rel/i18n/emqx_ee_bridge_dynamo.hocon +++ b/rel/i18n/emqx_ee_bridge_dynamo.hocon @@ -39,7 +39,7 @@ will be forwarded.""" desc_config { desc { - en: """Configuration for an DynamoDB bridge.""" + en: """Configuration for a DynamoDB bridge.""" zh: """DynamoDB 桥接配置""" } label: { diff --git a/rel/i18n/emqx_ee_bridge_pgsql.hocon b/rel/i18n/emqx_ee_bridge_pgsql.hocon index 81d2330c5..1ce2818ca 100644 --- a/rel/i18n/emqx_ee_bridge_pgsql.hocon +++ b/rel/i18n/emqx_ee_bridge_pgsql.hocon @@ -39,7 +39,7 @@ will be forwarded.""" desc_config { desc { - en: """Configuration for an PostgreSQL bridge.""" + en: """Configuration for a PostgreSQL bridge.""" zh: """PostgreSQL 桥接配置""" } label: { diff --git a/rel/i18n/emqx_ee_bridge_sqlserver.hocon b/rel/i18n/emqx_ee_bridge_sqlserver.hocon index bb82c82f8..007b64ae6 100644 --- a/rel/i18n/emqx_ee_bridge_sqlserver.hocon +++ b/rel/i18n/emqx_ee_bridge_sqlserver.hocon @@ -2,15 +2,13 @@ emqx_ee_bridge_sqlserver { local_topic { desc { - en: """The MQTT topic filter to be forwarded to MicroSoft SQL Server. All MQTT 'PUBLISH' messages with the topic + en: """The MQTT topic filter to be forwarded to Microsoft SQL Server. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic -will be forwarded. -""" - zh: """发送到 'local_topic' 的消息都会转发到 MicroSoft SQL Server。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。 -""" +will be forwarded.""" + zh: """发送到 'local_topic' 的消息都会转发到 Microsoft SQL Server。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" } label { en: """Local Topic""" @@ -53,12 +51,12 @@ will be forwarded. desc_config { desc { - en: """Configuration for an MicroSoft SQL Server bridge.""" - zh: """MicroSoft SQL Server 桥接配置""" + en: """Configuration for a Microsoft SQL Server bridge.""" + zh: """Microsoft SQL Server 桥接配置""" } label: { - en: """MicroSoft SQL Server Bridge Configuration""" - zh: """MicroSoft SQL Server 桥接配置""" + en: """Microsoft SQL Server Bridge Configuration""" + zh: """Microsoft SQL Server 桥接配置""" } } diff --git a/rel/i18n/emqx_ee_bridge_tdengine.hocon b/rel/i18n/emqx_ee_bridge_tdengine.hocon index 21fc013df..12eb6f062 100644 --- a/rel/i18n/emqx_ee_bridge_tdengine.hocon +++ b/rel/i18n/emqx_ee_bridge_tdengine.hocon @@ -39,7 +39,7 @@ will be forwarded.""" desc_config { desc { - en: """Configuration for an TDengine bridge.""" + en: """Configuration for a TDengine bridge.""" zh: """TDengine 桥接配置""" } label: { diff --git a/rel/i18n/emqx_ee_connector_sqlserver.hocon b/rel/i18n/emqx_ee_connector_sqlserver.hocon index 9ff24c1c1..85280d833 100644 --- a/rel/i18n/emqx_ee_connector_sqlserver.hocon +++ b/rel/i18n/emqx_ee_connector_sqlserver.hocon @@ -2,16 +2,12 @@ emqx_ee_connector_sqlserver { server { desc { - en: """ -The IPv4 or IPv6 address or the hostname to connect to.
+ en: """The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
-The SQL Server default port 1433 is used if `[:Port]` is not specified. -""" - zh: """ -将要连接的 IPv4 或 IPv6 地址，或者主机名。
+The SQL Server default port 1433 is used if `[:Port]` is not specified.""" + zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 SQL Server 默认端口 1433。 -""" +如果未指定 `[:Port]`，则使用 SQL Server 默认端口 1433。""" } label: { en: "Server Host" diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index cbe79c82e..a76fce90e 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -9,7 +9,7 @@ -define(RESET, "\e[39m"). main([Files0]) -> - io:format(user, "checking i18n file styles", []), + io:format(user, "checking i18n file styles~n", []), _ = put(errors, 0), Files = string:tokens(Files0, "\n"), ok = load_hocon(), @@ -20,7 +20,7 @@ main([Files0]) -> N when is_integer(N) andalso N > 1 -> logerr("~p errors found~n", [N]); _ -> - io:format(user, "OK~n", []) + io:format(user, "~nOK~n", []) end. load_hocon() -> @@ -42,7 +42,7 @@ die(Msg, Args) -> halt(1). logerr(Fmt, Args) -> - io:format(standard_error, ?RED ++ "ERROR: " ++ Fmt ++ ?RESET, Args), + io:format(standard_error, "~n" ++ ?RED ++ "ERROR: " ++ Fmt ++ ?RESET, Args), N = get(errors), _ = put(errors, N + 1), ok. From 1b2cf6d421f7e3dae05ec1be1ccaa1e057b449b5 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 13 Apr 2023 17:58:06 +0800 Subject: [PATCH 101/279] ci: newest SQL Serevr 2019 image and rm deprecated env `SA_PASSWORD` --- .ci/docker-compose-file/.env | 2 +- .ci/docker-compose-file/docker-compose-sqlserver.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index b394d6bb1..d33637ea0 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -9,6 +9,6 @@ DYNAMO_TAG=1.21.0 CASSANDRA_TAG=3.11.6 MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server -SQLSERVER_TAG=2019-CU11-ubuntu-20.04 +SQLSERVER_TAG=2019-CU19-ubuntu-20.04 TARGET=emqx/emqx diff --git a/.ci/docker-compose-file/docker-compose-sqlserver.yaml b/.ci/docker-compose-file/docker-compose-sqlserver.yaml index 37799e20b..a578983f8 100644 --- a/.ci/docker-compose-file/docker-compose-sqlserver.yaml +++ b/.ci/docker-compose-file/docker-compose-sqlserver.yaml @@ -11,7 +11,7 @@ services: # See also: # https://learn.microsoft.com/en-us/sql/linux/sql-server-linux-configure-environment-variables ACCEPT_EULA: "Y" - SA_PASSWORD: "mqtt_public" + MSSQL_SA_PASSWORD: "mqtt_public" restart: always # ports: # - "1433:1433" From 2d1b94f7f79e841802e2e3de2e898f4a8ff90af0 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 14 Apr 2023 01:34:30 +0800 Subject: [PATCH 102/279] fix: file mode 755 magic number --- .../src/emqx_ee_connector_sqlserver.erl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index 99b5c2809..e48570c96 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -66,6 +66,11 @@ -define(BATCH_INSERT_PART, batch_insert_part). -define(BATCH_PARAMS_TOKENS, batch_insert_tks). +-define(FILE_MODE_755, 33261). +%% 32768 + 8#00400 + 8#00200 + 8#00100 + 8#00040 + 8#00010 + 8#00004 + 8#00001 +%% See also +%% https://www.erlang.org/doc/man/file.html#read_file_info-2 + %% Copied from odbc reference page %% https://www.erlang.org/doc/man/odbc.html @@ -190,11 +195,11 @@ on_start( ODBCDir = code:priv_dir(odbc), OdbcserverDir = filename:join(ODBCDir, "bin/odbcserver"), {ok, Info = #file_info{mode = Mode}} = file:read_file_info(OdbcserverDir), - case 33261 =:= Mode of + case ?FILE_MODE_755 =:= Mode of true -> ok; false -> - _ = file:write_file_info(OdbcserverDir, Info#file_info{mode = 33261}), + _ = file:write_file_info(OdbcserverDir, Info#file_info{mode = ?FILE_MODE_755}), ok end, From 65c4234829a9f52758803b4987a8275e8e90a24d Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 14 Apr 2023 01:39:08 +0800 Subject: [PATCH 103/279] refactor: rename instance_id, which is also PoolName --- .../src/emqx_ee_connector_sqlserver.erl | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index e48570c96..586444947 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -74,7 +74,6 @@ %% Copied from odbc reference page %% https://www.erlang.org/doc/man/odbc.html --type bridge_id() :: binary(). %% as returned by connect/2 -type connection_reference() :: pid(). -type time_out() :: milliseconds() | infinity. @@ -175,7 +174,7 @@ callback_mode() -> async_if_possible. is_buffer_supported() -> false. on_start( - BridgeId = PoolName, + InstanceId = PoolName, #{ server := Server, username := Username, @@ -188,7 +187,7 @@ on_start( ) -> ?SLOG(info, #{ msg => "starting_sqlserver_connector", - connector => BridgeId, + connector => InstanceId, config => emqx_misc:redact(Config) }), @@ -214,7 +213,7 @@ on_start( ], State = #{ - %% also BridgeId + %% also InstanceId poolname => PoolName, sql_templates => parse_sql_template(Config), resource_opts => ResourceOpts @@ -238,7 +237,7 @@ on_stop(InstanceId, #{poolname := PoolName} = _State) -> emqx_plugin_libs_pool:stop_pool(PoolName). -spec on_query( - bridge_id(), + manager_id(), {?ACTION_SEND_MESSAGE, map()}, state() ) -> @@ -246,16 +245,16 @@ on_stop(InstanceId, #{poolname := PoolName} = _State) -> | {ok, list()} | {error, {recoverable_error, term()}} | {error, term()}. -on_query(BridgeId, {?ACTION_SEND_MESSAGE, _Msg} = Query, State) -> +on_query(InstanceId, {?ACTION_SEND_MESSAGE, _Msg} = Query, State) -> ?TRACE( "SINGLE_QUERY_SYNC", "bridge_sqlserver_received", - #{requests => Query, connector => BridgeId, state => State} + #{requests => Query, connector => InstanceId, state => State} ), - do_query(BridgeId, Query, ?SYNC_QUERY_MODE, State). + do_query(InstanceId, Query, ?SYNC_QUERY_MODE, State). -spec on_query_async( - bridge_id(), + manager_id(), {?ACTION_SEND_MESSAGE, map()}, {ReplyFun :: function(), Args :: list()}, state() @@ -263,7 +262,7 @@ on_query(BridgeId, {?ACTION_SEND_MESSAGE, _Msg} = Query, State) -> {ok, any()} | {error, term()}. on_query_async( - BridgeId, + InstanceId, {?ACTION_SEND_MESSAGE, _Msg} = Query, ReplyFunAndArgs, %% #{poolname := PoolName, sql_templates := Templates} = State @@ -272,12 +271,12 @@ on_query_async( ?TRACE( "SINGLE_QUERY_ASYNC", "bridge_sqlserver_received", - #{requests => Query, connector => BridgeId, state => State} + #{requests => Query, connector => InstanceId, state => State} ), - do_query(BridgeId, Query, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). + do_query(InstanceId, Query, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). -spec on_batch_query( - bridge_id(), + manager_id(), [{?ACTION_SEND_MESSAGE, map()}], state() ) -> @@ -285,27 +284,27 @@ on_query_async( | {ok, list()} | {error, {recoverable_error, term()}} | {error, term()}. -on_batch_query(BridgeId, BatchRequests, State) -> +on_batch_query(InstanceId, BatchRequests, State) -> ?TRACE( "BATCH_QUERY_SYNC", "bridge_sqlserver_received", - #{requests => BatchRequests, connector => BridgeId, state => State} + #{requests => BatchRequests, connector => InstanceId, state => State} ), - do_query(BridgeId, BatchRequests, ?SYNC_QUERY_MODE, State). + do_query(InstanceId, BatchRequests, ?SYNC_QUERY_MODE, State). -spec on_batch_query_async( - bridge_id(), + manager_id(), [{?ACTION_SEND_MESSAGE, map()}], {ReplyFun :: function(), Args :: list()}, state() ) -> {ok, any()}. -on_batch_query_async(BridgeId, Requests, ReplyFunAndArgs, State) -> +on_batch_query_async(InstanceId, Requests, ReplyFunAndArgs, State) -> ?TRACE( "BATCH_QUERY_ASYNC", "bridge_sqlserver_received", - #{requests => Requests, connector => BridgeId, state => State} + #{requests => Requests, connector => InstanceId, state => State} ), - do_query(BridgeId, Requests, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). + do_query(InstanceId, Requests, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). on_get_status(_InstanceId, #{poolname := Pool, resource_opts := ResourceOpts} = _State) -> RequestTimeout = ?REQUEST_TIMEOUT(ResourceOpts), @@ -369,7 +368,7 @@ conn_str([{_, _} | Opts], Acc) -> %% Sync & Async query with singe & batch sql statement -spec do_query( - bridge_id(), + manager_id(), Query :: {?ACTION_SEND_MESSAGE, map()} | [{?ACTION_SEND_MESSAGE, map()}], ApplyMode :: handover @@ -380,7 +379,7 @@ conn_str([{_, _} | Opts], Acc) -> | {error, {recoverable_error, term()}} | {error, term()}. do_query( - BridgeId, + InstanceId, Query, ApplyMode, #{poolname := PoolName, sql_templates := Templates} = State @@ -388,7 +387,7 @@ do_query( ?TRACE( "SINGLE_QUERY_SYNC", "sqlserver_connector_received", - #{query => Query, connector => BridgeId, state => State} + #{query => Query, connector => InstanceId, state => State} ), %% only insert sql statement for single query and batch query @@ -412,7 +411,7 @@ do_query( ), ?SLOG(error, #{ msg => "sqlserver_connector_do_query_failed", - connector => BridgeId, + connector => InstanceId, query => Query, reason => Reason }), @@ -426,9 +425,9 @@ do_query( end. worker_do_insert( - Conn, SQL, #{resource_opts := ResourceOpts, poolname := BridgeId} = State + Conn, SQL, #{resource_opts := ResourceOpts, poolname := InstanceId} = State ) -> - LogMeta = #{connector => BridgeId, state => State}, + LogMeta = #{connector => InstanceId, state => State}, try case execute(Conn, SQL, ?REQUEST_TIMEOUT(ResourceOpts)) of {selected, Rows, _} -> From 06b1ec862efda931c975521c831b316331f964cc Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 14 Apr 2023 10:24:40 +0800 Subject: [PATCH 104/279] fix: hocon label field --- rel/i18n/emqx_ee_bridge_sqlserver.hocon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rel/i18n/emqx_ee_bridge_sqlserver.hocon b/rel/i18n/emqx_ee_bridge_sqlserver.hocon index 007b64ae6..a6f96f86d 100644 --- a/rel/i18n/emqx_ee_bridge_sqlserver.hocon +++ b/rel/i18n/emqx_ee_bridge_sqlserver.hocon @@ -32,7 +32,7 @@ will be forwarded.""" en: """SQL Server Driver Name""" zh: """SQL Server Driver 名称""" } - desc { + label { en: """SQL Server Driver Name""" zh: """SQL Server Driver 名称""" } From 8cec62bd877f7e59e283b4e73486d02dde5cd97b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 14 Apr 2023 12:41:34 +0800 Subject: [PATCH 105/279] fix: async reply arg list --- lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index 586444947..dc9759341 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -525,4 +525,4 @@ apply_template(Query, Templates) -> {error, failed_to_apply_sql_template}. do_async_reply(Result, {ReplyFun, Args}) -> - erlang:apply(ReplyFun, Args ++ Result). + erlang:apply(ReplyFun, Args ++ [Result]). From 2484a79c7aa0d712464b7c413069f3b615e0b601 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 14 Apr 2023 12:42:09 +0800 Subject: [PATCH 106/279] test: create bridge with invalid password --- .../docker-compose-sqlserver.yaml | 2 +- .../test/emqx_ee_bridge_sqlserver_SUITE.erl | 32 +++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-sqlserver.yaml b/.ci/docker-compose-file/docker-compose-sqlserver.yaml index a578983f8..63fcfeecd 100644 --- a/.ci/docker-compose-file/docker-compose-sqlserver.yaml +++ b/.ci/docker-compose-file/docker-compose-sqlserver.yaml @@ -11,7 +11,7 @@ services: # See also: # https://learn.microsoft.com/en-us/sql/linux/sql-server-linux-configure-environment-variables ACCEPT_EULA: "Y" - MSSQL_SA_PASSWORD: "mqtt_public" + MSSQL_SA_PASSWORD: "mqtt_public1" restart: always # ports: # - "1433:1433" diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl index 7a9c1baf9..152bdf2b7 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -44,7 +44,7 @@ % DB defaults -define(SQL_SERVER_DATABASE, "mqtt"). -define(SQL_SERVER_USERNAME, "sa"). --define(SQL_SERVER_PASSWORD, "mqtt_public"). +-define(SQL_SERVER_PASSWORD, "mqtt_public1"). -define(BATCH_SIZE, 10). -define(REQUEST_TIMEOUT_MS, 500). @@ -56,7 +56,7 @@ ok = disconnect(Con) ). -%% How to run it locally: +%% How to run it locally (all commands are run in $PROJ_ROOT dir): %% A: run ct on host %% 1. Start all deps services %% sudo docker compose -f .ci/docker-compose-file/docker-compose.yaml \ @@ -65,7 +65,7 @@ %% up --build %% %% 2. Run use cases with special environment variables -%% 11433 is toxiproxy exported port +%% 11433 is toxiproxy exported port. %% Local: %% ``` %% SQLSERVER_HOST=toxiproxy SQLSERVER_PORT=11433 \ @@ -241,6 +241,32 @@ t_create_disconnected(Config) -> health_check_resource_ok(Config), ok. +t_create_with_invalid_password(Config) -> + BridgeType = ?config(sqlserver_bridge_type, Config), + Name = ?config(sqlserver_name, Config), + SQLServerConfig0 = ?config(sqlserver_config, Config), + SQLServerConfig = SQLServerConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType, + <<"password">> => <<"wrong_password">> + }, + ?check_trace( + begin + ?assertMatch( + {ok, _}, + create_bridge_http(SQLServerConfig) + ) + end, + fun(Trace) -> + ?assertMatch( + [#{error := {start_pool_failed, _, _}}], + ?of_kind(sqlserver_connector_start_failed, Trace) + ), + ok + end + ), + ok. + t_write_failure(Config) -> ProxyName = ?config(proxy_name, Config), ProxyPort = ?config(proxy_port, Config), From 9ce0dbae7b297f27d97bd27c7a536f9e8ae02d4e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 14 Apr 2023 13:11:45 +0800 Subject: [PATCH 107/279] test: fix batch_size parameter --- lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl index 152bdf2b7..c1b33c3c9 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -398,7 +398,7 @@ common_init(ConfigT) -> {sqlserver_port, Port}, %% see also for `proxy_name` : $PROJ_ROOT/.ci/docker-compose-file/toxiproxy.json {proxy_name, "sqlserver"}, - {batch_size, ?BATCH_SIZE} + {batch_size, batch_size(ConfigT)} | ConfigT ], From 987ef6c51daece05188de4943190d2eb6142b34e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 14 Apr 2023 15:42:44 +0800 Subject: [PATCH 108/279] revert: "build: docker file add unixodbc and msodbcsql17" This reverts commit 611edf422719069297219370fe120968737fc0cf. --- .gitignore | 1 - deploy/docker/Dockerfile | 16 +++------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 3f7e268a2..62e8ddc81 100644 --- a/.gitignore +++ b/.gitignore @@ -71,4 +71,3 @@ apps/emqx/test/emqx_static_checks_data/master.bpapi lux_logs/ /.prepare bom.json -.dockerignore diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index dcebe9b1a..8f04c433c 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -29,19 +29,9 @@ COPY --from=builder /emqx-rel/emqx /opt/emqx RUN ln -s /opt/emqx/bin/* /usr/local/bin/ -RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates procps - -RUN apt-get update \ - && apt-get install -y gnupg2 curl apt-utils \ - && curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ - && curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list \ - && apt-get update \ - && ACCEPT_EULA=Y apt-get install -y msodbcsql17 unixodbc-dev \ - && sed -i 's/ODBC Driver 17 for SQL Server/ms-sql/g' /etc/odbcinst.ini - -RUN apt-get clean && \ - rm -rf /var/lib/apt/lists/*; +RUN apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates procps; \ + rm -rf /var/lib/apt/lists/* WORKDIR /opt/emqx From be35ae2132085b82ee077ff2cfa28b9d514adcad Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 17:14:11 +0800 Subject: [PATCH 109/279] feat: hide shared_subscription_group,rpc,slow_subs --- apps/emqx/src/emqx_schema.erl | 4 +++- apps/emqx_conf/src/emqx_conf_schema.erl | 5 ++++- apps/emqx_slow_subs/src/emqx_slow_subs.app.src | 2 +- apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl | 3 ++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 851fa3d3b..ed20848d2 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1498,12 +1498,14 @@ fields("broker") -> ref("broker_perf"), #{importance => ?IMPORTANCE_HIDDEN} )}, + %% FIXME: Need new design for shared subscription group {"shared_subscription_group", sc( map(name, ref("shared_subscription_group")), #{ example => #{<<"example_group">> => #{<<"strategy">> => <<"random">>}}, - desc => ?DESC(shared_subscription_group_strategy) + desc => ?DESC(shared_subscription_group_strategy), + importance => ?IMPORTANCE_HIDDEN } )} ]; diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index a24f43801..f689997c2 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -98,7 +98,10 @@ roots() -> {"rpc", sc( ?R_REF("rpc"), - #{translate_to => ["gen_rpc"]} + #{ + translate_to => ["gen_rpc"], + importance => ?IMPORTANCE_HIDDEN + } )} ] ++ emqx_schema:roots(medium) ++ diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src index 7d3fc341d..a06ff2595 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src @@ -1,7 +1,7 @@ {application, emqx_slow_subs, [ {description, "EMQX Slow Subscribers Statistics"}, % strict semver, bump manually! - {vsn, "1.0.4"}, + {vsn, "1.0.5"}, {modules, []}, {registered, [emqx_slow_subs_sup]}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index 9e9e6488a..47ea18c3c 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -22,7 +22,8 @@ namespace() -> "slow_subs". -roots() -> ["slow_subs"]. +roots() -> + [{"slow_subs", ?HOCON(?R_REF("slow_subs"), #{importance => ?IMPORTANCE_HIDDEN})}]. fields("slow_subs") -> [ From 4df48fbcfe4678e3379842b6d95d19c5c599d87f Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Fri, 14 Apr 2023 18:36:46 +0800 Subject: [PATCH 110/279] chore: upgrade dashboard to e1.0.6-beta.1 for ee --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 45218bf46..7c0a133f4 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v1.2.1 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.5 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6-beta.1 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) From a66d01d6f016af6e603cb72862178309b64a8ad8 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Fri, 14 Apr 2023 12:48:03 +0200 Subject: [PATCH 111/279] style: remove code duplication Co-authored-by: Thales Macedo Garitezi --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 7f7662b1b..dfbcdefed 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -1087,12 +1087,7 @@ format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> ). date_to_unix_ts(TimeUnit, FormatString, InputString) -> - emqx_rule_date:parse_date( - time_unit(TimeUnit), - "Z", - emqx_plugin_libs_rule:str(FormatString), - emqx_plugin_libs_rule:str(InputString) - ). + date_to_unix_ts(TimeUnit, "Z", FormatString, InputString). date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> emqx_rule_date:parse_date( From f8e9e54393204114eaaa1afcccdf457d350e4147 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 14:02:37 +0200 Subject: [PATCH 112/279] refactor: move emqx_json to emqx_utils_json --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_alarm.erl | 2 +- apps/emqx/src/emqx_logger_jsonfmt.erl | 2 +- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx/src/emqx_sys.erl | 2 +- apps/emqx/test/emqx_common_test_helpers.erl | 6 +- apps/emqx/test/emqx_common_test_http.erl | 4 +- apps/emqx/test/emqx_config_SUITE.erl | 2 +- apps/emqx/test/emqx_crl_cache_SUITE.erl | 6 +- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 14 +-- apps/emqx/test/props/prop_emqx_json.erl | 4 +- apps/emqx_authn/rebar.config | 1 + .../src/simple_authn/emqx_authn_http.erl | 4 +- .../simple_authn/emqx_authn_jwks_client.erl | 2 +- .../src/simple_authn/emqx_authn_jwt.erl | 2 +- .../src/simple_authn/emqx_authn_mnesia.erl | 2 +- apps/emqx_authn/test/emqx_authn_api_SUITE.erl | 24 ++--- .../emqx_authn/test/emqx_authn_http_SUITE.erl | 20 ++-- .../test/emqx_authn_https_SUITE.erl | 2 +- apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl | 2 +- apps/emqx_authz/rebar.config | 1 + apps/emqx_authz/src/emqx_authz_utils.erl | 2 +- .../test/emqx_authz_api_cache_SUITE.erl | 4 +- .../test/emqx_authz_api_sources_SUITE.erl | 23 ++-- .../emqx_authz/test/emqx_authz_http_SUITE.erl | 6 +- apps/emqx_auto_subscribe/rebar.config | 5 +- .../test/emqx_auto_subscribe_SUITE.erl | 4 +- apps/emqx_bridge/rebar.config | 8 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- .../test/emqx_bridge_api_SUITE.erl | 2 +- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 18 ++-- .../emqx_bridge_kafka_impl_producer_SUITE.erl | 2 +- apps/emqx_connector/rebar.config | 1 + .../src/emqx_connector_http.erl | 2 +- .../src/mqtt/emqx_connector_mqtt_msg.erl | 2 +- apps/emqx_dashboard/rebar.config | 5 +- .../test/emqx_dashboard_SUITE.erl | 2 +- .../test/emqx_dashboard_error_code_SUITE.erl | 2 +- .../test/emqx_dashboard_monitor_SUITE.erl | 4 +- apps/emqx_exhook/rebar.config | 3 +- .../test/emqx_exhook_api_SUITE.erl | 4 +- apps/emqx_gateway/rebar.config | 3 +- apps/emqx_gateway/src/emqx_gateway_cli.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_http.erl | 2 +- .../test/emqx_gateway_auth_ct.erl | 2 +- .../test/emqx_gateway_authn_SUITE.erl | 2 +- .../test/emqx_gateway_authz_SUITE.erl | 6 +- .../test/emqx_gateway_test_utils.erl | 4 +- apps/emqx_gateway_coap/rebar.config | 8 +- .../test/emqx_coap_api_SUITE.erl | 2 +- apps/emqx_gateway_exproto/rebar.config | 8 +- .../test/emqx_exproto_echo_svr.erl | 20 ++-- .../src/emqx_lwm2m_message.erl | 2 +- .../src/emqx_lwm2m_session.erl | 4 +- .../test/emqx_lwm2m_SUITE.erl | 100 +++++++++--------- .../test/emqx_lwm2m_api_SUITE.erl | 8 +- apps/emqx_gateway_stomp/rebar.config | 8 +- apps/emqx_machine/rebar.config | 5 +- apps/emqx_machine/src/emqx_machine.app.src | 2 +- apps/emqx_machine/src/emqx_machine.erl | 2 +- apps/emqx_management/rebar.config | 5 +- .../src/emqx_mgmt_api_banned.erl | 2 +- .../test/emqx_mgmt_api_alarms_SUITE.erl | 2 +- .../test/emqx_mgmt_api_api_keys_SUITE.erl | 10 +- .../test/emqx_mgmt_api_banned_SUITE.erl | 4 +- .../test/emqx_mgmt_api_clients_SUITE.erl | 10 +- .../test/emqx_mgmt_api_configs_SUITE.erl | 10 +- .../test/emqx_mgmt_api_listeners_SUITE.erl | 2 +- .../test/emqx_mgmt_api_metrics_SUITE.erl | 4 +- .../test/emqx_mgmt_api_nodes_SUITE.erl | 10 +- .../test/emqx_mgmt_api_plugins_SUITE.erl | 10 +- .../test/emqx_mgmt_api_publish_SUITE.erl | 2 +- .../test/emqx_mgmt_api_stats_SUITE.erl | 4 +- .../test/emqx_mgmt_api_subscription_SUITE.erl | 4 +- .../test/emqx_mgmt_api_test_util.erl | 3 +- .../test/emqx_mgmt_api_topics_SUITE.erl | 6 +- .../test/emqx_mgmt_api_trace_SUITE.erl | 2 +- apps/emqx_modules/rebar.config | 1 + apps/emqx_modules/src/emqx_telemetry.erl | 2 +- apps/emqx_modules/src/emqx_telemetry_api.erl | 2 +- .../test/emqx_delayed_api_SUITE.erl | 2 +- .../test/emqx_rewrite_api_SUITE.erl | 2 +- .../test/emqx_telemetry_SUITE.erl | 2 +- apps/emqx_plugin_libs/rebar.config | 5 +- .../emqx_plugin_libs/src/emqx_placeholder.erl | 2 +- .../src/emqx_plugin_libs_rule.erl | 10 +- apps/emqx_prometheus/rebar.config | 1 + .../test/emqx_prometheus_api_SUITE.erl | 6 +- apps/emqx_retainer/rebar.config | 5 +- .../test/emqx_retainer_api_SUITE.erl | 10 +- apps/emqx_rule_engine/rebar.config | 3 +- .../src/emqx_rule_actions.erl | 2 +- .../src/emqx_rule_engine_api.erl | 2 +- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 4 +- apps/emqx_rule_engine/src/emqx_rule_maps.erl | 2 +- .../src/emqx_rule_runtime.erl | 2 +- .../test/emqx_rule_engine_SUITE.erl | 14 ++- .../test/emqx_rule_engine_api_SUITE.erl | 10 +- apps/emqx_slow_subs/rebar.config | 5 +- .../test/emqx_slow_subs_api_SUITE.erl | 6 +- apps/emqx_statsd/rebar.config | 1 + apps/emqx_statsd/test/emqx_statsd_SUITE.erl | 2 +- apps/emqx_utils/README.md | 43 ++++++++ apps/emqx_utils/rebar.config | 11 ++ apps/emqx_utils/src/emqx_utils.app.src | 25 +++++ .../src/emqx_utils_json.erl} | 12 +-- apps/emqx_utils/test/emqx_utils_SUITE.erl | 48 +++++++++ .../test/emqx_utils_json_SUITE.erl} | 14 +-- lib-ee/emqx_ee_bridge/rebar.config | 1 + .../test/emqx_ee_bridge_cassa_SUITE.erl | 2 +- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 2 +- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 16 +-- .../test/emqx_ee_bridge_influxdb_SUITE.erl | 12 +-- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 2 +- .../test/emqx_ee_bridge_mysql_SUITE.erl | 2 +- .../test/emqx_ee_bridge_pgsql_SUITE.erl | 2 +- .../test/emqx_ee_bridge_rocketmq_SUITE.erl | 2 +- .../test/emqx_ee_bridge_tdengine_SUITE.erl | 2 +- lib-ee/emqx_ee_connector/rebar.config | 3 +- .../src/emqx_ee_connector_dynamo.erl | 4 +- .../src/emqx_ee_connector_gcp_pubsub.erl | 4 +- .../src/emqx_ee_connector_mongodb.erl | 2 +- .../src/emqx_ee_connector_rocketmq.erl | 4 +- lib-ee/emqx_ee_schema_registry/rebar.config | 1 + .../src/emqx_ee_schema_registry_serde.erl | 4 +- .../test/emqx_ee_schema_registry_SUITE.erl | 10 +- ...emqx_ee_schema_registry_http_api_SUITE.erl | 6 +- .../emqx_ee_schema_registry_serde_SUITE.erl | 2 +- lib-ee/emqx_license/rebar.config | 5 +- .../test/emqx_license_http_api_SUITE.erl | 8 +- 130 files changed, 508 insertions(+), 331 deletions(-) create mode 100644 apps/emqx_utils/README.md create mode 100644 apps/emqx_utils/rebar.config create mode 100644 apps/emqx_utils/src/emqx_utils.app.src rename apps/{emqx/src/emqx_json.erl => emqx_utils/src/emqx_utils_json.erl} (90%) create mode 100644 apps/emqx_utils/test/emqx_utils_SUITE.erl rename apps/{emqx/test/emqx_json_SUITE.erl => emqx_utils/test/emqx_utils_json_SUITE.erl} (93%) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 9079322eb..a3eff6c16 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -22,9 +22,9 @@ %% This rebar.config is necessary because the app may be used as a %% `git_subdir` dependency in other projects. {deps, [ + {emqx_utils, {path, "../emqx_utils"}}, {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}}, {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}, - {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 84c40ef2a..6aa3cb95d 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -423,7 +423,7 @@ do_actions(deactivate, Alarm = #deactivated_alarm{name = Name}, [log | More]) -> do_actions(deactivate, Alarm, More); do_actions(Operation, Alarm, [publish | More]) -> Topic = topic(Operation), - {ok, Payload} = emqx_json:safe_encode(normalize(Alarm)), + {ok, Payload} = emqx_utils_json:safe_encode(normalize(Alarm)), Message = emqx_message:make( ?MODULE, 0, diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index 22cf75153..f60f5a716 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -92,7 +92,7 @@ format(Msg, Meta, Config) -> } end, Data = maps:without([report_cb], Data0), - jiffy:encode(json_obj(Data, Config)). + emqx_utils_json:encode(json_obj(Data, Config)). maybe_format_msg({report, Report} = Msg, #{report_cb := Cb} = Meta, Config) -> case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 851fa3d3b..8c91ae782 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2583,7 +2583,7 @@ to_url(Str) -> end. to_json_binary(Str) -> - case emqx_json:safe_decode(Str) of + case emqx_utils_json:safe_decode(Str) of {ok, _} -> {ok, iolist_to_binary(Str)}; Error -> diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index a5f14e32a..af2b052aa 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -348,7 +348,7 @@ publish(Event, Payload) when Event == unsubscribed -> Topic = event_topic(Event, Payload), - safe_publish(Topic, emqx_json:encode(Payload)). + safe_publish(Topic, emqx_utils_json:encode(Payload)). metric_topic(Name) -> translate_topic("metrics/", Name). diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 406183094..85cb46106 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -1036,7 +1036,7 @@ switch_proxy(Switch, Name, ProxyHost, ProxyPort) -> off -> #{<<"enabled">> => false}; on -> #{<<"enabled">> => true} end, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, @@ -1056,7 +1056,7 @@ timeout_proxy(on, Name, ProxyHost, ProxyPort) -> <<"toxicity">> => 1.0, <<"attributes">> => #{<<"timeout">> => 0} }, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, @@ -1091,7 +1091,7 @@ latency_up_proxy(on, Name, ProxyHost, ProxyPort) -> <<"jitter">> => 3_000 } }, - BodyBin = emqx_json:encode(Body), + BodyBin = emqx_utils_json:encode(Body), {ok, {{_, 200, _}, _, _}} = httpc:request( post, {Url, [], "application/json", BodyBin}, diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl index 575bed5c3..e9064715d 100644 --- a/apps/emqx/test/emqx_common_test_http.erl +++ b/apps/emqx/test/emqx_common_test_http.erl @@ -54,7 +54,7 @@ request_api(Method, Url, QueryParams, Auth, Body, HttpOpts) -> [] -> {NewUrl, [Auth]}; _ -> - {NewUrl, [Auth], "application/json", emqx_json:encode(Body)} + {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)} end, do_request_api(Method, Request, HttpOpts). @@ -70,7 +70,7 @@ do_request_api(Method, Request, HttpOpts) -> end. get_http_data(ResponseBody) -> - emqx_json:decode(ResponseBody, [return_maps]). + emqx_utils_json:decode(ResponseBody, [return_maps]). auth_header(User, Pass) -> Encoded = base64:encode_to_string(lists:append([User, ":", Pass])), diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl index fe8a5fed8..7befd7a16 100644 --- a/apps/emqx/test/emqx_config_SUITE.erl +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -57,5 +57,5 @@ t_fill_default_values(_) -> WithDefaults ), %% ensure JSON compatible - _ = emqx_json:encode(WithDefaults), + _ = emqx_utils_json:encode(WithDefaults), ok. diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl index 01f9c7172..21bfc39df 100644 --- a/apps/emqx/test/emqx_crl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -402,7 +402,7 @@ request(Method, Url, QueryParams, Body) -> Opts = #{return_all => true}, case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of {ok, {Reason, Headers, BodyR}} -> - {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}}; + {ok, {Reason, Headers, emqx_utils_json:decode(BodyR, [return_maps])}}; Error -> Error end. @@ -1052,7 +1052,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} = - emqx_json:decode(ResRaw1, [return_maps]), + emqx_utils_json:decode(ResRaw1, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -1064,7 +1064,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw1, [return_maps]) + emqx_utils_json:decode(MsgRaw1, [return_maps]) ), ok. diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index 3c3fd0341..a9b80c276 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -430,7 +430,7 @@ request(Method, Url, QueryParams, Body) -> Opts = #{return_all => true}, case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body, Opts) of {ok, {Reason, Headers, BodyR}} -> - {ok, {Reason, Headers, emqx_json:decode(BodyR, [return_maps])}}; + {ok, {Reason, Headers, emqx_utils_json:decode(BodyR, [return_maps])}}; Error -> Error end. @@ -827,7 +827,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw1}} = update_listener_via_api(ListenerId, ListenerData1), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw1} = - emqx_json:decode(ResRaw1, [return_maps]), + emqx_utils_json:decode(ResRaw1, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -839,7 +839,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw1, [return_maps]) + emqx_utils_json:decode(MsgRaw1, [return_maps]) ), ListenerData2 = @@ -857,7 +857,7 @@ do_t_validations(_Config) -> ), {error, {_, _, ResRaw2}} = update_listener_via_api(ListenerId, ListenerData2), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw2} = - emqx_json:decode(ResRaw2, [return_maps]), + emqx_utils_json:decode(ResRaw2, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -869,7 +869,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw2, [return_maps]) + emqx_utils_json:decode(MsgRaw2, [return_maps]) ), ListenerData3a = @@ -889,7 +889,7 @@ do_t_validations(_Config) -> ListenerData3 = emqx_map_lib:deep_remove([<<"ssl_options">>, <<"certfile">>], ListenerData3a), {error, {_, _, ResRaw3}} = update_listener_via_api(ListenerId, ListenerData3), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw3} = - emqx_json:decode(ResRaw3, [return_maps]), + emqx_utils_json:decode(ResRaw3, [return_maps]), ?assertMatch( #{ <<"mismatches">> := @@ -901,7 +901,7 @@ do_t_validations(_Config) -> } } }, - emqx_json:decode(MsgRaw3, [return_maps]) + emqx_utils_json:decode(MsgRaw3, [return_maps]) ), ok. diff --git a/apps/emqx/test/props/prop_emqx_json.erl b/apps/emqx/test/props/prop_emqx_json.erl index 2bc079634..86f60b39c 100644 --- a/apps/emqx/test/props/prop_emqx_json.erl +++ b/apps/emqx/test/props/prop_emqx_json.erl @@ -14,10 +14,10 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(prop_emqx_json). +-module(prop_emqx_utils_json). -import( - emqx_json, + emqx_utils_json, [ decode/1, decode/2, diff --git a/apps/emqx_authn/rebar.config b/apps/emqx_authn/rebar.config index 8fd9cea0f..5f0043c39 100644 --- a/apps/emqx_authn/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_connector, {path, "../emqx_connector"}} ]}. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 9be1d0a33..3c34d878e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -357,7 +357,7 @@ qs([{K, V} | More], Acc) -> qs(More, [["&", uri_encode(K), "=", uri_encode(V)] | Acc]). serialize_body(<<"application/json">>, Body) -> - emqx_json:encode(Body); + emqx_utils_json:encode(Body); serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> qs(maps:to_list(Body)). @@ -395,7 +395,7 @@ safely_parse_body(ContentType, Body) -> end. parse_body(<<"application/json", _/binary>>, Body) -> - {ok, emqx_json:decode(Body, [return_maps])}; + {ok, emqx_utils_json:decode(Body, [return_maps])}; parse_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> Flags = [<<"result">>, <<"is_superuser">>], RawMap = maps:from_list(cow_qs:parse_qs(Body)), diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl index 5ee923859..23d939f7d 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_client.erl @@ -99,7 +99,7 @@ handle_info( State1; {StatusLine, Headers, Body} -> try - JWKS = jose_jwk:from(emqx_json:decode(Body, [return_maps])), + JWKS = jose_jwk:from(emqx_utils_json:decode(Body, [return_maps])), {_, JWKs} = JWKS#jose_jwk.keys, State1#{jwks := JWKs} catch diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 73ba6a2c1..a891a55e2 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -407,7 +407,7 @@ do_verify(_JWT, [], _VerifyClaims) -> do_verify(JWT, [JWK | More], VerifyClaims) -> try jose_jws:verify(JWK, JWT) of {true, Payload, _JWT} -> - Claims0 = emqx_json:decode(Payload, [return_maps]), + Claims0 = emqx_utils_json:decode(Payload, [return_maps]), Claims = try_convert_to_num(Claims0, [<<"exp">>, <<"iat">>, <<"nbf">>]), case verify_claims(Claims, VerifyClaims) of ok -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index 25a3a5976..20b604e70 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -332,7 +332,7 @@ run_fuzzy_filter( %% Example: data/user-credentials.json import_users_from_json(Bin, #{user_group := UserGroup}) -> - case emqx_json:safe_decode(Bin, [return_maps]) of + case emqx_utils_json:safe_decode(Bin, [return_maps]) of {ok, List} -> trans(fun ?MODULE:import/2, [UserGroup, List]); {error, Reason} -> diff --git a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl index 11e2c6773..c7f718dfc 100644 --- a/apps/emqx_authn/test/emqx_authn_api_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_api_SUITE.erl @@ -29,7 +29,7 @@ -define(assertAuthenticatorsMatch(Guard, Path), (fun() -> {ok, 200, Response} = request(get, uri(Path)), - ?assertMatch(Guard, jiffy:decode(Response, [return_maps])) + ?assertMatch(Guard, emqx_utils_json:decode(Response, [return_maps])) end)() ). @@ -234,7 +234,7 @@ test_authenticator(PathPrefix) -> get, uri(PathPrefix ++ [?CONF_NS, "password_based:http", "status"]) ), - {ok, RList} = emqx_json:safe_decode(Res), + {ok, RList} = emqx_utils_json:safe_decode(Res), Snd = fun({_, Val}) -> Val end, LookupVal = fun LookupV(List, RestJson) -> case List of @@ -353,7 +353,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 0, <<"nomatch">> := 1 } - } = jiffy:decode(PageData0, [return_maps]); + } = emqx_utils_json:decode(PageData0, [return_maps]); ["listeners", 'tcp:default'] -> #{ <<"metrics">> := #{ @@ -361,7 +361,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 0, <<"nomatch">> := 1 } - } = jiffy:decode(PageData0, [return_maps]) + } = emqx_utils_json:decode(PageData0, [return_maps]) end, InvalidUsers = [ @@ -384,7 +384,7 @@ test_authenticator_users(PathPrefix) -> lists:foreach( fun(User) -> {ok, 201, UserData} = request(post, UsersUri, User), - CreatedUser = jiffy:decode(UserData, [return_maps]), + CreatedUser = emqx_utils_json:decode(UserData, [return_maps]), ?assertMatch(#{<<"user_id">> := _}, CreatedUser) end, ValidUsers @@ -411,7 +411,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 1, <<"nomatch">> := 1 } - } = jiffy:decode(PageData01, [return_maps]); + } = emqx_utils_json:decode(PageData01, [return_maps]); ["listeners", 'tcp:default'] -> #{ <<"metrics">> := #{ @@ -419,7 +419,7 @@ test_authenticator_users(PathPrefix) -> <<"success">> := 1, <<"nomatch">> := 1 } - } = jiffy:decode(PageData01, [return_maps]) + } = emqx_utils_json:decode(PageData01, [return_maps]) end, {ok, 200, Page1Data} = request(get, UsersUri ++ "?page=1&limit=2"), @@ -433,7 +433,7 @@ test_authenticator_users(PathPrefix) -> <<"count">> := 3 } } = - jiffy:decode(Page1Data, [return_maps]), + emqx_utils_json:decode(Page1Data, [return_maps]), {ok, 200, Page2Data} = request(get, UsersUri ++ "?page=2&limit=2"), @@ -445,7 +445,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 2, <<"count">> := 3 } - } = jiffy:decode(Page2Data, [return_maps]), + } = emqx_utils_json:decode(Page2Data, [return_maps]), ?assertEqual(2, length(Page1Users)), ?assertEqual(1, length(Page2Users)), @@ -465,7 +465,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 3, <<"count">> := 1 } - } = jiffy:decode(Super1Data, [return_maps]), + } = emqx_utils_json:decode(Super1Data, [return_maps]), ?assertEqual( [<<"u2">>], @@ -482,7 +482,7 @@ test_authenticator_users(PathPrefix) -> <<"limit">> := 3, <<"count">> := 2 } - } = jiffy:decode(Super2Data, [return_maps]), + } = emqx_utils_json:decode(Super2Data, [return_maps]), ?assertEqual( [<<"u1">>, <<"u3">>], @@ -509,7 +509,7 @@ test_authenticator_user(PathPrefix) -> {ok, 200, UserData} = request(get, UsersUri ++ "/u1"), - FetchedUser = jiffy:decode(UserData, [return_maps]), + FetchedUser = emqx_utils_json:decode(UserData, [return_maps]), ?assertMatch(#{<<"user_id">> := <<"u1">>}, FetchedUser), ?assertNotMatch(#{<<"password">> := _}, FetchedUser), diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 9a3c7c833..851e80f6d 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -41,7 +41,7 @@ -define(SERVER_RESPONSE_JSON(Result), ?SERVER_RESPONSE_JSON(Result, false)). -define(SERVER_RESPONSE_JSON(Result, IsSuperuser), - jiffy:encode(#{ + emqx_utils_json:encode(#{ result => Result, is_superuser => IsSuperuser }) @@ -172,11 +172,11 @@ t_no_value_for_placeholder(_Config) -> #{ <<"cert_subject">> := <<"">>, <<"cert_common_name">> := <<"">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -444,7 +444,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req0 ), {ok, Req, State} @@ -459,7 +459,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => true}), + emqx_utils_json:encode(#{result => allow, is_superuser => true}), Req0 ), {ok, Req, State} @@ -512,11 +512,11 @@ samples() -> #{ <<"username">> := <<"plain">>, <<"password">> := <<"plain">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -539,7 +539,7 @@ samples() -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} @@ -565,11 +565,11 @@ samples() -> <<"peerhost">> := <<"127.0.0.1">>, <<"cert_subject">> := <<"cert_subject_data">>, <<"cert_common_name">> := <<"cert_common_name_data">> - } = jiffy:decode(RawBody, [return_maps]), + } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req1 ), {ok, Req, State} diff --git a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl index 7d51ff425..c4315b69f 100644 --- a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl @@ -168,7 +168,7 @@ cowboy_handler(Req0, State) -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{result => allow, is_superuser => false}), + emqx_utils_json:encode(#{result => allow, is_superuser => false}), Req0 ), {ok, Req, State}. diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 7a51d2bbb..94c07ca96 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -467,7 +467,7 @@ jwks_handler(Req0, State) -> Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(JWKS), + emqx_utils_json:encode(JWKS), Req0 ), {ok, Req, State}. diff --git a/apps/emqx_authz/rebar.config b/apps/emqx_authz/rebar.config index da2fa7807..9fd61b060 100644 --- a/apps/emqx_authz/rebar.config +++ b/apps/emqx_authz/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info, nowarn_unused_import]}. {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_connector, {path, "../emqx_connector"}} ]}. diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index df77673a2..560141d0a 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -144,7 +144,7 @@ parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> end; parse_http_resp_body(<<"application/json", _/binary>>, Body) -> try - result(emqx_json:decode(Body, [return_maps])) + result(emqx_utils_json:decode(Body, [return_maps])) catch _:_ -> error end. diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index 45e6d7287..cfd9f581f 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -67,12 +67,12 @@ t_clean_cahce(_) -> ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0), {ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(2, length(emqx_json:decode(Result3))), + ?assertEqual(2, length(emqx_utils_json:decode(Result3))), request(delete, uri(["authorization", "cache"])), {ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(0, length(emqx_json:decode(Result4))), + ?assertEqual(0, length(emqx_utils_json:decode(Result4))), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 411399d64..b717775a6 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -182,7 +182,7 @@ t_api(_) -> {ok, 404, ErrResult} = request(get, uri(["authorization", "sources", "http"]), []), ?assertMatch( #{<<"code">> := <<"NOT_FOUND">>, <<"message">> := <<"Not found: http">>}, - emqx_json:decode(ErrResult, [return_maps]) + emqx_utils_json:decode(ErrResult, [return_maps]) ), [ @@ -215,7 +215,8 @@ t_api(_) -> ), {ok, 200, Result3} = request(get, uri(["authorization", "sources", "http"]), []), ?assertMatch( - #{<<"type">> := <<"http">>, <<"enable">> := false}, emqx_json:decode(Result3, [return_maps]) + #{<<"type">> := <<"http">>, <<"enable">> := false}, + emqx_utils_json:decode(Result3, [return_maps]) ), Keyfile = emqx_common_test_helpers:app_path( @@ -253,7 +254,7 @@ t_api(_) -> <<"total">> := 0, <<"nomatch">> := 0 } - } = emqx_json:decode(Status4, [return_maps]), + } = emqx_utils_json:decode(Status4, [return_maps]), ?assertMatch( #{ <<"type">> := <<"mongodb">>, @@ -265,7 +266,7 @@ t_api(_) -> <<"verify">> := <<"verify_none">> } }, - emqx_json:decode(Result4, [return_maps]) + emqx_utils_json:decode(Result4, [return_maps]) ), {ok, Cacert} = file:read_file(Cacertfile), @@ -297,7 +298,7 @@ t_api(_) -> <<"verify">> := <<"verify_none">> } }, - emqx_json:decode(Result5, [return_maps]) + emqx_utils_json:decode(Result5, [return_maps]) ), {ok, 200, Status5_1} = request(get, uri(["authorization", "sources", "mongodb", "status"]), []), @@ -308,7 +309,7 @@ t_api(_) -> <<"total">> := 0, <<"nomatch">> := 0 } - } = emqx_json:decode(Status5_1, [return_maps]), + } = emqx_utils_json:decode(Status5_1, [return_maps]), #{ ssl := #{ @@ -355,7 +356,7 @@ t_api(_) -> <<"code">> := <<"BAD_REQUEST">>, <<"message">> := <<"Type mismatch", _/binary>> }, - emqx_json:decode(TypeMismatch, [return_maps]) + emqx_utils_json:decode(TypeMismatch, [return_maps]) ), lists:foreach( @@ -443,7 +444,7 @@ t_api(_) -> <<"total">> := 1, <<"nomatch">> := 0 } - } = emqx_json:decode(Status5, [return_maps]) + } = emqx_utils_json:decode(Status5, [return_maps]) end ), @@ -469,7 +470,7 @@ t_api(_) -> <<"total">> := 2, <<"nomatch">> := 0 } - } = emqx_json:decode(Status6, [return_maps]) + } = emqx_utils_json:decode(Status6, [return_maps]) end ), @@ -495,7 +496,7 @@ t_api(_) -> <<"total">> := 3, <<"nomatch">> := 0 } - } = emqx_json:decode(Status7, [return_maps]) + } = emqx_utils_json:decode(Status7, [return_maps]) end ), ok. @@ -621,7 +622,7 @@ t_aggregate_metrics(_) -> ). get_sources(Result) -> - maps:get(<<"sources">>, emqx_json:decode(Result, [return_maps])). + maps:get(<<"sources">>, emqx_utils_json:decode(Result, [return_maps])). data_dir() -> emqx:data_dir(). diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index 0757113f3..9ff84b805 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -311,7 +311,7 @@ t_json_body(_Config) -> <<"topic">> := <<"t">>, <<"action">> := <<"publish">> }, - jiffy:decode(RawBody, [return_maps]) + emqx_utils_json:decode(RawBody, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, @@ -366,7 +366,7 @@ t_placeholder_and_body(_Config) -> <<"CN">> := ?PH_CERT_CN_NAME, <<"CS">> := ?PH_CERT_SUBJECT }, - jiffy:decode(PostVars, [return_maps]) + emqx_utils_json:decode(PostVars, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, @@ -418,7 +418,7 @@ t_no_value_for_placeholder(_Config) -> #{ <<"mountpoint">> := <<"[]">> }, - jiffy:decode(RawBody, [return_maps]) + emqx_utils_json:decode(RawBody, [return_maps]) ), {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} end, diff --git a/apps/emqx_auto_subscribe/rebar.config b/apps/emqx_auto_subscribe/rebar.config index 33e077f50..a19783033 100644 --- a/apps/emqx_auto_subscribe/rebar.config +++ b/apps/emqx_auto_subscribe/rebar.config @@ -1,7 +1,10 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {shell, [ {apps, [emqx_auto_subscribe]} diff --git a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl index 5c5a3ee79..9d8d47bf2 100644 --- a/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl +++ b/apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl @@ -141,7 +141,7 @@ t_update(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Body = [#{topic => ?TOPIC_S}], {ok, Response} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, Body), - ResponseMap = emqx_json:decode(Response, [return_maps]), + ResponseMap = emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(1, erlang:length(ResponseMap)), BadBody1 = #{topic => ?TOPIC_S}, @@ -177,7 +177,7 @@ t_update(_) -> emqtt:disconnect(Client), {ok, GETResponse} = emqx_mgmt_api_test_util:request_api(get, Path), - GETResponseMap = emqx_json:decode(GETResponse, [return_maps]), + GETResponseMap = emqx_utils_json:decode(GETResponse, [return_maps]), ?assertEqual(1, erlang:length(GETResponseMap)), ok. diff --git a/apps/emqx_bridge/rebar.config b/apps/emqx_bridge/rebar.config index 79f2caf50..864c45e9a 100644 --- a/apps/emqx_bridge/rebar.config +++ b/apps/emqx_bridge/rebar.config @@ -1,7 +1,9 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../emqx"}} - , {emqx_resource, {path, "../../apps/emqx_resource"}} - ]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}} + ]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index b29cefacd..675541311 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -1028,6 +1028,6 @@ deobfuscate(NewConf, OldConf) -> ). map_to_json(M) -> - emqx_json:encode( + emqx_utils_json:encode( emqx_map_lib:jsonable_map(M, fun(K, V) -> {K, emqx_map_lib:binary_string(V)} end) ). diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 4d5495ffe..3ec6061d8 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1304,4 +1304,4 @@ str(S) when is_list(S) -> S; str(S) when is_binary(S) -> binary_to_list(S). json(B) when is_binary(B) -> - emqx_json:decode(B, [return_maps]). + emqx_utils_json:decode(B, [return_maps]). diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index fb7cf524c..fc9a6dc29 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -705,7 +705,7 @@ create_bridge_api(Config, Overrides) -> Res = case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params, Opts) of {ok, {Status, Headers, Body0}} -> - {ok, {Status, Headers, emqx_json:decode(Body0, [return_maps])}}; + {ok, {Status, Headers, emqx_utils_json:decode(Body0, [return_maps])}}; Error -> Error end, @@ -728,7 +728,7 @@ update_bridge_api(Config, Overrides) -> ct:pal("updating bridge (via http): ~p", [Params]), Res = case emqx_mgmt_api_test_util:request_api(put, Path, "", AuthHeader, Params, Opts) of - {ok, {_Status, _Headers, Body0}} -> {ok, emqx_json:decode(Body0, [return_maps])}; + {ok, {_Status, _Headers, Body0}} -> {ok, emqx_utils_json:decode(Body0, [return_maps])}; Error -> Error end, ct:pal("bridge update result: ~p", [Res]), @@ -776,7 +776,7 @@ do_wait_for_expected_published_messages(Messages, Acc, _Timeout) when map_size(M do_wait_for_expected_published_messages(Messages0, Acc0, Timeout) -> receive {publish, Msg0 = #{payload := Payload}} -> - case emqx_json:safe_decode(Payload, [return_maps]) of + case emqx_utils_json:safe_decode(Payload, [return_maps]) of {error, _} -> ct:pal("unexpected message: ~p; discarding", [Msg0]), do_wait_for_expected_published_messages(Messages0, Acc0, Timeout); @@ -928,7 +928,7 @@ create_rule_and_action_http(Config) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), ct:pal("rule action params: ~p", [Params]), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -1188,7 +1188,7 @@ t_start_and_consume_ok(Config) -> <<"offset">> := OffsetReply, <<"headers">> := #{<<"hkey">> := <<"hvalue">>} }, - emqx_json:decode(PayloadBin, [return_maps]), + emqx_utils_json:decode(PayloadBin, [return_maps]), #{ offset_reply => OffsetReply, kafka_topic => KafkaTopic, @@ -1300,7 +1300,7 @@ t_multiple_topic_mappings(Config) -> %% as configured. Payloads = lists:sort([ - case emqx_json:safe_decode(P, [return_maps]) of + case emqx_utils_json:safe_decode(P, [return_maps]) of {ok, Decoded} -> Decoded; {error, _} -> P end @@ -1441,7 +1441,7 @@ do_t_failed_creation_then_fixed(Config) -> <<"offset">> := _, <<"headers">> := #{<<"hkey">> := <<"hvalue">>} }, - emqx_json:decode(PayloadBin, [return_maps]), + emqx_utils_json:decode(PayloadBin, [return_maps]), #{ kafka_topic => KafkaTopic, payload => Payload @@ -1636,7 +1636,7 @@ t_bridge_rule_action_source(Config) -> <<"headers">> := #{<<"hkey">> := <<"hvalue">>}, <<"topic">> := KafkaTopic }, - emqx_json:decode(RawPayload, [return_maps]) + emqx_utils_json:decode(RawPayload, [return_maps]) ), ?retry( _Interval = 200, @@ -2004,7 +2004,7 @@ t_begin_offset_earliest(Config) -> %% the consumers Published = receive_published(#{n => NumMessages}), Payloads = lists:map( - fun(#{payload := P}) -> emqx_json:decode(P, [return_maps]) end, + fun(#{payload := P}) -> emqx_utils_json:decode(P, [return_maps]) end, Published ), ?assert( diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index 6e3ddf5bb..5e2a2a7a8 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -802,7 +802,7 @@ api_path(Parts) -> ?HOST ++ filename:join([?BASE_PATH | Parts]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. delete_all_bridges() -> diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 2ce6b00f8..03be87356 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -7,6 +7,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_resource, {path, "../emqx_resource"}}, {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}}, {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}}, diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 401fc8812..dfa6dab81 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -516,7 +516,7 @@ process_request( }. process_request_body(undefined, Msg) -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); process_request_body(BodyTks, Msg) -> emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index defbbaea2..67fc40efa 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -123,7 +123,7 @@ process_payload(From, MapMsg) -> do_process_payload(maps:get(payload, From, undefined), MapMsg). do_process_payload(undefined, Msg) -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); do_process_payload(Tks, Msg) -> replace_vars_in_str(Tks, Msg). diff --git a/apps/emqx_dashboard/rebar.config b/apps/emqx_dashboard/rebar.config index 9657d0bbf..440fde465 100644 --- a/apps/emqx_dashboard/rebar.config +++ b/apps/emqx_dashboard/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index e951a9a2a..ee850ec17 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -246,5 +246,5 @@ api_path(Parts) -> ?HOST ++ filename:join([?BASE_PATH | Parts]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl index 19d3f471e..c0a772d2d 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl @@ -100,7 +100,7 @@ request(Url) -> {ok, {{"HTTP/1.1", Code, _}, _, Return}} when Code >= 200 andalso Code =< 299 -> - {ok, emqx_json:decode(Return, [return_maps])}; + {ok, emqx_utils_json:decode(Return, [return_maps])}; {ok, {Reason, _, _}} -> {error, Reason} end. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index f35652f8e..a24fc2337 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -137,10 +137,10 @@ do_request_api(Method, Request) -> Code >= 200 andalso Code =< 299 -> ct:pal("Resp ~p ~p~n", [Code, Return]), - {ok, emqx_json:decode(Return, [return_maps])}; + {ok, emqx_utils_json:decode(Return, [return_maps])}; {ok, {{"HTTP/1.1", Code, _}, _, Return}} -> ct:pal("Resp ~p ~p~n", [Code, Return]), - {error, {Code, emqx_json:decode(Return, [return_maps])}}; + {error, {Code, emqx_utils_json:decode(Return, [return_maps])}}; {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index fad539ed1..7abc601b4 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -5,7 +5,8 @@ ]}. {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. {grpc, [ diff --git a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl index 8a4fb7a44..58f2f29b4 100644 --- a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl @@ -310,7 +310,7 @@ t_update(Cfg) -> ?assertMatch([], emqx_exhook_mgr:running()). decode_json(Data) -> - BinJosn = emqx_json:decode(Data, [return_maps]), + BinJosn = emqx_utils_json:decode(Data, [return_maps]), emqx_map_lib:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> @@ -332,7 +332,7 @@ request_api(Method, Url, QueryParams, Auth, Body) -> "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)}). do_request_api(Method, Request) -> case httpc:request(Method, Request, [], [{body_format, binary}]) of diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config index 7e5228a9e..2340a2dd8 100644 --- a/apps/emqx_gateway/rebar.config +++ b/apps/emqx_gateway/rebar.config @@ -1,5 +1,6 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index df808f295..fb4261065 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -74,7 +74,7 @@ gateway(["load", Name, Conf]) -> case emqx_gateway_conf:load_gateway( bin(Name), - emqx_json:decode(Conf, [return_maps]) + emqx_utils_json:decode(Conf, [return_maps]) ) of {ok, _} -> diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index a0155a126..5982334b4 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -404,7 +404,7 @@ return_http_error(Code, Msg) -> -spec reason2msg({atom(), map()} | any()) -> error | string(). reason2msg({badconf, #{key := Key, value := Value, reason := Reason}}) -> NValue = - case emqx_json:safe_encode(Value) of + case emqx_utils_json:safe_encode(Value) of {ok, Str} -> Str; {error, _} -> emqx_gateway_utils:stringfy(Value) end, diff --git a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl index d75bf80eb..0ed66a38d 100644 --- a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl +++ b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl @@ -153,7 +153,7 @@ on_start_auth(authn_http) -> Handler = fun(Req0, State) -> ct:pal("Authn Req:~p~nState:~p~n", [Req0, State]), Headers = #{<<"content-type">> => <<"application/json">>}, - Response = jiffy:encode(#{result => allow, is_superuser => false}), + Response = emqx_utils_json:encode(#{result => allow, is_superuser => false}), case cowboy_req:match_qs([username, password], Req0) of #{ username := <<"admin">>, diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 1a4bab5f3..1ab36f7b8 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -266,7 +266,7 @@ t_case_exproto(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, 5000), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(Expect), C(Recv)) end ) diff --git a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl index 9bbcf2711..c62e840df 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl @@ -165,7 +165,7 @@ t_case_lwm2m(_) -> Test("lwm2m", fun(SubTopic, Msg) -> ?assertEqual(true, lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics())), Payload = emqx_message:payload(Msg), - Cmd = emqx_json:decode(Payload, [return_maps]), + Cmd = emqx_utils_json:decode(Payload, [return_maps]), ?assertMatch(#{<<"msgType">> := <<"register">>, <<"data">> := _}, Cmd) end), @@ -350,7 +350,7 @@ t_case_exproto_publish(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, 5000), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(SvrMod:frame_connack(0)), C(Recv)), Send = fun() -> @@ -387,7 +387,7 @@ t_case_exproto_subscribe(_) -> Mod:send(Sock, ConnBin), {ok, Recv} = Mod:recv(Sock, WaitTime), - C = ?FUNCTOR(Bin, emqx_json:decode(Bin, [return_maps])), + C = ?FUNCTOR(Bin, emqx_utils_json:decode(Bin, [return_maps])), ?assertEqual(C(SvrMod:frame_connack(0)), C(Recv)), SubBin = SvrMod:frame_subscribe(Topic, 0), diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index 7ec0c9538..6d80b63d8 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -160,7 +160,7 @@ do_request(Mth, Req) -> #{}; _ -> emqx_map_lib:unsafe_atom_key_map( - emqx_json:decode(Resp, [return_maps]) + emqx_utils_json:decode(Resp, [return_maps]) ) end, {Code, NResp}; @@ -172,7 +172,7 @@ req(Path, Qs) -> {url(Path, Qs), auth([])}. req(Path, Qs, Body) -> - {url(Path, Qs), auth([]), "application/json", emqx_json:encode(Body)}. + {url(Path, Qs), auth([]), "application/json", emqx_utils_json:encode(Body)}. url(Path, []) -> lists:concat([?http_api_host, Path]); diff --git a/apps/emqx_gateway_coap/rebar.config b/apps/emqx_gateway_coap/rebar.config index c8675c3ba..3b070a72a 100644 --- a/apps/emqx_gateway_coap/rebar.config +++ b/apps/emqx_gateway_coap/rebar.config @@ -1,4 +1,6 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../../apps/emqx"}}, - {emqx_gateway, {path, "../../apps/emqx_gateway"}} - ]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../emqx_gateway"}} +]}. diff --git a/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl index a0d6dbaaa..cec09a016 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_api_SUITE.erl @@ -92,7 +92,7 @@ t_send_request_api(_) -> Req ), #{<<"token">> := RToken, <<"payload">> := RPayload} = - emqx_json:decode(Response, [return_maps]), + emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(Token, RToken), ?assertEqual(Payload, RPayload) end, diff --git a/apps/emqx_gateway_exproto/rebar.config b/apps/emqx_gateway_exproto/rebar.config index 928949c69..473fa9b67 100644 --- a/apps/emqx_gateway_exproto/rebar.config +++ b/apps/emqx_gateway_exproto/rebar.config @@ -1,7 +1,9 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../../apps/emqx"}}, - {emqx_gateway, {path, "../../apps/emqx_gateway"}} - ]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../emqx_gateway"}} +]}. {plugins, [ {grpc_plugin, {git, "https://github.com/HJianBo/grpc_plugin", {tag, "v0.10.2"}}} diff --git a/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl b/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl index b2e3ad4a7..e04990f5f 100644 --- a/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl +++ b/apps/emqx_gateway_exproto/test/emqx_exproto_echo_svr.erl @@ -148,7 +148,7 @@ on_received_bytes(Stream, _Md) -> fun(Reqs) -> lists:foreach( fun(#{conn := Conn, bytes := Bytes}) -> - #{<<"type">> := Type} = Params = emqx_json:decode(Bytes, [return_maps]), + #{<<"type">> := Type} = Params = emqx_utils_json:decode(Bytes, [return_maps]), _ = handle_in(Conn, Type, Params) end, Reqs @@ -284,16 +284,16 @@ handle_out(Conn, ?TYPE_DISCONNECT) -> %% Frame frame_connect(ClientInfo, Password) -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ type => ?TYPE_CONNECT, clientinfo => ClientInfo, password => Password }). frame_connack(Code) -> - emqx_json:encode(#{type => ?TYPE_CONNACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_CONNACK, code => Code}). frame_publish(Topic, Qos, Payload) -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ type => ?TYPE_PUBLISH, topic => Topic, qos => Qos, @@ -301,19 +301,19 @@ frame_publish(Topic, Qos, Payload) -> }). frame_puback(Code) -> - emqx_json:encode(#{type => ?TYPE_PUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_PUBACK, code => Code}). frame_subscribe(Topic, Qos) -> - emqx_json:encode(#{type => ?TYPE_SUBSCRIBE, topic => Topic, qos => Qos}). + emqx_utils_json:encode(#{type => ?TYPE_SUBSCRIBE, topic => Topic, qos => Qos}). frame_suback(Code) -> - emqx_json:encode(#{type => ?TYPE_SUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_SUBACK, code => Code}). frame_unsubscribe(Topic) -> - emqx_json:encode(#{type => ?TYPE_UNSUBSCRIBE, topic => Topic}). + emqx_utils_json:encode(#{type => ?TYPE_UNSUBSCRIBE, topic => Topic}). frame_unsuback(Code) -> - emqx_json:encode(#{type => ?TYPE_UNSUBACK, code => Code}). + emqx_utils_json:encode(#{type => ?TYPE_UNSUBACK, code => Code}). frame_disconnect() -> - emqx_json:encode(#{type => ?TYPE_DISCONNECT}). + emqx_utils_json:encode(#{type => ?TYPE_DISCONNECT}). diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl index 90a0306b7..c3d23d4d9 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl @@ -351,7 +351,7 @@ opaque_to_json(BaseName, Binary) -> [#{path => BaseName, value => base64:encode(Binary)}]. translate_json(JSONBin) -> - JSONTerm = emqx_json:decode(JSONBin, [return_maps]), + JSONTerm = emqx_utils_json:decode(JSONBin, [return_maps]), BaseName = maps:get(<<"bn">>, JSONTerm, <<>>), ElementList = maps:get(<<"e">>, JSONTerm, []), translate_element(BaseName, ElementList, []). diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl index 2752804dc..c8bfa8974 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl @@ -737,7 +737,7 @@ proto_publish( Epn, Qos, MountedTopic, - emqx_json:encode(Payload), + emqx_utils_json:encode(Payload), #{}, Headers ), @@ -786,7 +786,7 @@ deliver_to_coap(AlternatePath, JsonData, MQTT, CacheMode, WithContext, Session) is_binary(JsonData) -> try - TermData = emqx_json:decode(JsonData, [return_maps]), + TermData = emqx_utils_json:decode(JsonData, [return_maps]), deliver_to_coap(AlternatePath, TermData, MQTT, CacheMode, WithContext, Session) catch ExClass:Error:ST -> diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl index 33ccec2c7..9f388b07c 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -402,7 +402,7 @@ case01_register_report(Config) -> timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -478,7 +478,7 @@ case02_update_deregister(Config) -> ?LOGT("Options got: ~p", [Opts]), Location = maps:get(location_path, Opts), - Register = emqx_json:encode( + Register = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -521,7 +521,7 @@ case02_update_deregister(Config) -> } = test_recv_coap_response(UdpSock), {ok, changed} = Method2, MsgId2 = RspId2, - Update = emqx_json:encode( + Update = emqx_utils_json:encode( #{ <<"msgType">> => <<"update">>, <<"data">> => #{ @@ -754,7 +754,7 @@ case08_reregister(Config) -> timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"msgType">> => <<"register">>, <<"data">> => #{ @@ -871,7 +871,7 @@ case10_read(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -902,7 +902,7 @@ case10_read(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -957,7 +957,7 @@ case10_read_bad_request(Config) -> <<"path">> => <<"/3333/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -979,7 +979,7 @@ case10_read_bad_request(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode(#{ + ReadResult = emqx_utils_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"read">>, @@ -1015,7 +1015,7 @@ case10_read_separate_ack(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1032,7 +1032,7 @@ case10_read_separate_ack(Config) -> ?assertEqual(<<>>, Payload2), test_send_empty_ack(UdpSock, "127.0.0.1", ?PORT, Request2), - ReadResultACK = emqx_json:encode( + ReadResultACK = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1057,7 +1057,7 @@ case10_read_separate_ack(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1100,7 +1100,7 @@ case11_read_object_tlv(Config) -> <<"path">> => <<"/3/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1132,7 +1132,7 @@ case11_read_object_tlv(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1185,7 +1185,7 @@ case11_read_object_json(Config) -> <<"path">> => <<"/3/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1215,7 +1215,7 @@ case11_read_object_json(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1267,7 +1267,7 @@ case12_read_resource_opaque(Config) -> <<"path">> => <<"/3/0/8">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1293,7 +1293,7 @@ case12_read_resource_opaque(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1335,7 +1335,7 @@ case13_read_no_xml(Config) -> <<"msgType">> => <<"read">>, <<"data">> => #{<<"path">> => <<"/9723/0/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1360,7 +1360,7 @@ case13_read_no_xml(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1399,7 +1399,7 @@ case20_single_write(Config) -> <<"value">> => <<"12345">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1426,7 +1426,7 @@ case20_single_write(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1470,7 +1470,7 @@ case20_write(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1497,7 +1497,7 @@ case20_write(Config) -> ), timer:sleep(100), - WriteResult = emqx_json:encode( + WriteResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1547,7 +1547,7 @@ case21_write_object(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1574,7 +1574,7 @@ case21_write_object(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1618,7 +1618,7 @@ case22_write_error(Config) -> ] } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1639,7 +1639,7 @@ case22_write_error(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1677,7 +1677,7 @@ case_create_basic(Config) -> <<"basePath">> => <<"/5">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1703,7 +1703,7 @@ case_create_basic(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1738,7 +1738,7 @@ case_delete_basic(Config) -> <<"msgType">> => <<"delete">>, <<"data">> => #{<<"path">> => <<"/5/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1764,7 +1764,7 @@ case_delete_basic(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1804,7 +1804,7 @@ case30_execute(Config) -> <<"args">> => <<"2,7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1830,7 +1830,7 @@ case30_execute(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1868,7 +1868,7 @@ case31_execute_error(Config) -> <<"args">> => <<"2,7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1894,7 +1894,7 @@ case31_execute_error(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -1931,7 +1931,7 @@ case40_discover(Config) -> <<"path">> => <<"/3/0/7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -1961,7 +1961,7 @@ case40_discover(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2006,7 +2006,7 @@ case50_write_attribute(Config) -> <<"lt">> => <<"5">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(100), Request2 = test_recv_coap_request(UdpSock), @@ -2042,7 +2042,7 @@ case50_write_attribute(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2079,7 +2079,7 @@ case60_observe(Config) -> <<"msgType">> => <<"observe">>, <<"data">> => #{<<"path">> => <<"/3/0/10">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), Request2 = test_recv_coap_request(UdpSock), @@ -2106,7 +2106,7 @@ case60_observe(Config) -> ), timer:sleep(100), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2141,7 +2141,7 @@ case60_observe(Config) -> timer:sleep(100), #coap_message{} = test_recv_coap_response(UdpSock), - ReadResult2 = emqx_json:encode( + ReadResult2 = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, @@ -2173,7 +2173,7 @@ case60_observe(Config) -> <<"path">> => <<"/3/0/10">> } }, - CommandJson3 = emqx_json:encode(Command3), + CommandJson3 = emqx_utils_json:encode(Command3), test_mqtt_broker:publish(CommandTopic, CommandJson3, 0), timer:sleep(50), Request3 = test_recv_coap_request(UdpSock), @@ -2200,7 +2200,7 @@ case60_observe(Config) -> ), timer:sleep(100), - ReadResult3 = emqx_json:encode( + ReadResult3 = emqx_utils_json:encode( #{ <<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, @@ -2242,7 +2242,7 @@ case60_observe(Config) -> %% MsgId1), %% #coap_message{method = Method1} = test_recv_coap_response(UdpSock), %% ?assertEqual({ok,created}, Method1), -%% ReadResult = emqx_json:encode( +%% ReadResult = emqx_utils_json:encode( %% #{<<"msgType">> => <<"register">>, %% <<"data">> => #{ %% <<"alternatePath">> => <<"/">>, @@ -2268,7 +2268,7 @@ case60_observe(Config) -> %% <<"path">> => <<"/19/0/0">> %% } %% }, -%% CommandJson = emqx_json:encode(Command), +%% CommandJson = emqx_utils_json:encode(Command), %% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), %% timer:sleep(50), %% Request2 = test_recv_coap_request(UdpSock), @@ -2325,7 +2325,7 @@ case60_observe(Config) -> %% <<"value">> => base64:encode(<<12345:32>>) %% }}, %% -%% CommandJson = emqx_json:encode(Command), +%% CommandJson = emqx_utils_json:encode(Command), %% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), %% timer:sleep(50), %% Request2 = test_recv_coap_request(UdpSock), @@ -2342,7 +2342,7 @@ case60_observe(Config) -> %% {ok, changed}, #coap_content{}, Request2, true), %% timer:sleep(100), %% -%% ReadResult = emqx_json:encode( +%% ReadResult = emqx_utils_json:encode( %% #{<<"requestID">> => CmdId, %% <<"cacheID">> => CmdId, %% <<"data">> => #{ @@ -2502,7 +2502,7 @@ send_read_command_1(CmdId, _UdpSock) -> <<"msgType">> => <<"read">>, <<"data">> => #{<<"path">> => <<"/3/0/0">>} }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50). @@ -2528,7 +2528,7 @@ verify_read_response_1(CmdId, UdpSock) -> true ), - ReadResult = emqx_json:encode( + ReadResult = emqx_utils_json:encode( #{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl index b3c059b31..6fa46ebbc 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_api_SUITE.erl @@ -131,7 +131,7 @@ t_lookup_read(Config) -> <<"path">> => <<"/3/0/0">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -178,7 +178,7 @@ t_lookup_discover(Config) -> <<"path">> => <<"/3/0/7">> } }, - CommandJson = emqx_json:encode(Command), + CommandJson = emqx_utils_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(200), @@ -350,10 +350,10 @@ no_received_request(ClientId, Path, Action) -> <<"codeMsg">> => <<"reply_not_received">>, <<"path">> => Path }, - ?assertEqual(NotReceived, emqx_json:decode(Response, [return_maps])). + ?assertEqual(NotReceived, emqx_utils_json:decode(Response, [return_maps])). normal_received_request(ClientId, Path, Action) -> Response = call_lookup_api(ClientId, Path, Action), - RCont = emqx_json:decode(Response, [return_maps]), + RCont = emqx_utils_json:decode(Response, [return_maps]), ?assertEqual(list_to_binary(ClientId), maps:get(<<"clientid">>, RCont, undefined)), ?assertEqual(Path, maps:get(<<"path">>, RCont, undefined)), ?assertEqual(Action, maps:get(<<"action">>, RCont, undefined)), diff --git a/apps/emqx_gateway_stomp/rebar.config b/apps/emqx_gateway_stomp/rebar.config index c8675c3ba..cfeb0a195 100644 --- a/apps/emqx_gateway_stomp/rebar.config +++ b/apps/emqx_gateway_stomp/rebar.config @@ -1,4 +1,6 @@ {erl_opts, [debug_info]}. -{deps, [ {emqx, {path, "../../apps/emqx"}}, - {emqx_gateway, {path, "../../apps/emqx_gateway"}} - ]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} +]}. diff --git a/apps/emqx_machine/rebar.config b/apps/emqx_machine/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_machine/rebar.config +++ b/apps/emqx_machine/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 0bee30e35..6bd36aab5 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index 243c4bb8c..6872b150c 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -88,7 +88,7 @@ start_sysmon() -> end. node_status() -> - emqx_json:encode(#{ + emqx_utils_json:encode(#{ backend => mria_rlog:backend(), role => mria_rlog:role() }). diff --git a/apps/emqx_management/rebar.config b/apps/emqx_management/rebar.config index 73cbf471f..b2f5a40af 100644 --- a/apps/emqx_management/rebar.config +++ b/apps/emqx_management/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_management/src/emqx_mgmt_api_banned.erl b/apps/emqx_management/src/emqx_mgmt_api_banned.erl index 0a5cc3afe..508cf7d07 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_banned.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_banned.erl @@ -165,7 +165,7 @@ banned(post, #{body := Body}) -> {ok, Banned} -> {200, format(Banned)}; {error, {already_exist, Old}} -> - OldBannedFormat = emqx_json:encode(format(Old)), + OldBannedFormat = emqx_utils_json:encode(format(Old)), {400, 'ALREADY_EXISTS', OldBannedFormat} end end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl index 69ace16e8..dc7be7671 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl @@ -56,7 +56,7 @@ get_alarms(AssertCount, Activated) -> Qs = "activated=" ++ Activated, Headers = emqx_mgmt_api_test_util:auth_header_(), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, Qs, Headers), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, Data), Page = maps:get(<<"page">>, Meta), Limit = maps:get(<<"limit">>, Meta), diff --git a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl index 241a73dc4..1a396d795 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl @@ -228,7 +228,7 @@ list_app() -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key"]), case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. @@ -236,7 +236,7 @@ read_app(Name) -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["api_key", Name]), case emqx_mgmt_api_test_util:request_api(get, Path, AuthHeader) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -251,7 +251,7 @@ create_app(Name) -> enable => true }, case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -260,7 +260,7 @@ create_unexpired_app(Name, Params) -> Path = emqx_mgmt_api_test_util:api_path(["api_key"]), App = maps:merge(#{name => Name, desc => <<"Note"/utf8>>, enable => true}, Params), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, App) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -273,7 +273,7 @@ update_app(Name, Change) -> AuthHeader = emqx_dashboard_SUITE:auth_header_(), UpdatePath = emqx_mgmt_api_test_util:api_path(["api_key", Name]), case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of - {ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])}; + {ok, Update} -> {ok, emqx_utils_json:decode(Update, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl index c765f00bc..9f1b560f7 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl @@ -160,7 +160,7 @@ t_delete(_Config) -> list_banned() -> Path = emqx_mgmt_api_test_util:api_path(["banned"]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. @@ -168,7 +168,7 @@ create_banned(Banned) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["banned"]), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Banned) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 9f26f8542..6d7733b22 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -58,7 +58,7 @@ t_clients(_) -> %% get /clients ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), - ClientsResponse = emqx_json:decode(Clients, [return_maps]), + ClientsResponse = emqx_utils_json:decode(Clients, [return_maps]), ClientsMeta = maps:get(<<"meta">>, ClientsResponse), ClientsPage = maps:get(<<"page">>, ClientsMeta), ClientsLimit = maps:get(<<"limit">>, ClientsMeta), @@ -70,7 +70,7 @@ t_clients(_) -> %% get /clients/:clientid Client1Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId1)]), {ok, Client1} = emqx_mgmt_api_test_util:request_api(get, Client1Path), - Client1Response = emqx_json:decode(Client1, [return_maps]), + Client1Response = emqx_utils_json:decode(Client1, [return_maps]), ?assertEqual(Username1, maps:get(<<"username">>, Client1Response)), ?assertEqual(ClientId1, maps:get(<<"clientid">>, Client1Response)), ?assertEqual(120, maps:get(<<"expiry_interval">>, Client1Response)), @@ -130,7 +130,7 @@ t_clients(_) -> "", AuthHeader ), - [SubscriptionsData] = emqx_json:decode(SubscriptionsRes, [return_maps]), + [SubscriptionsData] = emqx_utils_json:decode(SubscriptionsRes, [return_maps]), ?assertMatch( #{ <<"clientid">> := ClientId1, @@ -210,7 +210,7 @@ t_query_clients_with_time(_) -> GteParamRfc3339 ++ GteParamStamp ], DecodedResults = [ - emqx_json:decode(Response, [return_maps]) + emqx_utils_json:decode(Response, [return_maps]) || {ok, Response} <- RequestResults ], {LteResponseDecodeds, GteResponseDecodeds} = lists:split(4, DecodedResults), @@ -247,7 +247,7 @@ t_keepalive(_Config) -> {ok, C1} = emqtt:start_link(#{username => Username, clientid => ClientId}), {ok, _} = emqtt:connect(C1), {ok, NewClient} = emqx_mgmt_api_test_util:request_api(put, Path, <<"">>, AuthHeader, Body), - #{<<"keepalive">> := 11} = emqx_json:decode(NewClient, [return_maps]), + #{<<"keepalive">> := 11} = emqx_utils_json:decode(NewClient, [return_maps]), [Pid] = emqx_cm:lookup_channels(list_to_binary(ClientId)), #{conninfo := #{keepalive := Keepalive}} = emqx_connection:info(Pid), ?assertEqual(11, Keepalive), diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 2d24bce99..cc200a4d5 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -235,7 +235,7 @@ t_configs_node(_) -> ?assertEqual(error, ExpType), ?assertMatch({{_, 404, _}, _, _}, ExpRes), {_, _, Body} = ExpRes, - ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_json:decode(Body, [return_maps])), + ?assertMatch(#{<<"code">> := <<"NOT_FOUND">>}, emqx_utils_json:decode(Body, [return_maps])), ?assertMatch({error, {_, 500, _}}, get_configs("bad_node")). @@ -245,7 +245,7 @@ get_config(Name) -> Path = emqx_mgmt_api_test_util:api_path(["configs", Name]), case emqx_mgmt_api_test_util:request_api(get, Path) of {ok, Res} -> - {ok, emqx_json:decode(Res, [return_maps])}; + {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -264,8 +264,8 @@ get_configs(Node, Opts) -> end, URI = emqx_mgmt_api_test_util:api_path(Path), case emqx_mgmt_api_test_util:request_api(get, URI, [], [], [], Opts) of - {ok, {_, _, Res}} -> {ok, emqx_json:decode(Res, [return_maps])}; - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, {_, _, Res}} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -273,7 +273,7 @@ update_config(Name, Change) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), UpdatePath = emqx_mgmt_api_test_util:api_path(["configs", Name]), case emqx_mgmt_api_test_util:request_api(put, UpdatePath, "", AuthHeader, Change) of - {ok, Update} -> {ok, emqx_json:decode(Update, [return_maps])}; + {ok, Update} -> {ok, emqx_utils_json:decode(Update, [return_maps])}; Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index 3238588e2..90808d4be 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -385,7 +385,7 @@ action_listener(ID, Action, Running) -> request(Method, Url, QueryParams, Body) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(Method, Url, QueryParams, AuthHeader, Body) of - {ok, Res} -> emqx_json:decode(Res, [return_maps]); + {ok, Res} -> emqx_utils_json:decode(Res, [return_maps]); Error -> Error end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl index 93cb69f0a..7ecfe9817 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl @@ -32,13 +32,13 @@ end_per_suite(_) -> t_metrics_api(_) -> {ok, MetricsResponse} = request_helper("metrics?aggregate=true"), - MetricsFromAPI = emqx_json:decode(MetricsResponse, [return_maps]), + MetricsFromAPI = emqx_utils_json:decode(MetricsResponse, [return_maps]), AggregateMetrics = emqx_mgmt:get_metrics(), match_helper(AggregateMetrics, MetricsFromAPI). t_single_node_metrics_api(_) -> {ok, MetricsResponse} = request_helper("metrics"), - [MetricsFromAPI] = emqx_json:decode(MetricsResponse, [return_maps]), + [MetricsFromAPI] = emqx_utils_json:decode(MetricsResponse, [return_maps]), LocalNodeMetrics = maps:from_list( emqx_mgmt:get_metrics(node()) ++ [{node, to_bin(node())}] ), diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 30313e555..7e4ccc541 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -53,7 +53,7 @@ end_per_testcase(_, Config) -> t_nodes_api(_) -> NodesPath = emqx_mgmt_api_test_util:api_path(["nodes"]), {ok, Nodes} = emqx_mgmt_api_test_util:request_api(get, NodesPath), - NodesResponse = emqx_json:decode(Nodes, [return_maps]), + NodesResponse = emqx_utils_json:decode(Nodes, [return_maps]), LocalNodeInfo = hd(NodesResponse), Node = binary_to_atom(maps:get(<<"node">>, LocalNodeInfo), utf8), ?assertEqual(Node, node()), @@ -63,7 +63,7 @@ t_nodes_api(_) -> NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]), {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), NodeNameResponse = - binary_to_atom(maps:get(<<"node">>, emqx_json:decode(NodeInfo, [return_maps])), utf8), + binary_to_atom(maps:get(<<"node">>, emqx_utils_json:decode(NodeInfo, [return_maps])), utf8), ?assertEqual(node(), NodeNameResponse), BadNodePath = emqx_mgmt_api_test_util:api_path(["nodes", "badnode"]), @@ -75,7 +75,7 @@ t_nodes_api(_) -> t_log_path(_) -> NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]), {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath), - #{<<"log_path">> := Path} = emqx_json:decode(NodeInfo, [return_maps]), + #{<<"log_path">> := Path} = emqx_utils_json:decode(NodeInfo, [return_maps]), ?assertEqual( <<"log">>, filename:basename(Path) @@ -85,7 +85,7 @@ t_node_stats_api(_) -> StatsPath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "stats"]), SystemStats = emqx_mgmt:get_stats(), {ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath), - Stats = emqx_json:decode(StatsResponse, [return_maps]), + Stats = emqx_utils_json:decode(StatsResponse, [return_maps]), Fun = fun(Key) -> ?assertEqual(maps:get(Key, SystemStats), maps:get(atom_to_binary(Key, utf8), Stats)) @@ -103,7 +103,7 @@ t_node_metrics_api(_) -> emqx_mgmt_api_test_util:api_path(["nodes", atom_to_binary(node(), utf8), "metrics"]), SystemMetrics = emqx_mgmt:get_metrics(), {ok, MetricsResponse} = emqx_mgmt_api_test_util:request_api(get, MetricsPath), - Metrics = emqx_json:decode(MetricsResponse, [return_maps]), + Metrics = emqx_utils_json:decode(MetricsResponse, [return_maps]), Fun = fun(Key) -> ?assertEqual(maps:get(Key, SystemMetrics), maps:get(atom_to_binary(Key, utf8), Metrics)) diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 24e55494d..62fed8211 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -136,14 +136,14 @@ t_bad_plugin(Config) -> list_plugins() -> Path = emqx_mgmt_api_test_util:api_path(["plugins"]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Apps} -> {ok, emqx_json:decode(Apps, [return_maps])}; + {ok, Apps} -> {ok, emqx_utils_json:decode(Apps, [return_maps])}; Error -> Error end. describe_plugins(Name) -> Path = emqx_mgmt_api_test_util:api_path(["plugins", Name]), case emqx_mgmt_api_test_util:request_api(get, Path) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -172,7 +172,7 @@ update_boot_order(Name, MoveBody) -> Auth = emqx_mgmt_api_test_util:auth_header_(), Path = emqx_mgmt_api_test_util:api_path(["plugins", Name, "move"]), case emqx_mgmt_api_test_util:request_api(post, Path, "", Auth, MoveBody) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -206,7 +206,7 @@ create_renamed_package(PackagePath, NewNameVsn) -> NewPackagePath. update_release_json(["release.json"], FileContent, NewName) -> - ContentMap = emqx_json:decode(FileContent, [return_maps]), - emqx_json:encode(ContentMap#{<<"name">> => NewName}); + ContentMap = emqx_utils_json:decode(FileContent, [return_maps]), + emqx_utils_json:encode(ContentMap#{<<"name">> => NewName}); update_release_json(_FileName, FileContent, _NewName) -> FileContent. diff --git a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl index 44b8069f5..303a73b41 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_publish_SUITE.erl @@ -352,4 +352,4 @@ receive_assert(Topic, Qos, Payload) -> end. decode_json(In) -> - emqx_json:decode(In, [return_maps]). + emqx_utils_json:decode(In, [return_maps]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl index 7236ac9e4..4099426b8 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl @@ -33,7 +33,7 @@ end_per_suite(_) -> t_stats_api(_) -> S = emqx_mgmt_api_test_util:api_path(["stats?aggregate=false"]), {ok, S1} = emqx_mgmt_api_test_util:request_api(get, S), - [Stats1] = emqx_json:decode(S1, [return_maps]), + [Stats1] = emqx_utils_json:decode(S1, [return_maps]), SystemStats1 = emqx_mgmt:get_stats(), Fun1 = fun(Key) -> @@ -43,7 +43,7 @@ t_stats_api(_) -> StatsPath = emqx_mgmt_api_test_util:api_path(["stats?aggregate=true"]), SystemStats = emqx_mgmt:get_stats(), {ok, StatsResponse} = emqx_mgmt_api_test_util:request_api(get, StatsPath), - Stats = emqx_json:decode(StatsResponse, [return_maps]), + Stats = emqx_utils_json:decode(StatsResponse, [return_maps]), ?assertEqual(erlang:length(maps:keys(SystemStats)), erlang:length(maps:keys(Stats))), Fun = fun(Key) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index b9e9fffd8..a23d70f2f 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -55,7 +55,7 @@ t_subscription_api(Config) -> {ok, _, _} = emqtt:subscribe(Client, ?TOPIC2), Path = emqx_mgmt_api_test_util:api_path(["subscriptions"]), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, Data), ?assertEqual(1, maps:get(<<"page">>, Meta)), ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), @@ -158,7 +158,7 @@ t_list_with_internal_subscription(_Config) -> request_json(Method, Query, Headers) when is_list(Query) -> Qs = uri_string:compose_query(Query), {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers), - emqx_json:decode(MatchRes, [return_maps]). + emqx_utils_json:decode(MatchRes, [return_maps]). path() -> emqx_mgmt_api_test_util:api_path(["subscriptions"]). diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 782f493fe..5fe4ca937 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -108,7 +108,8 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when end, do_request_api( Method, - {NewUrl, build_http_header(AuthOrHeaders), "application/json", emqx_json:encode(Body)}, + {NewUrl, build_http_header(AuthOrHeaders), "application/json", + emqx_utils_json:encode(Body)}, Opts ). diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl index 0c2e684b4..659ae0d44 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl @@ -49,7 +49,7 @@ t_nodes_api(Config) -> %% list all Path = emqx_mgmt_api_test_util:api_path(["topics"]), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path), - RoutesData = emqx_json:decode(Response, [return_maps]), + RoutesData = emqx_utils_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, RoutesData), ?assertEqual(1, maps:get(<<"page">>, Meta)), ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), @@ -68,7 +68,7 @@ t_nodes_api(Config) -> ]), Headers = emqx_mgmt_api_test_util:auth_header_(), {ok, MatchResponse} = emqx_mgmt_api_test_util:request_api(get, Path, QS, Headers), - MatchData = emqx_json:decode(MatchResponse, [return_maps]), + MatchData = emqx_utils_json:decode(MatchResponse, [return_maps]), ?assertMatch( #{<<"count">> := 1, <<"page">> := 1, <<"limit">> := 100}, maps:get(<<"meta">>, MatchData) @@ -90,6 +90,6 @@ t_nodes_api(Config) -> [ #{<<"topic">> := Topic, <<"node">> := Node1}, #{<<"topic">> := Topic, <<"node">> := Node2} - ] = emqx_json:decode(RouteResponse, [return_maps]), + ] = emqx_utils_json:decode(RouteResponse, [return_maps]), ?assertEqual(lists:usort([Node, atom_to_binary(Slave)]), lists:usort([Node1, Node2])). diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 7922bbb40..0102eb56c 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -384,7 +384,7 @@ api_path(Path) -> emqx_mgmt_api_test_util:api_path([Path]). json(Data) -> - {ok, Jsx} = emqx_json:safe_decode(Data, [return_maps]), + {ok, Jsx} = emqx_utils_json:safe_decode(Data, [return_maps]), Jsx. load() -> diff --git a/apps/emqx_modules/rebar.config b/apps/emqx_modules/rebar.config index 9688d5043..ff542aed7 100644 --- a/apps/emqx_modules/rebar.config +++ b/apps/emqx_modules/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {emqx_conf, {path, "../emqx_conf"}} ]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl index 6d5c772f0..1d49fb3e7 100644 --- a/apps/emqx_modules/src/emqx_telemetry.erl +++ b/apps/emqx_modules/src/emqx_telemetry.erl @@ -356,7 +356,7 @@ get_telemetry(State0 = #state{node_uuid = NodeUUID, cluster_uuid = ClusterUUID}) report_telemetry(State0 = #state{url = URL}) -> {State, Data} = get_telemetry(State0), - case emqx_json:safe_encode(Data) of + case emqx_utils_json:safe_encode(Data) of {ok, Bin} -> httpc_request(post, URL, [], Bin), ?tp(debug, telemetry_data_reported, #{}); diff --git a/apps/emqx_modules/src/emqx_telemetry_api.erl b/apps/emqx_modules/src/emqx_telemetry_api.erl index 798c3ad17..b7209d146 100644 --- a/apps/emqx_modules/src/emqx_telemetry_api.erl +++ b/apps/emqx_modules/src/emqx_telemetry_api.erl @@ -243,7 +243,7 @@ status(put, #{body := Body}) -> data(get, _Request) -> case emqx_modules_conf:is_telemetry_enabled() of true -> - {200, emqx_json:encode(get_telemetry_data())}; + {200, emqx_utils_json:encode(get_telemetry_data())}; false -> {404, #{ code => ?NOT_FOUND, diff --git a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl index ed3cd9292..b2d854d22 100644 --- a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl @@ -229,7 +229,7 @@ t_large_payload(_) -> %%-------------------------------------------------------------------- decode_json(Data) -> - BinJson = emqx_json:decode(Data, [return_maps]), + BinJson = emqx_utils_json:decode(Data, [return_maps]), emqx_map_lib:unsafe_atom_key_map(BinJson). clear_all_record() -> diff --git a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl index ddb136f1e..68d12a2c1 100644 --- a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl @@ -75,7 +75,7 @@ t_mqtt_topic_rewrite(_) -> ?assertEqual( Rules, - emqx_json:decode(Result, [return_maps]) + emqx_utils_json:decode(Result, [return_maps]) ). t_mqtt_topic_rewrite_limit(_) -> diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl index a61781e13..bb5f39c1f 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -512,7 +512,7 @@ t_send_after_enable(_) -> ), receive {request, post, _URL, _Headers, Body} -> - {ok, Decoded} = emqx_json:safe_decode(Body, [return_maps]), + {ok, Decoded} = emqx_utils_json:safe_decode(Body, [return_maps]), ?assertMatch( #{ <<"uuid">> := _, diff --git a/apps/emqx_plugin_libs/rebar.config b/apps/emqx_plugin_libs/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_plugin_libs/rebar.config +++ b/apps/emqx_plugin_libs/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_plugin_libs/src/emqx_placeholder.erl b/apps/emqx_plugin_libs/src/emqx_placeholder.erl index 1f93c1d3e..18ef9e8fb 100644 --- a/apps/emqx_plugin_libs/src/emqx_placeholder.erl +++ b/apps/emqx_plugin_libs/src/emqx_placeholder.erl @@ -240,7 +240,7 @@ sql_data(Bin) when is_binary(Bin) -> Bin; sql_data(Num) when is_number(Num) -> Num; sql_data(Bool) when is_boolean(Bool) -> Bool; sql_data(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -sql_data(Map) when is_map(Map) -> emqx_json:encode(Map). +sql_data(Map) when is_map(Map) -> emqx_utils_json:encode(Map). -spec bin(term()) -> binary(). bin(Val) -> emqx_plugin_libs_rule:bin(Val). diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index d1a821895..40f88a0dc 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -236,11 +236,11 @@ tcp_connectivity(Host, Port, Timeout) -> str(Bin) when is_binary(Bin) -> binary_to_list(Bin); str(Num) when is_number(Num) -> number_to_list(Num); str(Atom) when is_atom(Atom) -> atom_to_list(Atom); -str(Map) when is_map(Map) -> binary_to_list(emqx_json:encode(Map)); +str(Map) when is_map(Map) -> binary_to_list(emqx_utils_json:encode(Map)); str(List) when is_list(List) -> case io_lib:printable_list(List) of true -> List; - false -> binary_to_list(emqx_json:encode(List)) + false -> binary_to_list(emqx_utils_json:encode(List)) end; str(Data) -> error({invalid_str, Data}). @@ -258,11 +258,11 @@ utf8_str(Str) -> bin(Bin) when is_binary(Bin) -> Bin; bin(Num) when is_number(Num) -> number_to_binary(Num); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -bin(Map) when is_map(Map) -> emqx_json:encode(Map); +bin(Map) when is_map(Map) -> emqx_utils_json:encode(Map); bin(List) when is_list(List) -> case io_lib:printable_list(List) of true -> list_to_binary(List); - false -> emqx_json:encode(List) + false -> emqx_utils_json:encode(List) end; bin(Data) -> error({invalid_bin, Data}). @@ -312,7 +312,7 @@ float2str(Float, Precision) when is_float(Float) and is_integer(Precision) -> float_to_binary(Float, [{decimals, Precision}, compact]). map(Bin) when is_binary(Bin) -> - case emqx_json:decode(Bin, [return_maps]) of + case emqx_utils_json:decode(Bin, [return_maps]) of Map = #{} -> Map; _ -> error({invalid_map, Bin}) end; diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 88b3d27a2..7b9a6cc48 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -2,6 +2,7 @@ {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}} ]}. diff --git a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl index c4867d9fd..e29d46720 100644 --- a/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl +++ b/apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl @@ -66,7 +66,7 @@ t_prometheus_api(_) -> Auth = emqx_mgmt_api_test_util:auth_header_(), {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), - Conf = emqx_json:decode(Response, [return_maps]), + Conf = emqx_utils_json:decode(Response, [return_maps]), ?assertMatch( #{ <<"push_gateway_server">> := _, @@ -84,7 +84,7 @@ t_prometheus_api(_) -> NewConf = Conf#{<<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"disabled">>}, {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf), - Conf2 = emqx_json:decode(Response2, [return_maps]), + Conf2 = emqx_utils_json:decode(Response2, [return_maps]), ?assertMatch(NewConf, Conf2), ?assertEqual({ok, []}, application:get_env(prometheus, vm_statistics_collector_metrics)), ?assertEqual({ok, all}, application:get_env(prometheus, vm_memory_collector_metrics)), @@ -106,7 +106,7 @@ t_stats_api(_) -> Headers = [{"accept", "application/json"}, Auth], {ok, Response} = emqx_mgmt_api_test_util:request_api(get, Path, "", Headers), - Data = emqx_json:decode(Response, [return_maps]), + Data = emqx_utils_json:decode(Response, [return_maps]), ?assertMatch(#{<<"client">> := _, <<"delivery">> := _}, Data), {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path, "", Auth), diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index 65de71fdd..a178e10a1 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -1,6 +1,9 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [ diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index ba96887a2..2b92b3f59 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -72,7 +72,7 @@ t_config(_Config) -> ), UpdateConf = fun(Enable) -> - RawConf = emqx_json:decode(ConfJson, [return_maps]), + RawConf = emqx_utils_json:decode(ConfJson, [return_maps]), UpdateJson = RawConf#{<<"enable">> := Enable}, {ok, UpdateResJson} = request_api( put, @@ -81,7 +81,7 @@ t_config(_Config) -> auth_header_(), UpdateJson ), - UpdateRawConf = emqx_json:decode(UpdateResJson, [return_maps]), + UpdateRawConf = emqx_utils_json:decode(UpdateResJson, [return_maps]), ?assertEqual(Enable, maps:get(<<"enable">>, UpdateRawConf)) end, @@ -224,7 +224,7 @@ t_lookup_and_delete(_) -> t_change_storage_type(_Config) -> Path = api_path(["mqtt", "retainer"]), {ok, ConfJson} = request_api(get, Path), - RawConf = emqx_json:decode(ConfJson, [return_maps]), + RawConf = emqx_utils_json:decode(ConfJson, [return_maps]), %% pre-conditions ?assertMatch( #{ @@ -271,7 +271,7 @@ t_change_storage_type(_Config) -> auth_header_(), ChangedConf ), - UpdatedRawConf = emqx_json:decode(UpdateResJson, [return_maps]), + UpdatedRawConf = emqx_utils_json:decode(UpdateResJson, [return_maps]), ?assertMatch( #{ <<"backend">> := #{ @@ -311,7 +311,7 @@ t_change_storage_type(_Config) -> %% HTTP Request %%-------------------------------------------------------------------- decode_json(Data) -> - BinJson = emqx_json:decode(Data, [return_maps]), + BinJson = emqx_utils_json:decode(Data, [return_maps]), emqx_map_lib:unsafe_atom_key_map(BinJson). %%-------------------------------------------------------------------- diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 110caa33d..07c53d3e3 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- {deps, [ - {emqx, {path, "../emqx"}} + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} ]}. {erl_opts, [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index c4a6e2e73..40158d342 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -213,7 +213,7 @@ replace_simple_var(Val, _Data, _Default) -> Val. format_msg([], Selected) -> - emqx_json:encode(Selected); + emqx_utils_json:encode(Selected); format_msg(Tokens, Selected) -> emqx_plugin_libs_rule:proc_tmpl(Tokens, Selected). diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index f640f8303..c63d6275f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -468,7 +468,7 @@ err_msg(Msg) -> encode_nested_error(RuleError, Reason) when is_tuple(Reason) -> encode_nested_error(RuleError, element(1, Reason)); encode_nested_error(RuleError, Reason) -> - case emqx_json:safe_encode([{RuleError, Reason}]) of + case emqx_utils_json:safe_encode([{RuleError, Reason}]) of {ok, Json} -> Json; _ -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 79e0406c1..944aa02f8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -987,10 +987,10 @@ base64_decode(Data) when is_binary(Data) -> base64:decode(Data). json_encode(Data) -> - emqx_json:encode(Data). + emqx_utils_json:encode(Data). json_decode(Data) -> - emqx_json:decode(Data, [return_maps]). + emqx_utils_json:decode(Data, [return_maps]). term_encode(Term) -> erlang:term_to_binary(Term). diff --git a/apps/emqx_rule_engine/src/emqx_rule_maps.erl b/apps/emqx_rule_engine/src/emqx_rule_maps.erl index 13f99c88b..3dfffca46 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_maps.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_maps.erl @@ -86,7 +86,7 @@ general_map_put(Key, Val, Map, OrgData) -> ). general_find(KeyOrIndex, Data, OrgData, Handler) when is_binary(Data) -> - try emqx_json:decode(Data, [return_maps]) of + try emqx_utils_json:decode(Data, [return_maps]) of Json -> general_find(KeyOrIndex, Json, OrgData, Handler) catch _:_ -> Handler(not_found) diff --git a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl index 51491df53..d7412d03c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_runtime.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_runtime.erl @@ -495,7 +495,7 @@ cache_payload(DecodedP) -> safe_decode_and_cache(MaybeJson) -> try - cache_payload(emqx_json:decode(MaybeJson, [return_maps])) + cache_payload(emqx_utils_json:decode(MaybeJson, [return_maps])) catch _:_ -> error({decode_json_failed, MaybeJson}) end. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 93d7c7352..eb253e516 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -614,7 +614,9 @@ t_event_client_disconnected_normal(_Config) -> receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), - ?assertMatch(#{<<"reason">> := <<"normal">>}, emqx_json:decode(Payload, [return_maps])) + ?assertMatch( + #{<<"reason">> := <<"normal">>}, emqx_utils_json:decode(Payload, [return_maps]) + ) after 1000 -> ct:fail(wait_for_repub_disconnected_normal) end, @@ -651,7 +653,9 @@ t_event_client_disconnected_kicked(_Config) -> receive {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), - ?assertMatch(#{<<"reason">> := <<"kicked">>}, emqx_json:decode(Payload, [return_maps])) + ?assertMatch( + #{<<"reason">> := <<"kicked">>}, emqx_utils_json:decode(Payload, [return_maps]) + ) after 1000 -> ct:fail(wait_for_repub_disconnected_kicked) end, @@ -692,7 +696,7 @@ t_event_client_disconnected_discarded(_Config) -> {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), ?assertMatch( - #{<<"reason">> := <<"discarded">>}, emqx_json:decode(Payload, [return_maps]) + #{<<"reason">> := <<"discarded">>}, emqx_utils_json:decode(Payload, [return_maps]) ) after 1000 -> ct:fail(wait_for_repub_disconnected_discarded) @@ -737,7 +741,7 @@ t_event_client_disconnected_takenover(_Config) -> {publish, #{topic := T, payload := Payload}} -> ?assertEqual(RepubT, T), ?assertMatch( - #{<<"reason">> := <<"takenover">>}, emqx_json:decode(Payload, [return_maps]) + #{<<"reason">> := <<"takenover">>}, emqx_utils_json:decode(Payload, [return_maps]) ) after 1000 -> ct:fail(wait_for_repub_disconnected_discarded) @@ -2800,7 +2804,7 @@ verify_event(EventName) -> [ begin %% verify fields can be formatted to JSON string - _ = emqx_json:encode(Fields), + _ = emqx_utils_json:encode(Fields), %% verify metadata fields verify_metadata_fields(EventName, Fields), %% verify available fields for each event name diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl index e94806a7b..8d7546fca 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl @@ -46,13 +46,13 @@ end_per_suite(_Config) -> ok. init_per_testcase(t_crud_rule_api, Config) -> - meck:new(emqx_json, [passthrough]), + meck:new(emqx_utils_json, [passthrough]), init_per_testcase(common, Config); init_per_testcase(_, Config) -> Config. end_per_testcase(t_crud_rule_api, Config) -> - meck:unload(emqx_json), + meck:unload(emqx_utils_json), end_per_testcase(common, Config); end_per_testcase(_, _Config) -> {200, #{data := Rules}} = @@ -136,7 +136,7 @@ t_crud_rule_api(_Config) -> ), ?assertMatch( #{<<"select_and_transform_error">> := <<"decode_json_failed">>}, - emqx_json:decode(SelectAndTransformJsonError, [return_maps]) + emqx_utils_json:decode(SelectAndTransformJsonError, [return_maps]) ), {400, #{ code := 'BAD_REQUEST', @@ -150,7 +150,7 @@ t_crud_rule_api(_Config) -> ), ?assertMatch( #{<<"select_and_transform_error">> := <<"badarg">>}, - emqx_json:decode(SelectAndTransformBadArgError, [return_maps]) + emqx_utils_json:decode(SelectAndTransformBadArgError, [return_maps]) ), {400, #{ code := 'BAD_REQUEST', @@ -162,7 +162,7 @@ t_crud_rule_api(_Config) -> ) ), ?assertMatch({match, _}, re:run(BadSqlMessage, "syntax error")), - meck:expect(emqx_json, safe_encode, 1, {error, foo}), + meck:expect(emqx_utils_json, safe_encode, 1, {error, foo}), ?assertMatch( {400, #{ code := 'BAD_REQUEST', diff --git a/apps/emqx_slow_subs/rebar.config b/apps/emqx_slow_subs/rebar.config index 9f17b7657..dee2902a5 100644 --- a/apps/emqx_slow_subs/rebar.config +++ b/apps/emqx_slow_subs/rebar.config @@ -1,5 +1,8 @@ %% -*- mode: erlang -*- -{deps, [{emqx, {path, "../emqx"}}]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl index 6b0721e3d..ae74348e0 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl @@ -108,7 +108,7 @@ t_get_history(_) -> "page=1&limit=10", auth_header_() ), - #{<<"data">> := [First | _]} = emqx_json:decode(Data, [return_maps]), + #{<<"data">> := [First | _]} = emqx_utils_json:decode(Data, [return_maps]), ?assertMatch( #{ @@ -165,7 +165,7 @@ t_settting(_) -> ?assertEqual(Conf2#{stats_type := <<"internal">>}, GetReturn). decode_json(Data) -> - BinJosn = emqx_json:decode(Data, [return_maps]), + BinJosn = emqx_utils_json:decode(Data, [return_maps]), emqx_map_lib:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> @@ -187,7 +187,7 @@ request_api(Method, Url, QueryParams, Auth, Body) -> "" -> Url; _ -> Url ++ "?" ++ QueryParams end, - do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_json:encode(Body)}). + do_request_api(Method, {NewUrl, [Auth], "application/json", emqx_utils_json:encode(Body)}). do_request_api(Method, Request) -> ct:pal("Method: ~p, Request: ~p", [Method, Request]), diff --git a/apps/emqx_statsd/rebar.config b/apps/emqx_statsd/rebar.config index bb9a14272..a1383d920 100644 --- a/apps/emqx_statsd/rebar.config +++ b/apps/emqx_statsd/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, {estatsd, {git, "https://github.com/emqx/estatsd", {tag, "0.1.0"}}} ]}. diff --git a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl index bcc710050..b5669e4b9 100644 --- a/apps/emqx_statsd/test/emqx_statsd_SUITE.erl +++ b/apps/emqx_statsd/test/emqx_statsd_SUITE.erl @@ -200,7 +200,7 @@ request(Method) -> request(Method, []). request(Method, Body) -> case request(Method, uri(["statsd"]), Body) of {ok, 200, Res} -> - {ok, emqx_json:decode(Res, [return_maps])}; + {ok, emqx_utils_json:decode(Res, [return_maps])}; {ok, _Status, _} -> error end. diff --git a/apps/emqx_utils/README.md b/apps/emqx_utils/README.md new file mode 100644 index 000000000..482b7a947 --- /dev/null +++ b/apps/emqx_utils/README.md @@ -0,0 +1,43 @@ +# [Application Name] - [Mandatory] +> 0. App overview introduction +> 1. let people know what your project can do specifically. Is it a base +> library dependency, or what kind of functionality is provided to the user? +> 2. Provide context and add a link to any reference visitors might be +> unfamiliar with. +> 3. Design details, implementation technology architecture, Roadmap, etc. + +# [Features] - [Optional] +> A List of features your application provided. If the feature is quite simple, just +> list in the previous section. + +# [Limitation] - [Optional] +> Explain the limitations of the implementation used in your application. + +# [Documention links] - [Mandatory] +> You can use the official docs link to provide a detailed explanation of +> the concept and functions of this application. + +# [Installation] - [Optional] +> How users can obtain this application. In most cases, this section is unnecessary. +> Most of the applications are released with the EMQX distribution package. +> Otherwise, it is necessary to provide step-by-step instructions on how +> to install this application. + +# [Usage] - [Optional] +> This section explains how users can use the features provided by the application. + +# [Basic Usage Guide] - [Optional] +> Simple and reproducible introduction on how to use/start it. + +# [Configurations] - [Mandatory] +> Most important configurations that this application depends on. +> It would be best to attach official documentation if it's available. + +# [HTTP APIs] - [Optional] +> Same as configuration. + +# [Other] - [Optional] +> Other topics that users may need to know can be placed here. + +# Contributing - [Mandatory] +Please see our [contributing.md](../../CONTRIBUTING.md). diff --git a/apps/emqx_utils/rebar.config b/apps/emqx_utils/rebar.config new file mode 100644 index 000000000..4c39cfe64 --- /dev/null +++ b/apps/emqx_utils/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang -*- + +{erl_opts, [ + debug_info +]}. + +{deps, [ + {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} +]}. + +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src new file mode 100644 index 000000000..e1f10c139 --- /dev/null +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -0,0 +1,25 @@ +%% -*- mode: erlang -*- +{application, emqx_utils, [ + {description, "An OTP application"}, + % strict semver, bump manually! + {vsn, "5.0.0"}, + {modules, [ + emqx_utils, + emqx_utils_binary, + emqx_utils_ets, + emqx_utils_json, + emqx_utils_maps + ]}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {licenses, ["Apache-2.0"]}, + {maintainers, ["EMQX Team "]}, + {links, [ + {"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx/emqx"} + ]} +]}. diff --git a/apps/emqx/src/emqx_json.erl b/apps/emqx_utils/src/emqx_utils_json.erl similarity index 90% rename from apps/emqx/src/emqx_json.erl rename to apps/emqx_utils/src/emqx_utils_json.erl index 7827b98c9..2f0c6f88b 100644 --- a/apps/emqx/src/emqx_json.erl +++ b/apps/emqx_utils/src/emqx_utils_json.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_json). +-module(emqx_utils_json). -compile(inline). @@ -46,11 +46,11 @@ ]} ). --type encode_options() :: jiffy:encode_options(). --type decode_options() :: jiffy:decode_options(). +-type encode_options() :: emqx_utils_json:encode_options(). +-type decode_options() :: emqx_utils_json:decode_options(). -type json_text() :: iolist() | binary(). --type json_term() :: jiffy:jiffy_decode_result(). +-type json_term() :: emqx_utils_json:jiffy_decode_result(). -export_type([json_text/0, json_term/0]). -export_type([decode_options/0, encode_options/0]). @@ -61,7 +61,7 @@ encode(Term) -> -spec encode(json_term(), encode_options()) -> json_text(). encode(Term, Opts) -> - to_binary(jiffy:encode(to_ejson(Term), Opts)). + to_binary(emqx_utils_json:encode(to_ejson(Term), Opts)). -spec safe_encode(json_term()) -> {ok, json_text()} | {error, Reason :: term()}. @@ -83,7 +83,7 @@ decode(Json) -> decode(Json, []). -spec decode(json_text(), decode_options()) -> json_term(). decode(Json, Opts) -> - from_ejson(jiffy:decode(Json, Opts)). + from_ejson(emqx_utils_json:decode(Json, Opts)). -spec safe_decode(json_text()) -> {ok, json_term()} | {error, Reason :: term()}. diff --git a/apps/emqx_utils/test/emqx_utils_SUITE.erl b/apps/emqx_utils/test/emqx_utils_SUITE.erl new file mode 100644 index 000000000..09a8d085c --- /dev/null +++ b/apps/emqx_utils/test/emqx_utils_SUITE.erl @@ -0,0 +1,48 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_utils_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([emqx_conf, emqx_utils]), + Config. + +end_per_suite(_Config) -> + emqx_common_test_helpers:stop_apps([emqx_conf, emqx_utils]), + ok. + +init_per_testcase(TestCase, Config) -> + emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config). + +end_per_testcase(TestCase, Config) -> + emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config). + +t_fail(init, Config) -> + Config; +t_fail('end', _Config) -> + ok. + +t_fail(_Config) -> + ?assert(false). diff --git a/apps/emqx/test/emqx_json_SUITE.erl b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl similarity index 93% rename from apps/emqx/test/emqx_json_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_json_SUITE.erl index a0bf48e4e..889cde595 100644 --- a/apps/emqx/test/emqx_json_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_json_SUITE). +-module(emqx_utils_json_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -22,7 +22,7 @@ -include_lib("eunit/include/eunit.hrl"). -import( - emqx_json, + emqx_utils_json, [ encode/1, decode/1, @@ -51,7 +51,7 @@ %% #{<<"foo">> => <<"bar">>} -> {"foo": "bar"} -> #{<<"foo">> => <<"bar">>} %%-------------------------------------------------------------------- -%% but in emqx_json, we use the jsx style for it: +%% but in emqx_utils_json, we use the jsx style for it: %%-------------------------------------------------------------------- %% Erlang JSON Erlang %% ------------------------------------------------------------------- @@ -127,12 +127,12 @@ t_safe_decode_encode(_) -> [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{foo, bar}]), [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{<<"foo">>, <<"bar">>}]), [[{<<"foo">>, <<"bar">>}]] = safe_encode_decode([[{<<"foo">>, <<"bar">>}]]), - {ok, Json} = emqx_json:safe_encode(#{<<"foo">> => <<"bar">>}), - {ok, #{<<"foo">> := <<"bar">>}} = emqx_json:safe_decode(Json, [return_maps]). + {ok, Json} = emqx_utils_json:safe_encode(#{<<"foo">> => <<"bar">>}), + {ok, #{<<"foo">> := <<"bar">>}} = emqx_utils_json:safe_decode(Json, [return_maps]). safe_encode_decode(Term) -> - {ok, Json} = emqx_json:safe_encode(Term), - case emqx_json:safe_decode(Json) of + {ok, Json} = emqx_utils_json:safe_encode(Term), + case emqx_utils_json:safe_decode(Json) of {ok, {NTerm}} -> NTerm; {ok, NTerm} -> NTerm end. diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index afd90f622..b26df658a 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -3,6 +3,7 @@ , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + , {emqx_utils, {path, "../emqx_utils"}} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index 208b68de6..f37d7fb6a 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -273,7 +273,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 0c0dbbe04..e899b0fcd 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -208,7 +208,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 75d2d2d8c..3cc4c75e4 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -204,7 +204,7 @@ create_bridge_http(Config, GCPPubSubConfigOverrides) -> ct:pal("probe result: ~p", [ProbeResult]), Res = case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res0} -> {ok, emqx_json:decode(Res0, [return_maps])}; + {ok, Res0} -> {ok, emqx_utils_json:decode(Res0, [return_maps])}; Error -> Error end, ct:pal("bridge creation result: ~p", [Res]), @@ -222,7 +222,7 @@ create_rule_and_action_http(Config) -> Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -234,7 +234,7 @@ success_http_handler() -> Rep = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{messageIds => [<<"6058891368195201">>]}), + emqx_utils_json:encode(#{messageIds => [<<"6058891368195201">>]}), Req ), {ok, Rep, State} @@ -274,7 +274,7 @@ gcp_pubsub_config(Config) -> PubSubTopic = proplists:get_value(pubsub_topic, Config, <<"mytopic">>), PipelineSize = proplists:get_value(pipeline_size, Config, 100), ServiceAccountJSON = proplists:get_value(pubsub_topic, Config, generate_service_account_json()), - ServiceAccountJSONStr = emqx_json:encode(ServiceAccountJSON), + ServiceAccountJSONStr = emqx_utils_json:encode(ServiceAccountJSON), GUID = emqx_guid:to_hexstr(emqx_guid:gen()), Name = <<(atom_to_binary(?MODULE))/binary, (GUID)/binary>>, ConfigString = @@ -463,7 +463,7 @@ assert_valid_request_headers(Headers, ServiceAccountJSON) -> end. assert_valid_request_body(Body) -> - BodyMap = emqx_json:decode(Body, [return_maps]), + BodyMap = emqx_utils_json:decode(Body, [return_maps]), ?assertMatch(#{<<"messages">> := [_ | _]}, BodyMap), #{<<"messages">> := Messages} = BodyMap, lists:map( @@ -471,7 +471,7 @@ assert_valid_request_body(Body) -> ?assertMatch(#{<<"data">> := <<_/binary>>}, Msg), #{<<"data">> := Content64} = Msg, Content = base64:decode(Content64), - Decoded = emqx_json:decode(Content, [return_maps]), + Decoded = emqx_utils_json:decode(Content, [return_maps]), ct:pal("decoded payload: ~p", [Decoded]), ?assert(is_map(Decoded)), Decoded @@ -1014,7 +1014,7 @@ t_publish_timeout(Config) -> Rep = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{messageIds => [<<"6058891368195201">>]}), + emqx_utils_json:encode(#{messageIds => [<<"6058891368195201">>]}), Req ), {ok, Rep, State} @@ -1180,7 +1180,7 @@ t_failure_with_body(Config) -> Rep = cowboy_req:reply( 400, #{<<"content-type">> => <<"application/json">>}, - jiffy:encode(#{}), + emqx_utils_json:encode(#{}), Req ), {ok, Rep, State} diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl index 1b4b4aeb2..4891d5d9b 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl @@ -394,7 +394,7 @@ create_rule_and_action_http(Config, Overrides) -> Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -435,7 +435,7 @@ query_by_clientid(ClientId, Config) -> {"Content-Type", "application/json"} ], Body = - emqx_json:encode(#{ + emqx_utils_json:encode(#{ query => Query, dialect => #{ header => true, @@ -545,7 +545,7 @@ t_start_ok(Config) -> int_value => <<"-123">>, uint_value => <<"123">>, float_value => <<"24.5">>, - payload => emqx_json:encode(Payload) + payload => emqx_utils_json:encode(Payload) }, assert_persisted_data(ClientId, Expected, PersistedData), ok @@ -764,7 +764,7 @@ t_boolean_variants(Config) -> bool => atom_to_binary(Translation), int_value => <<"-123">>, uint_value => <<"123">>, - payload => emqx_json:encode(Payload) + payload => emqx_utils_json:encode(Payload) }, assert_persisted_data(ClientId, Expected, PersistedData), ok @@ -1024,9 +1024,9 @@ t_missing_field(Config) -> ClientId0 = emqx_guid:to_hexstr(emqx_guid:gen()), ClientId1 = emqx_guid:to_hexstr(emqx_guid:gen()), %% Message with the field that we "forgot" to select in the rule - Msg0 = emqx_message:make(ClientId0, <<"t/topic">>, emqx_json:encode(#{foo => 123})), + Msg0 = emqx_message:make(ClientId0, <<"t/topic">>, emqx_utils_json:encode(#{foo => 123})), %% Message without any fields - Msg1 = emqx_message:make(ClientId1, <<"t/topic">>, emqx_json:encode(#{})), + Msg1 = emqx_message:make(ClientId1, <<"t/topic">>, emqx_utils_json:encode(#{})), ?check_trace( begin emqx:publish(Msg0), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 116dcc729..2d4ee9222 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -258,7 +258,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl index 38e31c7ae..69b1e26ce 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl @@ -226,7 +226,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl index 83cb8b1f3..8f0b9ad9c 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl @@ -241,7 +241,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl index 95ec47e7f..0cb14e5c3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_rocketmq_SUITE.erl @@ -176,7 +176,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl index c956a93c6..8e058f2f1 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl @@ -219,7 +219,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index e754bd573..352c54629 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -6,7 +6,8 @@ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}, {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}}, {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}}, - {emqx, {path, "../../apps/emqx"}} + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 4703e0a21..ee56e0c5f 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -322,7 +322,7 @@ convert_to_item(Msg) when is_map(Msg), map_size(Msg) > 0 -> Msg ); convert_to_item(MsgBin) when is_binary(MsgBin) -> - Msg = emqx_json:decode(MsgBin), + Msg = emqx_utils_json:decode(MsgBin), convert_to_item(Msg); convert_to_item(Item) -> erlang:throw({invalid_item, Item}). @@ -334,7 +334,7 @@ convert2binary(Value) when is_binary(Value); is_number(Value) -> convert2binary(Value) when is_list(Value) -> unicode:characters_to_binary(Value); convert2binary(Value) when is_map(Value) -> - emqx_json:encode(Value). + emqx_utils_json:encode(Value). do_async_reply(Result, {ReplyFun, [Context]}) -> ReplyFun(Context, Result). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl index f07cbceab..a2045b8c5 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl @@ -334,14 +334,14 @@ ensure_jwt_worker(InstanceId, #{ encode_payload(_State = #{payload_template := PayloadTemplate}, Selected) -> Interpolated = case PayloadTemplate of - [] -> emqx_json:encode(Selected); + [] -> emqx_utils_json:encode(Selected); _ -> emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Selected) end, #{data => base64:encode(Interpolated)}. -spec to_pubsub_request([#{data := binary()}]) -> binary(). to_pubsub_request(Payloads) -> - emqx_json:encode(#{messages => Payloads}). + emqx_utils_json:encode(#{messages => Payloads}). -spec publish_path(state()) -> binary(). publish_path( diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl index aa03863b0..4e7adcd6e 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl @@ -84,4 +84,4 @@ render_message(PayloadTemplate, Message) -> %% Note: mongo expects a map as a document, so the rendered result %% must be JSON-serializable Rendered = emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Message), - emqx_json:decode(Rendered, [return_maps]). + emqx_utils_json:decode(Rendered, [return_maps]). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index 84f2e2a89..d296b0b4f 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -248,14 +248,14 @@ get_topic_key([Query | _], RawTopic, TopicTks) -> apply_template({Key, Msg} = _Req, Templates) -> case maps:get(Key, Templates, undefined) of undefined -> - emqx_json:encode(Msg); + emqx_utils_json:encode(Msg); Template -> emqx_plugin_libs_rule:proc_tmpl(Template, Msg) end; apply_template([{Key, _} | _] = Reqs, Templates) -> case maps:get(Key, Templates, undefined) of undefined -> - [emqx_json:encode(Msg) || {_, Msg} <- Reqs]; + [emqx_utils_json:encode(Msg) || {_, Msg} <- Reqs]; Template -> [emqx_plugin_libs_rule:proc_tmpl(Template, Msg) || {_, Msg} <- Reqs] end. diff --git a/lib-ee/emqx_ee_schema_registry/rebar.config b/lib-ee/emqx_ee_schema_registry/rebar.config index b19fb05ae..223ebf533 100644 --- a/lib-ee/emqx_ee_schema_registry/rebar.config +++ b/lib-ee/emqx_ee_schema_registry/rebar.config @@ -3,6 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../../apps/emqx_utils"}}, {erlavro, {git, "https://github.com/klarna/erlavro.git", {tag, "2.9.8"}}} ]}. diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl index 43145fb16..9835ec7c2 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl @@ -65,6 +65,6 @@ make_serde(avro, Name, Source0) -> inject_avro_name(Name, Source0) -> %% The schema checks that the source is a valid JSON when %% typechecking, so we shouldn't need to validate here. - Schema0 = emqx_json:decode(Source0, [return_maps]), + Schema0 = emqx_utils_json:decode(Source0, [return_maps]), Schema = Schema0#{<<"name">> => Name}, - emqx_json:encode(Schema). + emqx_utils_json:encode(Schema). diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 9b2f64c03..3c26d1966 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -95,7 +95,7 @@ create_rule_http(RuleParams) -> Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. @@ -107,7 +107,7 @@ schema_params(avro) -> #{name => <<"s">>, type => <<"string">>} ] }, - SourceBin = emqx_json:encode(Source), + SourceBin = emqx_utils_json:encode(Source), #{type => avro, source => SourceBin}. create_serde(SerdeType, SerdeName) -> @@ -162,7 +162,7 @@ receive_published(Line) -> maps:update_with( payload, fun(Raw) -> - case emqx_json:safe_decode(Raw, [return_maps]) of + case emqx_utils_json:safe_decode(Raw, [return_maps]) of {ok, Decoded} -> Decoded; {error, _} -> Raw end @@ -262,7 +262,7 @@ t_encode_decode(Config) -> {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode_decode1)}), on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, - PayloadBin = emqx_json:encode(Payload), + PayloadBin = emqx_utils_json:encode(Payload), emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), Res = receive_action_results(), ?assertMatch( @@ -311,7 +311,7 @@ t_encode(Config) -> {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode1)}), on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, - PayloadBin = emqx_json:encode(Payload), + PayloadBin = emqx_utils_json:encode(Payload), emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), Published = receive_published(?LINE), ?assertMatch( diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl index bbb6d5ef0..e7034d562 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl @@ -64,7 +64,7 @@ do_request(Method, PathParts, Body) -> {ok, Code, <<>>} -> {ok, Code, <<>>}; {ok, Code, Res1} -> - Res2 = emqx_json:decode(Res1, [return_maps]), + Res2 = emqx_utils_json:decode(Res1, [return_maps]), Res3 = try_decode_error_message(Res2), {ok, Code, Res3}; Error -> @@ -72,7 +72,7 @@ do_request(Method, PathParts, Body) -> end. try_decode_error_message(#{<<"message">> := Msg0} = Res0) -> - case emqx_json:safe_decode(Msg0, [return_maps]) of + case emqx_utils_json:safe_decode(Msg0, [return_maps]) of {ok, Msg} -> Res0#{<<"message">> := Msg}; {error, _} -> @@ -102,7 +102,7 @@ t_crud(_Config) -> #{name => <<"s">>, type => <<"string">>} ] }, - SourceBin = emqx_json:encode(Source), + SourceBin = emqx_utils_json:encode(Source), Params = #{ <<"type">> => <<"avro">>, <<"source">> => SourceBin, diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl index be62717d3..12798c6a2 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl @@ -59,7 +59,7 @@ schema_params(avro) -> #{name => <<"s">>, type => <<"string">>} ] }, - SourceBin = emqx_json:encode(Source), + SourceBin = emqx_utils_json:encode(Source), #{type => avro, source => SourceBin}. assert_roundtrip(SerdeName, Original) -> diff --git a/lib-ee/emqx_license/rebar.config b/lib-ee/emqx_license/rebar.config index 1cb5ace88..e7620387a 100644 --- a/lib-ee/emqx_license/rebar.config +++ b/lib-ee/emqx_license/rebar.config @@ -1,3 +1,6 @@ -{deps, [{emqx, {path, "../../apps/emqx"}}]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. {project_plugins, [erlfmt]}. diff --git a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl index 11ac1ca89..3de5ae121 100644 --- a/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl +++ b/lib-ee/emqx_license/test/emqx_license_http_api_SUITE.erl @@ -103,7 +103,7 @@ t_license_info(_Config) -> <<"start_at">> => <<"2022-01-11">>, <<"type">> => <<"trial">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), ok. @@ -128,7 +128,7 @@ t_license_upload_key_success(_Config) -> <<"start_at">> => <<"2022-01-11">>, <<"type">> => <<"trial">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), ?assertMatch( #{max_connections := 999}, @@ -150,7 +150,7 @@ t_license_upload_key_bad_key(_Config) -> <<"code">> => <<"BAD_REQUEST">>, <<"message">> => <<"Bad license key">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. @@ -168,7 +168,7 @@ t_license_upload_key_not_json(_Config) -> <<"code">> => <<"BAD_REQUEST">>, <<"message">> => <<"Invalid request params">> }, - emqx_json:decode(Payload, [return_maps]) + emqx_utils_json:decode(Payload, [return_maps]) ), assert_untouched_license(), ok. From 1880da0a2e69df8a798de27d73bea17931da92c1 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 14:38:49 +0200 Subject: [PATCH 113/279] refactor: move binary_util to emqx_utils_binary --- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl | 2 +- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl | 4 ++-- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl | 14 +++++++------- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl | 6 +++--- .../src/emqx_utils_binary.erl} | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) rename apps/{emqx_gateway_lwm2m/src/binary_util.erl => emqx_utils/src/emqx_utils_binary.erl} (99%) diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl index 80afadb8e..ca32d03db 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_api.erl @@ -227,7 +227,7 @@ to_operations(Obj, ObjDefinition) -> }. path_list(Path) -> - case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(Path, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId, ResInstId] -> [ObjId, ObjInsId, ResId, ResInstId]; [ObjId, ObjInsId, ResId] -> [ObjId, ObjInsId, ResId]; [ObjId, ObjInsId] -> [ObjId, ObjInsId]; diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl index 53995c97c..8e4286343 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_cmd.erl @@ -335,7 +335,7 @@ remove_tmp_fields(Ref) -> -spec path_list(Path :: binary()) -> {[PathWord :: binary()], [Query :: binary()]}. path_list(Path) -> - case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(Path, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId, LastPart] -> {ResInstId, QueryList} = query_list(LastPart), {[ObjId, ObjInsId, ResId, ResInstId], QueryList}; @@ -389,7 +389,7 @@ observe_seq(Options) -> add_alternate_path_prefix(<<"/">>, PathList) -> PathList; add_alternate_path_prefix(AlternatePath, PathList) -> - [binary_util:trim(AlternatePath, $/) | PathList]. + [emqx_utils_binary:trim(AlternatePath, $/) | PathList]. extract_path(Ref = #{}) -> drop_query( diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl index c3d23d4d9..c65fff441 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl @@ -97,7 +97,7 @@ tlv_single_resource(BaseName, Id, Value, ObjDefinition) -> [#{path => BaseName, value => Val}]. basename(OldBaseName, _ObjectId, ObjectInstanceId, ResourceId, 3) -> - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary, $/, ResId/binary>>; [ObjId, ObjInsId] -> @@ -113,13 +113,13 @@ basename(OldBaseName, _ObjectId, ObjectInstanceId, ResourceId, 3) -> >> end; basename(OldBaseName, _ObjectId, ObjectInstanceId, _ResourceId, 2) -> - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of [ObjId, ObjInsId, _ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; [ObjId, ObjInsId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; [ObjId] -> <<$/, ObjId/binary, $/, (integer_to_binary(ObjectInstanceId))/binary>> end. % basename(OldBaseName, _ObjectId, _ObjectInstanceId, _ResourceId, 1) -> -% case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of +% case binary:split(emqx_utils_binary:trim(OldBaseName, $/), [<<$/>>], [global]) of % [ObjId, _ObjInsId, _ResId] -> <<$/, ObjId/binary>>; % [ObjId, _ObjInsId] -> <<$/, ObjId/binary>>; % [ObjId] -> <<$/, ObjId/binary>> @@ -129,7 +129,7 @@ make_path(RelativePath, Id) -> <>. object_id(BaseName) -> - case binary:split(binary_util:trim(BaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(BaseName, $/), [<<$/>>], [global]) of [ObjId] -> binary_to_integer(ObjId); [ObjId, _] -> binary_to_integer(ObjId); [ObjId, _, _] -> binary_to_integer(ObjId); @@ -137,7 +137,7 @@ object_id(BaseName) -> end. object_resource_id(BaseName) -> - case binary:split(binary_util:trim(BaseName, $/), [<<$/>>], [global]) of + case binary:split(emqx_utils_binary:trim(BaseName, $/), [<<$/>>], [global]) of [_ObjIdBin1] -> error({invalid_basename, BaseName}); [_ObjIdBin2, _] -> @@ -371,8 +371,8 @@ translate_element(BaseName, [Element | ElementList], Acc) -> translate_element(BaseName, ElementList, NewAcc). full_path(BaseName, RelativePath) -> - Prefix = binary_util:rtrim(BaseName, $/), - Path = binary_util:ltrim(RelativePath, $/), + Prefix = emqx_utils_binary:rtrim(BaseName, $/), + Path = emqx_utils_binary:ltrim(RelativePath, $/), <>. get_element_value(#{<<"t">> := Value}) -> Value; diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl index c8bfa8974..e267692a6 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl @@ -386,11 +386,11 @@ is_alternate_path(LinkAttrs) -> LinkAttrs ). -trim(Str) -> binary_util:trim(Str, $\s). +trim(Str) -> emqx_utils_binary:trim(Str, $\s). delink(Str) -> - Ltrim = binary_util:ltrim(Str, $<), - binary_util:rtrim(Ltrim, $>). + Ltrim = emqx_utils_binary:ltrim(Str, $<), + emqx_utils_binary:rtrim(Ltrim, $>). get_lifetime(#{<<"lt">> := LT}) -> case LT of diff --git a/apps/emqx_gateway_lwm2m/src/binary_util.erl b/apps/emqx_utils/src/emqx_utils_binary.erl similarity index 99% rename from apps/emqx_gateway_lwm2m/src/binary_util.erl rename to apps/emqx_utils/src/emqx_utils_binary.erl index 68ac7a0d7..d9e504f46 100644 --- a/apps/emqx_gateway_lwm2m/src/binary_util.erl +++ b/apps/emqx_utils/src/emqx_utils_binary.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(binary_util). +-module(emqx_utils_binary). %% copied from https://github.com/arcusfelis/binary2 From 9c11bfce8041e8d110b870d0b8ebffbefc9ea5e0 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 14:45:18 +0200 Subject: [PATCH 114/279] refactor: rename emqx_misc to emqx_utils --- apps/emqx/src/emqx_api_lib.erl | 2 +- apps/emqx/src/emqx_authentication_config.erl | 4 +- apps/emqx/src/emqx_banned.erl | 6 +- apps/emqx/src/emqx_batch.erl | 2 +- apps/emqx/src/emqx_broker.erl | 2 +- apps/emqx/src/emqx_broker_helper.erl | 2 +- apps/emqx/src/emqx_channel.erl | 12 +- apps/emqx/src/emqx_cm.erl | 2 +- apps/emqx/src/emqx_connection.erl | 16 +- apps/emqx/src/emqx_crl_cache.erl | 8 +- apps/emqx/src/emqx_flapping.erl | 2 +- apps/emqx/src/emqx_guid.erl | 4 +- .../src/emqx_limiter/src/emqx_htb_limiter.erl | 2 +- apps/emqx/src/emqx_listeners.erl | 4 +- apps/emqx/src/emqx_ocsp_cache.erl | 4 +- apps/emqx/src/emqx_os_mon.erl | 6 +- apps/emqx/src/emqx_pool.erl | 2 +- apps/emqx/src/emqx_router.erl | 2 +- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx/src/emqx_session.erl | 2 +- apps/emqx/src/emqx_session_router.erl | 2 +- apps/emqx/src/emqx_stats.erl | 4 +- apps/emqx/src/emqx_sys.erl | 4 +- apps/emqx/src/emqx_sys_mon.erl | 4 +- apps/emqx/src/emqx_trace/emqx_trace.erl | 6 +- .../src/emqx_trace/emqx_trace_handler.erl | 2 +- apps/emqx/src/emqx_vm_mon.erl | 2 +- apps/emqx/src/emqx_ws_connection.erl | 14 +- apps/emqx/test/emqx_connection_SUITE.erl | 6 +- apps/emqx/test/emqx_misc_SUITE.erl | 209 ------------------ apps/emqx/test/emqx_router_SUITE.erl | 2 +- apps/emqx_authz/src/emqx_authz_file.erl | 2 +- .../test/emqx_authz_api_sources_SUITE.erl | 6 +- apps/emqx_bridge/src/emqx_bridge.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 8 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 10 +- .../test/emqx_bridge_api_SUITE.erl | 2 +- .../src/emqx_bridge_kafka_impl_consumer.erl | 4 +- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 8 +- apps/emqx_conf/src/emqx_cluster_rpc.erl | 10 +- .../src/emqx_cluster_rpc_cleaner.erl | 4 +- .../src/emqx_connector_http.erl | 6 +- .../src/emqx_connector_ldap.erl | 2 +- .../src/emqx_connector_mongo.erl | 2 +- .../src/emqx_connector_mqtt.erl | 2 +- .../src/emqx_connector_mysql.erl | 2 +- .../src/emqx_connector_pgsql.erl | 2 +- .../src/emqx_connector_redis.erl | 2 +- .../src/mqtt/emqx_connector_mqtt_msg.erl | 4 +- .../src/mqtt/emqx_connector_mqtt_worker.erl | 2 +- .../src/emqx_dashboard_swagger.erl | 2 +- apps/emqx_exhook/src/emqx_exhook_api.erl | 2 +- .../src/bhvrs/emqx_gateway_conn.erl | 18 +- .../src/emqx_gateway_api_clients.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_cm.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_http.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_utils.erl | 4 +- .../src/emqx_coap_channel.erl | 6 +- .../src/emqx_coap_session.erl | 2 +- apps/emqx_gateway_coap/src/emqx_coap_tm.erl | 6 +- .../src/emqx_coap_transport.erl | 2 +- .../src/emqx_exproto_channel.erl | 6 +- .../src/emqx_exproto_gcli.erl | 2 +- .../src/emqx_lwm2m_channel.erl | 8 +- .../src/emqx_mqttsn_channel.erl | 22 +- .../src/emqx_stomp_channel.erl | 12 +- apps/emqx_machine/src/emqx_global_gc.erl | 2 +- .../src/emqx_mgmt_api_clients.erl | 2 +- .../src/emqx_mgmt_api_subscriptions.erl | 2 +- .../src/emqx_mgmt_api_trace.erl | 2 +- apps/emqx_management/src/emqx_mgmt_auth.erl | 6 +- apps/emqx_management/src/emqx_mgmt_cli.erl | 18 +- apps/emqx_modules/src/emqx_delayed.erl | 8 +- apps/emqx_modules/src/emqx_telemetry.erl | 4 +- .../src/emqx_plugin_libs_pool.erl | 2 +- .../src/emqx_plugin_libs_rule.erl | 2 +- apps/emqx_prometheus/src/emqx_prometheus.erl | 2 +- .../src/emqx_resource_buffer_worker.erl | 2 +- .../src/emqx_resource_manager.erl | 4 +- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- .../src/emqx_retainer_dispatcher.erl | 2 +- .../src/emqx_rule_actions.erl | 2 +- .../src/emqx_rule_engine_api.erl | 8 +- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 6 +- .../src/emqx_rule_sqltester.erl | 2 +- apps/emqx_statsd/src/emqx_statsd.app.src | 2 +- apps/emqx_statsd/src/emqx_statsd.erl | 2 +- .../src/emqx_utils.erl} | 10 +- apps/emqx_utils/test/emqx_utils_SUITE.erl | 199 +++++++++++++++-- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 6 +- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 2 +- .../test/emqx_ee_bridge_mysql_SUITE.erl | 2 +- .../src/emqx_ee_connector_cassa.erl | 2 +- .../src/emqx_ee_connector_clickhouse.erl | 6 +- .../src/emqx_ee_connector_dynamo.erl | 2 +- .../src/emqx_ee_connector_gcp_pubsub.erl | 4 +- .../src/emqx_ee_connector_influxdb.erl | 6 +- .../src/emqx_ee_connector_rocketmq.erl | 2 +- .../src/emqx_ee_connector_tdengine.erl | 2 +- .../test/emqx_ee_schema_registry_SUITE.erl | 2 +- .../src/emqx_license_http_api.erl | 2 +- 101 files changed, 401 insertions(+), 447 deletions(-) delete mode 100644 apps/emqx/test/emqx_misc_SUITE.erl rename apps/{emqx/src/emqx_misc.erl => emqx_utils/src/emqx_utils.erl} (99%) diff --git a/apps/emqx/src/emqx_api_lib.erl b/apps/emqx/src/emqx_api_lib.erl index f1d65f350..5eddf6188 100644 --- a/apps/emqx/src/emqx_api_lib.erl +++ b/apps/emqx/src/emqx_api_lib.erl @@ -51,7 +51,7 @@ with_node_or_cluster(Node, Fun) -> -spec lookup_node(atom() | binary()) -> {ok, atom()} | not_found. lookup_node(BinNode) when is_binary(BinNode) -> - case emqx_misc:safe_to_existing_atom(BinNode, utf8) of + case emqx_utils:safe_to_existing_atom(BinNode, utf8) of {ok, Node} -> is_running_node(Node); _Error -> diff --git a/apps/emqx/src/emqx_authentication_config.erl b/apps/emqx/src/emqx_authentication_config.erl index 5471b66fc..be3b35f57 100644 --- a/apps/emqx/src/emqx_authentication_config.erl +++ b/apps/emqx/src/emqx_authentication_config.erl @@ -277,9 +277,9 @@ atom(Bin) -> binary_to_existing_atom(Bin, utf8). certs_dir(ChainName, ConfigOrID) -> DirName = dir(ChainName, ConfigOrID), SubDir = iolist_to_binary(filename:join(["authn", DirName])), - emqx_misc:safe_filename(SubDir). + emqx_utils:safe_filename(SubDir). dir(ChainName, ID) when is_binary(ID) -> - emqx_misc:safe_filename(iolist_to_binary([to_bin(ChainName), "-", ID])); + emqx_utils:safe_filename(iolist_to_binary([to_bin(ChainName), "-", ID])); dir(ChainName, Config) when is_map(Config) -> dir(ChainName, authenticator_id(Config)). diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index 758c570da..a0ccd93d7 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -243,7 +243,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{expiry_timer := TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -254,10 +254,10 @@ code_change(_OldVsn, State, _Extra) -> -ifdef(TEST). ensure_expiry_timer(State) -> - State#{expiry_timer := emqx_misc:start_timer(10, expire)}. + State#{expiry_timer := emqx_utils:start_timer(10, expire)}. -else. ensure_expiry_timer(State) -> - State#{expiry_timer := emqx_misc:start_timer(timer:minutes(1), expire)}. + State#{expiry_timer := emqx_utils:start_timer(timer:minutes(1), expire)}. -endif. expire_banned_items(Now) -> diff --git a/apps/emqx/src/emqx_batch.erl b/apps/emqx/src/emqx_batch.erl index 2fe09942c..22e812975 100644 --- a/apps/emqx/src/emqx_batch.erl +++ b/apps/emqx/src/emqx_batch.erl @@ -85,7 +85,7 @@ commit(Batch = #batch{batch_q = Q, commit_fun = Commit}) -> reset(Batch). reset(Batch = #batch{linger_timer = TRef}) -> - _ = emqx_misc:cancel_timer(TRef), + _ = emqx_utils:cancel_timer(TRef), Batch#batch{batch_q = [], linger_timer = undefined}. -spec size(batch()) -> non_neg_integer(). diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index d56620123..7e00c050e 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -92,7 +92,7 @@ start_link(Pool, Id) -> ok = create_tabs(), gen_server:start_link( - {local, emqx_misc:proc_name(?BROKER, Id)}, + {local, emqx_utils:proc_name(?BROKER, Id)}, ?MODULE, [Pool, Id], [] diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl index 91b4c4994..87b291361 100644 --- a/apps/emqx/src/emqx_broker_helper.erl +++ b/apps/emqx/src/emqx_broker_helper.erl @@ -131,7 +131,7 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #{pmon := PMon}) -> - SubPids = [SubPid | emqx_misc:drain_down(?BATCH_SIZE)], + SubPids = [SubPid | emqx_utils:drain_down(?BATCH_SIZE)], ok = emqx_pool:async_submit( fun lists:foreach/2, [fun clean_down/1, SubPids] ), diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 29a59e482..745429536 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -61,7 +61,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [ run_fold/3, pipeline/3, @@ -622,7 +622,7 @@ process_connect( NChannel = Channel#channel{session = Session}, handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, ensure_connected(NChannel)); {ok, #{session := Session, present := true, pendings := Pendings}} -> - Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), + Pendings1 = lists:usort(lists:append(Pendings, emqx_utils:drain_deliver())), NChannel = Channel#channel{ session = Session, resuming = true, @@ -1203,7 +1203,7 @@ handle_call( ) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) - Delivers = emqx_misc:drain_deliver(), + Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), disconnect_and_shutdown(takenover, AllPendings, Channel); handle_call(list_authz_cache, Channel) -> @@ -1402,7 +1402,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> @@ -2045,7 +2045,7 @@ clear_keepalive(Channel = #channel{timers = Timers}) -> undefined -> Channel; TRef -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), Channel#channel{timers = maps:without([alive_timer], Timers)} end. %%-------------------------------------------------------------------- @@ -2241,7 +2241,7 @@ get_mqtt_conf(Zone, Key, Default) -> %%-------------------------------------------------------------------- set_field(Name, Value, Channel) -> - Pos = emqx_misc:index_of(Name, record_info(fields, channel)), + Pos = emqx_utils:index_of(Name, record_info(fields, channel)), setelement(Pos + 1, Channel, Value). get_mqueue(#channel{session = Session}) -> diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index f8c510482..346e8dcb5 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -672,7 +672,7 @@ handle_cast(Msg, State) -> handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> ?tp(emqx_cm_process_down, #{stale_pid => Pid, reason => _Reason}), - ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], + ChanPids = [Pid | emqx_utils:drain_down(?BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), lists:foreach(fun mark_channel_disconnected/1, ChanPids), ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]), diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index e5002cab4..8d47f033c 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -77,7 +77,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [start_timer/2] ). @@ -260,7 +260,7 @@ stats(#state{ {error, _} -> [] end, ChanStats = emqx_channel:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ChanStats, ProcStats]). %% @doc Set TCP keepalive socket options to override system defaults. @@ -392,7 +392,7 @@ run_loop( emqx_channel:info(zone, Channel), [force_shutdown] ), - emqx_misc:tune_heap_size(ShutdownPolicy), + emqx_utils:tune_heap_size(ShutdownPolicy), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); @@ -472,7 +472,7 @@ ensure_stats_timer(_Timeout, State) -> -compile({inline, [cancel_stats_timer/1]}). cancel_stats_timer(State = #state{stats_timer = TRef}) when is_reference(TRef) -> ?tp(debug, cancel_stats_timer, #{}), - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), State#state{stats_timer = undefined}; cancel_stats_timer(State) -> State. @@ -558,7 +558,7 @@ handle_msg( {incoming, Packet = ?CONNECT_PACKET(ConnPkt)}, State = #state{idle_timer = IdleTimer} ) -> - ok = emqx_misc:cancel_timer(IdleTimer), + ok = emqx_utils:cancel_timer(IdleTimer), Serialize = emqx_frame:serialize_opts(ConnPkt), NState = State#state{ serialize = Serialize, @@ -593,7 +593,7 @@ handle_msg( #state{listener = {Type, Listener}} = State ) -> ActiveN = get_active_n(Type, Listener), - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent handle_msg({inet_reply, _Sock, ok}, State = #state{listener = {Type, Listener}}) -> @@ -1073,7 +1073,7 @@ check_oom(State = #state{channel = Channel}) -> emqx_channel:info(zone, Channel), [force_shutdown] ), ?tp(debug, check_oom, #{policy => ShutdownPolicy}), - case emqx_misc:check_oom(ShutdownPolicy) of + case emqx_utils:check_oom(ShutdownPolicy) of {shutdown, Reason} -> %% triggers terminate/2 callback immediately erlang:exit({shutdown, Reason}); @@ -1200,7 +1200,7 @@ inc_counter(Key, Inc) -> %%-------------------------------------------------------------------- set_field(Name, Value, State) -> - Pos = emqx_misc:index_of(Name, record_info(fields, state)), + Pos = emqx_utils:index_of(Name, record_info(fields, state)), setelement(Pos + 1, State, Value). get_state(Pid) -> diff --git a/apps/emqx/src/emqx_crl_cache.erl b/apps/emqx/src/emqx_crl_cache.erl index 79e47a6dc..084313420 100644 --- a/apps/emqx/src/emqx_crl_cache.erl +++ b/apps/emqx/src/emqx_crl_cache.erl @@ -117,7 +117,7 @@ handle_call(Call, _From, State) -> handle_cast({evict, URL}, State0 = #state{refresh_timers = RefreshTimers0}) -> emqx_ssl_crl_cache:delete(URL), MTimer = maps:get(URL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = maps:without([URL], RefreshTimers0), State = State0#state{refresh_timers = RefreshTimers}, ?tp( @@ -223,9 +223,9 @@ ensure_timer(URL, State = #state{refresh_interval = Timeout}) -> ensure_timer(URL, State = #state{refresh_timers = RefreshTimers0}, Timeout) -> ?tp(crl_cache_ensure_timer, #{url => URL, timeout => Timeout}), MTimer = maps:get(URL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = RefreshTimers0#{ - URL => emqx_misc:start_timer( + URL => emqx_utils:start_timer( Timeout, {refresh, URL} ) @@ -297,7 +297,7 @@ handle_cache_overflow(State0) -> {_Time, OldestURL, InsertionTimes} = gb_trees:take_smallest(InsertionTimes0), emqx_ssl_crl_cache:delete(OldestURL), MTimer = maps:get(OldestURL, RefreshTimers0, undefined), - emqx_misc:cancel_timer(MTimer), + emqx_utils:cancel_timer(MTimer), RefreshTimers = maps:remove(OldestURL, RefreshTimers0), CachedURLs = sets:del_element(OldestURL, CachedURLs0), ?tp(debug, crl_cache_overflow, #{oldest_url => OldestURL}), diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 64e4ed6c3..2cfed0a8e 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -184,7 +184,7 @@ code_change(_OldVsn, State, _Extra) -> start_timer(Zone) -> WindTime = maps:get(window_time, get_policy(Zone)), - emqx_misc:start_timer(WindTime, {garbage_collect, Zone}). + emqx_utils:start_timer(WindTime, {garbage_collect, Zone}). start_timers() -> lists:foreach( diff --git a/apps/emqx/src/emqx_guid.erl b/apps/emqx/src/emqx_guid.erl index fea4e70b0..d313723fb 100644 --- a/apps/emqx/src/emqx_guid.erl +++ b/apps/emqx/src/emqx_guid.erl @@ -145,10 +145,10 @@ npid() -> NPid. to_hexstr(I) when byte_size(I) =:= 16 -> - emqx_misc:bin_to_hexstr(I, upper). + emqx_utils:bin_to_hexstr(I, upper). from_hexstr(S) when byte_size(S) =:= 32 -> - emqx_misc:hexstr_to_bin(S). + emqx_utils:hexstr_to_bin(S). to_base62(<>) -> emqx_base62:encode(I). diff --git a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl index 83bc2ec72..bbebd9460 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl @@ -375,7 +375,7 @@ return_pause(infinity, PauseType, Fun, Diff, Limiter) -> {PauseType, ?MINIMUM_PAUSE, make_retry_context(Fun, Diff), Limiter}; return_pause(Rate, PauseType, Fun, Diff, Limiter) -> Val = erlang:round(Diff * emqx_limiter_schema:default_period() / Rate), - Pause = emqx_misc:clamp(Val, ?MINIMUM_PAUSE, ?MAXIMUM_PAUSE), + Pause = emqx_utils:clamp(Val, ?MINIMUM_PAUSE, ?MAXIMUM_PAUSE), {PauseType, Pause, make_retry_context(Fun, Diff), Limiter}. -spec make_retry_context(undefined | retry_fun(Limiter), non_neg_integer()) -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 4e5843166..96e28b1de 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -538,7 +538,7 @@ esockd_access_rules(StrRules) -> [A, CIDR] = string:tokens(S, " "), %% esockd rules only use words 'allow' and 'deny', both are existing %% comparison of strings may be better, but there is a loss of backward compatibility - case emqx_misc:safe_to_existing_atom(A) of + case emqx_utils:safe_to_existing_atom(A) of {ok, Action} -> [ { @@ -560,7 +560,7 @@ esockd_access_rules(StrRules) -> merge_default(Options) -> case lists:keytake(tcp_options, 1, Options) of {value, {tcp_options, TcpOpts}, Options1} -> - [{tcp_options, emqx_misc:merge_opts(?MQTT_SOCKOPTS, TcpOpts)} | Options1]; + [{tcp_options, emqx_utils:merge_opts(?MQTT_SOCKOPTS, TcpOpts)} | Options1]; false -> [{tcp_options, ?MQTT_SOCKOPTS} | Options] end. diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index 4e7ada044..7044d992e 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -476,9 +476,9 @@ ensure_timer(ListenerID, State, Timeout) -> ensure_timer(ListenerID, {refresh, ListenerID}, State, Timeout). ensure_timer(ListenerID, Message, State, Timeout) -> - emqx_misc:cancel_timer(maps:get(?REFRESH_TIMER(ListenerID), State, undefined)), + emqx_utils:cancel_timer(maps:get(?REFRESH_TIMER(ListenerID), State, undefined)), State#{ - ?REFRESH_TIMER(ListenerID) => emqx_misc:start_timer( + ?REFRESH_TIMER(ListenerID) => emqx_utils:start_timer( Timeout, Message ) diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index c5ce35bf9..4810798eb 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -180,8 +180,8 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- cancel_outdated_timer(#{mem_time_ref := MemRef, cpu_time_ref := CpuRef}) -> - emqx_misc:cancel_timer(MemRef), - emqx_misc:cancel_timer(CpuRef), + emqx_utils:cancel_timer(MemRef), + emqx_utils:cancel_timer(CpuRef), ok. start_cpu_check_timer() -> @@ -204,7 +204,7 @@ start_mem_check_timer() -> end. start_timer(Interval, Msg) -> - emqx_misc:start_timer(Interval, Msg). + emqx_utils:start_timer(Interval, Msg). update_mem_alarm_status(HWM) when HWM > 1.0 orelse HWM < 0.0 -> ?SLOG(warning, #{msg => "discarded_out_of_range_mem_alarm_threshold", value => HWM}), diff --git a/apps/emqx/src/emqx_pool.erl b/apps/emqx/src/emqx_pool.erl index 1691a533a..1cb5f429c 100644 --- a/apps/emqx/src/emqx_pool.erl +++ b/apps/emqx/src/emqx_pool.erl @@ -57,7 +57,7 @@ -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 7c9cc61b0..42430af5d 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -98,7 +98,7 @@ mnesia(boot) -> -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8c91ae782..96d9aea34 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2327,7 +2327,7 @@ mqtt_ssl_listener_ssl_options_validator(Conf) -> fun ocsp_outer_validator/1, fun crl_outer_validator/1 ], - case emqx_misc:pipeline(Checks, Conf, not_used) of + case emqx_utils:pipeline(Checks, Conf, not_used) of {ok, _, _} -> ok; {error, Reason, _NotUsed} -> diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index a13dfe491..8b15340e9 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -941,7 +941,7 @@ age(Now, Ts) -> Now - Ts. %%-------------------------------------------------------------------- set_field(Name, Value, Session) -> - Pos = emqx_misc:index_of(Name, record_info(fields, session)), + Pos = emqx_utils:index_of(Name, record_info(fields, session)), setelement(Pos + 1, Session, Value). get_mqueue(#session{mqueue = Q}) -> diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl index 0d4972e8c..8a21d4d03 100644 --- a/apps/emqx/src/emqx_session_router.erl +++ b/apps/emqx/src/emqx_session_router.erl @@ -104,7 +104,7 @@ create_init_tab() -> -spec start_link(atom(), pos_integer()) -> startlink_ret(). start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index ed901d9a9..017b83116 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -213,7 +213,7 @@ init(#{tick_ms := TickMs}) -> {ok, start_timer(#state{updates = [], tick_ms = TickMs}), hibernate}. start_timer(#state{tick_ms = Ms} = State) -> - State#state{timer = emqx_misc:start_timer(Ms, tick)}. + State#state{timer = emqx_utils:start_timer(Ms, tick)}. handle_call(stop, _From, State) -> {stop, normal, ok, State}; @@ -301,7 +301,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #state{timer = TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index af2b052aa..509429796 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -62,7 +62,7 @@ -endif. -import(emqx_topic, [systop/1]). --import(emqx_misc, [start_timer/2]). +-import(emqx_utils, [start_timer/2]). -record(state, { heartbeat :: maybe(reference()), @@ -222,7 +222,7 @@ handle_info(Info, State) -> terminate(_Reason, #state{heartbeat = TRef1, ticker = TRef2}) -> _ = emqx_config_handler:remove_handler(?CONF_KEY_PATH), unload_event_hooks(sys_event_messages()), - lists:foreach(fun emqx_misc:cancel_timer/1, [TRef1, TRef2]). + lists:foreach(fun emqx_utils:cancel_timer/1, [TRef1, TRef2]). unload_event_hooks([]) -> ok; diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index 6ff68820e..f1190f586 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -77,7 +77,7 @@ init([]) -> {ok, start_timer(#{timer => undefined, events => []})}. start_timer(State) -> - State#{timer := emqx_misc:start_timer(timer:seconds(2), reset)}. + State#{timer := emqx_utils:start_timer(timer:seconds(2), reset)}. sysm_opts(VM) -> sysm_opts(maps:to_list(VM), []). @@ -204,7 +204,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), ok. code_change(_OldVsn, State, _Extra) -> diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index f14dc0c15..91194772f 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -272,7 +272,7 @@ handle_info({timeout, TRef, update_trace}, #{timer := TRef} = State) -> ?tp(update_trace_done, #{}), {noreply, State#{timer => NextTRef}}; handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), handle_info({timeout, TRef, update_trace}, State); handle_info(Info, State) -> ?SLOG(error, #{unexpected_info => Info}), @@ -280,7 +280,7 @@ handle_info(Info, State) -> terminate(_Reason, #{timer := TRef}) -> _ = mnesia:unsubscribe({table, ?TRACE, simple}), - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), stop_all_trace_handler(), update_trace_handler(), _ = file:del_dir_r(zip_dir()), @@ -302,7 +302,7 @@ update_trace(Traces) -> ok = stop_trace(NeedStop, Started), clean_stale_trace_files(), NextTime = find_closest_time(Traces, Now), - emqx_misc:start_timer(NextTime, update_trace). + emqx_utils:start_timer(NextTime, update_trace). stop_all_trace_handler() -> lists:foreach( diff --git a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl index 231fd5e7b..528bc4d42 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace_handler.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace_handler.erl @@ -196,7 +196,7 @@ handler_id(Name, Type) -> do_handler_id(Name, Type) catch _:_ -> - Hash = emqx_misc:bin_to_hexstr(crypto:hash(md5, Name), lower), + Hash = emqx_utils:bin_to_hexstr(crypto:hash(md5, Name), lower), do_handler_id(Hash, Type) end. diff --git a/apps/emqx/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl index 1327a1bb0..d90d4139b 100644 --- a/apps/emqx/src/emqx_vm_mon.erl +++ b/apps/emqx/src/emqx_vm_mon.erl @@ -107,7 +107,7 @@ code_change(_OldVsn, State, _Extra) -> start_check_timer() -> Interval = emqx:get_config([sysmon, vm, process_check_interval]), - emqx_misc:start_timer(Interval, check). + emqx_utils:start_timer(Interval, check). usage(Percent) -> integer_to_list(floor(Percent * 100)) ++ "%". diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index ead609ed8..20962809f 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -52,7 +52,7 @@ -export([set_field/3]). -import( - emqx_misc, + emqx_utils, [ maybe_apply/2, start_timer/2 @@ -172,7 +172,7 @@ stats(WsPid) when is_pid(WsPid) -> stats(#state{channel = Channel}) -> SockStats = emqx_pd:get_counters(?SOCK_STATS), ChanStats = emqx_channel:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ChanStats, ProcStats]). %% kick|discard|takeover @@ -340,7 +340,7 @@ tune_heap_size(Channel) -> ) of #{enable := false} -> ok; - ShutdownPolicy -> emqx_misc:tune_heap_size(ShutdownPolicy) + ShutdownPolicy -> emqx_utils:tune_heap_size(ShutdownPolicy) end. get_stats_enable(Zone) -> @@ -454,7 +454,7 @@ websocket_info( State = #state{listener = {Type, Listener}} ) -> ActiveN = get_active_n(Type, Listener), - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); websocket_info( {timeout, _, limit_timeout}, @@ -678,7 +678,7 @@ check_oom(State = #state{channel = Channel}) -> #{enable := false} -> State; #{enable := true} -> - case emqx_misc:check_oom(ShutdownPolicy) of + case emqx_utils:check_oom(ShutdownPolicy) of Shutdown = {shutdown, _Reason} -> postpone(Shutdown, State); _Other -> @@ -913,7 +913,7 @@ inc_qos_stats_key(_, _) -> undefined. %% Cancel idle timer cancel_idle_timer(State = #state{idle_timer = IdleTimer}) -> - ok = emqx_misc:cancel_timer(IdleTimer), + ok = emqx_utils:cancel_timer(IdleTimer), State#state{idle_timer = undefined}. %%-------------------------------------------------------------------- @@ -1046,7 +1046,7 @@ check_max_connection(Type, Listener) -> %%-------------------------------------------------------------------- set_field(Name, Value, State) -> - Pos = emqx_misc:index_of(Name, record_info(fields, state)), + Pos = emqx_utils:index_of(Name, record_info(fields, state)), setelement(Pos + 1, State, Value). %% ensure lowercase letters in headers diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index cc9e03168..21ed45119 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -496,16 +496,16 @@ t_get_conn_info(_) -> t_oom_shutdown(init, Config) -> ok = snabbkaffe:start_trace(), - ok = meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]), + ok = meck:new(emqx_utils, [non_strict, passthrough, no_history, no_link]), meck:expect( - emqx_misc, + emqx_utils, check_oom, fun(_) -> {shutdown, "fake_oom"} end ), Config; t_oom_shutdown('end', _Config) -> snabbkaffe:stop(), - meck:unload(emqx_misc), + meck:unload(emqx_utils), ok. t_oom_shutdown(_) -> diff --git a/apps/emqx/test/emqx_misc_SUITE.erl b/apps/emqx/test/emqx_misc_SUITE.erl deleted file mode 100644 index c068a087e..000000000 --- a/apps/emqx/test/emqx_misc_SUITE.erl +++ /dev/null @@ -1,209 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_misc_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). - --define(SOCKOPTS, [ - binary, - {packet, raw}, - {reuseaddr, true}, - {backlog, 512}, - {nodelay, true} -]). - -all() -> emqx_common_test_helpers:all(?MODULE). - -t_merge_opts(_) -> - Opts = emqx_misc:merge_opts(?SOCKOPTS, [ - raw, - binary, - {backlog, 1024}, - {nodelay, false}, - {max_clients, 1024}, - {acceptors, 16} - ]), - ?assertEqual(1024, proplists:get_value(backlog, Opts)), - ?assertEqual(1024, proplists:get_value(max_clients, Opts)), - ?assertEqual( - [ - binary, - raw, - {acceptors, 16}, - {backlog, 1024}, - {max_clients, 1024}, - {nodelay, false}, - {packet, raw}, - {reuseaddr, true} - ], - lists:sort(Opts) - ). - -t_maybe_apply(_) -> - ?assertEqual(undefined, emqx_misc:maybe_apply(fun(A) -> A end, undefined)), - ?assertEqual(a, emqx_misc:maybe_apply(fun(A) -> A end, a)). - -t_run_fold(_) -> - ?assertEqual(1, emqx_misc:run_fold([], 1, state)), - Add = fun(I, St) -> I + St end, - Mul = fun(I, St) -> I * St end, - ?assertEqual(6, emqx_misc:run_fold([Add, Mul], 1, 2)). - -t_pipeline(_) -> - ?assertEqual({ok, input, state}, emqx_misc:pipeline([], input, state)), - Funs = [ - fun(_I, _St) -> ok end, - fun(_I, St) -> {ok, St + 1} end, - fun(I, St) -> {ok, I + 1, St + 1} end, - fun(I, St) -> {ok, I * 2, St * 2} end - ], - ?assertEqual({ok, 4, 6}, emqx_misc:pipeline(Funs, 1, 1)), - ?assertEqual( - {error, undefined, 1}, emqx_misc:pipeline([fun(_I) -> {error, undefined} end], 1, 1) - ), - ?assertEqual( - {error, undefined, 2}, emqx_misc:pipeline([fun(_I, _St) -> {error, undefined, 2} end], 1, 1) - ). - -t_start_timer(_) -> - TRef = emqx_misc:start_timer(1, tmsg), - timer:sleep(2), - ?assertEqual([{timeout, TRef, tmsg}], drain()), - ok = emqx_misc:cancel_timer(TRef). - -t_cancel_timer(_) -> - Timer = emqx_misc:start_timer(0, foo), - ok = emqx_misc:cancel_timer(Timer), - ?assertEqual([], drain()), - ok = emqx_misc:cancel_timer(undefined). - -t_proc_name(_) -> - ?assertEqual(emqx_pool_1, emqx_misc:proc_name(emqx_pool, 1)). - -t_proc_stats(_) -> - Pid1 = spawn(fun() -> exit(normal) end), - timer:sleep(10), - ?assertEqual([], emqx_misc:proc_stats(Pid1)), - Pid2 = spawn(fun() -> - ?assertMatch([{mailbox_len, 0} | _], emqx_misc:proc_stats()), - timer:sleep(200) - end), - timer:sleep(10), - Pid2 ! msg, - timer:sleep(10), - ?assertMatch([{mailbox_len, 1} | _], emqx_misc:proc_stats(Pid2)). - -t_drain_deliver(_) -> - self() ! {deliver, t1, m1}, - self() ! {deliver, t2, m2}, - ?assertEqual( - [ - {deliver, t1, m1}, - {deliver, t2, m2} - ], - emqx_misc:drain_deliver(2) - ). - -t_drain_down(_) -> - {Pid1, _Ref1} = erlang:spawn_monitor(fun() -> ok end), - {Pid2, _Ref2} = erlang:spawn_monitor(fun() -> ok end), - timer:sleep(100), - ?assertEqual([Pid1, Pid2], lists:sort(emqx_misc:drain_down(2))), - ?assertEqual([], emqx_misc:drain_down(1)). - -t_index_of(_) -> - try emqx_misc:index_of(a, []) of - _ -> ct:fail(should_throw_error) - catch - error:Reason -> - ?assertEqual(badarg, Reason) - end, - ?assertEqual(3, emqx_misc:index_of(a, [b, c, a, e, f])). - -t_check(_) -> - Policy = #{ - max_message_queue_len => 10, - max_heap_size => 1024 * 1024 * 8, - enable => true - }, - [self() ! {msg, I} || I <- lists:seq(1, 5)], - ?assertEqual(ok, emqx_misc:check_oom(Policy)), - [self() ! {msg, I} || I <- lists:seq(1, 6)], - ?assertEqual( - {shutdown, #{reason => message_queue_too_long, value => 11, max => 10}}, - emqx_misc:check_oom(Policy) - ). - -drain() -> - drain([]). - -drain(Acc) -> - receive - Msg -> drain([Msg | Acc]) - after 0 -> - lists:reverse(Acc) - end. - -t_rand_seed(_) -> - ?assert(is_tuple(emqx_misc:rand_seed())). - -t_now_to_secs(_) -> - ?assert(is_integer(emqx_misc:now_to_secs(os:timestamp()))). - -t_now_to_ms(_) -> - ?assert(is_integer(emqx_misc:now_to_ms(os:timestamp()))). - -t_gen_id(_) -> - ?assertEqual(10, length(emqx_misc:gen_id(10))), - ?assertEqual(20, length(emqx_misc:gen_id(20))). - -t_pmap_normal(_) -> - ?assertEqual( - [5, 7, 9], - emqx_misc:pmap( - fun({A, B}) -> A + B end, - [{2, 3}, {3, 4}, {4, 5}] - ) - ). - -t_pmap_timeout(_) -> - ?assertExit( - timeout, - emqx_misc:pmap( - fun - (timeout) -> ct:sleep(1000); - ({A, B}) -> A + B - end, - [{2, 3}, {3, 4}, timeout], - 100 - ) - ). - -t_pmap_exception(_) -> - ?assertError( - foobar, - emqx_misc:pmap( - fun - (error) -> error(foobar); - ({A, B}) -> A + B - end, - [{2, 3}, {3, 4}, error] - ) - ). diff --git a/apps/emqx/test/emqx_router_SUITE.erl b/apps/emqx/test/emqx_router_SUITE.erl index 298a33fe8..2db0acf82 100644 --- a/apps/emqx/test/emqx_router_SUITE.erl +++ b/apps/emqx/test/emqx_router_SUITE.erl @@ -119,7 +119,7 @@ t_has_routes(_) -> ?R:delete_route(<<"devices/+/messages">>). t_unexpected(_) -> - Router = emqx_misc:proc_name(?R, 1), + Router = emqx_utils:proc_name(?R, 1), ?assertEqual(ignored, gen_server:call(Router, bad_request)), ?assertEqual(ok, gen_server:cast(Router, bad_message)), Router ! bad_info. diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index 9aa2d506f..ede4a9582 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -47,7 +47,7 @@ create(#{path := Path} = Source) -> ?SLOG(alert, #{ msg => failed_to_read_acl_file, path => Path, - explain => emqx_misc:explain_posix(Reason) + explain => emqx_utils:explain_posix(Reason) }), throw(failed_to_read_acl_file); {error, Reason} -> diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index b717775a6..7a7dbb7e9 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -148,8 +148,8 @@ set_special_configs(_App) -> ok. init_per_testcase(t_api, Config) -> - meck:new(emqx_misc, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_misc, gen_id, fun() -> "fake" end), + meck:new(emqx_utils, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_utils, gen_id, fun() -> "fake" end), meck:new(emqx, [non_strict, passthrough, no_history, no_link]), meck:expect( @@ -165,7 +165,7 @@ init_per_testcase(_, Config) -> Config. end_per_testcase(t_api, _Config) -> - meck:unload(emqx_misc), + meck:unload(emqx_utils), meck:unload(emqx), ok; end_per_testcase(_, _Config) -> diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 2a20c5994..087bc6a3f 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -296,7 +296,7 @@ create(BridgeType, BridgeName, RawConf) -> brige_action => create, bridge_type => BridgeType, bridge_name => BridgeName, - bridge_raw_config => emqx_misc:redact(RawConf) + bridge_raw_config => emqx_utils:redact(RawConf) }), emqx_conf:update( emqx_bridge:config_key_path() ++ [BridgeType, BridgeName], diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 675541311..814fcdeb7 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -668,7 +668,7 @@ get_metrics_from_local_node(BridgeType, BridgeName) -> false -> ?BRIDGE_NOT_ENABLED; true -> - case emqx_misc:safe_to_existing_atom(Node, utf8) of + case emqx_utils:safe_to_existing_atom(Node, utf8) of {ok, TargetNode} -> call_operation(TargetNode, OperFunc, [ TargetNode, BridgeType, BridgeName @@ -835,7 +835,7 @@ format_resource_data(ResData) -> format_resource_data(error, undefined, Result) -> Result; format_resource_data(error, Error, Result) -> - Result#{status_reason => emqx_misc:readable_error_msg(Error)}; + Result#{status_reason => emqx_utils:readable_error_msg(Error)}; format_resource_data(K, V, Result) -> Result#{K => V}. @@ -1004,7 +1004,7 @@ supported_versions(get_metrics_from_all_nodes) -> [4]; supported_versions(_Call) -> [1, 2, 3, 4]. redact(Term) -> - emqx_misc:redact(Term). + emqx_utils:redact(Term). deobfuscate(NewConf, OldConf) -> maps:fold( @@ -1015,7 +1015,7 @@ deobfuscate(NewConf, OldConf) -> {ok, OldV} when is_map(V), is_map(OldV) -> Acc#{K => deobfuscate(V, OldV)}; {ok, OldV} -> - case emqx_misc:is_redacted(K, V) of + case emqx_utils:is_redacted(K, V) of true -> Acc#{K => OldV}; _ -> diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index b43cbe0ec..903e86443 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -157,7 +157,7 @@ create(Type, Name, Conf, Opts0) -> msg => "create bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), Opts = override_start_after_created(Conf, Opts0), {ok, _Data} = emqx_resource:create_local( @@ -192,7 +192,7 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> msg => "update bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), case recreate(Type, Name, Conf, Opts) of {ok, _} -> @@ -202,7 +202,7 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> msg => "updating_a_non_existing_bridge", type => Type, name => Name, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), create(Type, Name, Conf, Opts); {error, Reason} -> @@ -236,8 +236,8 @@ recreate(Type, Name, Conf, Opts) -> ). create_dry_run(Type, Conf0) -> - TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_misc:gen_id(8)]), - TmpPath = emqx_misc:safe_filename(TmpPath0), + TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), + TmpPath = emqx_utils:safe_filename(TmpPath0), Conf = emqx_map_lib:safe_atom_key_map(Conf0), case emqx_connector_ssl:convert_certs(TmpPath, Conf) of {error, Reason} -> diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 3ec6061d8..ef997f7e3 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -975,7 +975,7 @@ t_with_redact_update(Config) -> ), %% update with redacted config - BridgeConf = emqx_misc:redact(Template), + BridgeConf = emqx_utils:redact(Template), BridgeID = emqx_bridge_resource:bridge_id(Type, Name), {ok, 200, _} = request(put, uri(["bridges", BridgeID]), BridgeConf, Config), ?assertEqual( diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index 2dc43a130..fdfa3300c 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -156,7 +156,7 @@ on_start(InstanceId, Config) -> msg => "failed_to_start_kafka_consumer_client", instance_id => InstanceId, kafka_hosts => BootstrapHosts, - reason => emqx_misc:redact(Reason) + reason => emqx_utils:redact(Reason) }), throw(?CLIENT_DOWN_MESSAGE) end, @@ -344,7 +344,7 @@ start_consumer(Config, InstanceId, ClientID) -> msg => "failed_to_start_kafka_consumer", instance_id => InstanceId, kafka_hosts => emqx_bridge_kafka_impl:hosts(BootstrapHosts0), - reason => emqx_misc:redact(Reason2) + reason => emqx_utils:redact(Reason2) }), stop_client(ClientID), throw(failed_to_start_kafka_consumer) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index fc9a6dc29..254284b75 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -299,7 +299,7 @@ init_per_testcase(TestCase, Config) when common_init_per_testcase(TestCase, Config) end; init_per_testcase(t_cluster_group = TestCase, Config0) -> - Config = emqx_misc:merge_opts(Config0, [{num_partitions, 6}]), + Config = emqx_utils:merge_opts(Config0, [{num_partitions, 6}]), common_init_per_testcase(TestCase, Config); init_per_testcase(t_multiple_topic_mappings = TestCase, Config0) -> KafkaTopicBase = @@ -1543,7 +1543,7 @@ do_t_receive_after_recovery(Config) -> %% 2) publish messages while the consumer is down. %% we use `pmap' to avoid wolff sending the whole %% batch to a single partition. - emqx_misc:pmap(fun(Msg) -> publish(Config, [Msg]) end, Messages1), + emqx_utils:pmap(fun(Msg) -> publish(Config, [Msg]) end, Messages1), ok end), %% 3) restore and consume messages @@ -1667,7 +1667,7 @@ t_cluster_group(Config) -> || {Name, Opts} <- Cluster ], on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) @@ -1889,7 +1889,7 @@ t_cluster_node_down(Config) -> Cluster ), on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 0382045d4..c82191bc3 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -501,15 +501,17 @@ log_and_alarm(IsSuccess, Res, #{kind := ?APPLY_KIND_INITIATE} = Meta) -> %% because nothing is committed case IsSuccess of true -> - ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_result", result => emqx_misc:redact(Res)}); + ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_result", result => emqx_utils:redact(Res)}); false -> - ?SLOG(warning, Meta#{msg => "cluster_rpc_apply_result", result => emqx_misc:redact(Res)}) + ?SLOG(warning, Meta#{ + msg => "cluster_rpc_apply_result", result => emqx_utils:redact(Res) + }) end; log_and_alarm(true, Res, Meta) -> - ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_ok", result => emqx_misc:redact(Res)}), + ?SLOG(debug, Meta#{msg => "cluster_rpc_apply_ok", result => emqx_utils:redact(Res)}), do_alarm(deactivate, Res, Meta); log_and_alarm(false, Res, Meta) -> - ?SLOG(error, Meta#{msg => "cluster_rpc_apply_failed", result => emqx_misc:redact(Res)}), + ?SLOG(error, Meta#{msg => "cluster_rpc_apply_failed", result => emqx_utils:redact(Res)}), do_alarm(activate, Res, Meta). do_alarm(Fun, Res, #{tnx_id := Id} = Meta) -> diff --git a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl index bce866c2d..fe72cd65b 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc_cleaner.erl @@ -73,7 +73,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, #{timer := TRef}) -> - emqx_misc:cancel_timer(TRef). + emqx_utils:cancel_timer(TRef). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -82,7 +82,7 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- ensure_timer(State = #{cleanup_ms := Ms}) -> - State#{timer := emqx_misc:start_timer(Ms, del_stale_mfa)}. + State#{timer := emqx_utils:start_timer(Ms, del_stale_mfa)}. %% @doc Keep the latest completed 100 records for querying and troubleshooting. del_stale_mfa(MaxHistory) -> diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index dfa6dab81..ef2e11eb7 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -219,7 +219,7 @@ on_start( SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, Config)), {tls, SSLOpts} end, - NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + NTransportOpts = emqx_utils:ipv6_probe(TransportOpts), PoolOpts = [ {host, Host}, {port, Port}, @@ -425,7 +425,7 @@ do_get_status(PoolName, Timeout) -> Error end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of % we crash in case of non-empty lists since we don't know what to do in that case [_ | _] = Results -> case [E || {error, _} = E <- Results] of @@ -603,7 +603,7 @@ is_sensitive_key(_) -> %% Function that will do a deep traversal of Data and remove sensitive %% information (i.e., passwords) redact(Data) -> - emqx_misc:redact(Data, fun is_sensitive_key/1). + emqx_utils:redact(Data, fun is_sensitive_key/1). %% because the body may contain some sensitive data %% and at the same time the redact function will not scan the binary data diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 82d622e09..ac2af301e 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -65,7 +65,7 @@ on_start( ?SLOG(info, #{ msg => "starting_ldap_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), Servers = emqx_schema:parse_servers(Servers0, ?LDAP_HOST_OPTIONS), SslOpts = diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index ead3e2e49..a5873bcf6 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -162,7 +162,7 @@ on_start( rs -> "starting_mongodb_replica_set_connector"; sharded -> "starting_mongodb_sharded_connector" end, - ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_misc:redact(Config)}), + ?SLOG(info, #{msg => Msg, connector => InstId, config => emqx_utils:redact(Config)}), NConfig = #{hosts := Hosts} = maybe_resolve_srv_and_txt_records(Config), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl index 8f2d06517..5b488825b 100644 --- a/apps/emqx_connector/src/emqx_connector_mqtt.erl +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -149,7 +149,7 @@ on_start(InstanceId, Conf) -> ?SLOG(info, #{ msg => "starting_mqtt_connector", connector => InstanceId, - config => emqx_misc:redact(Conf) + config => emqx_utils:redact(Conf) }), BasicConf = basic_config(Conf), BridgeConf = BasicConf#{ diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index c69cf5ca0..6600a5f77 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -102,7 +102,7 @@ on_start( ?SLOG(info, #{ msg => "starting_mysql_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 14cbbc80f..8796e00a5 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -95,7 +95,7 @@ on_start( ?SLOG(info, #{ msg => "starting_postgresql_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), SslOpts = case maps:get(enable, SSL) of diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index c70e766af..4ef778e6b 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -123,7 +123,7 @@ on_start( ?SLOG(info, #{ msg => "starting_redis_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), ConfKey = case Type of diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl index 67fc40efa..df1114483 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -85,7 +85,7 @@ to_remote_msg(MapMsg, #{ qos = QoS, retain = Retain, topic = topic(Mountpoint, Topic), - props = emqx_misc:pub_props_to_packet(PubProps), + props = emqx_utils:pub_props_to_packet(PubProps), payload = Payload }; to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> @@ -112,7 +112,7 @@ to_broker_msg( Retain = replace_simple_var(RetainToken, MapMsg), PubProps = maps:get(pub_props, MapMsg, #{}), set_headers( - Props#{properties => emqx_misc:pub_props_to_packet(PubProps)}, + Props#{properties => emqx_utils:pub_props_to_packet(PubProps)}, emqx_message:set_flags( #{dup => Dup, retain => Retain}, emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload) diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 8e41a9f0f..5197a35df 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -124,7 +124,7 @@ start_link(Name, BridgeOpts) -> {error, Reason} = Error -> ?SLOG(error, #{ msg => "client_start_failed", - config => emqx_misc:redact(BridgeOpts), + config => emqx_utils:redact(BridgeOpts), reason => Reason }), Error diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index eb7f6c741..22cf484ff 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -915,4 +915,4 @@ schema_converter(Options) -> maps:get(schema_converter, Options, fun hocon_schema_to_spec/2). hocon_error_msg(Reason) -> - emqx_misc:readable_error_msg(Reason). + emqx_utils:readable_error_msg(Reason). diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl index aa5d1897f..9bfae9579 100644 --- a/apps/emqx_exhook/src/emqx_exhook_api.erl +++ b/apps/emqx_exhook/src/emqx_exhook_api.erl @@ -478,7 +478,7 @@ call_cluster(Fun) -> %%-------------------------------------------------------------------- %% Internal Funcs %%-------------------------------------------------------------------- -err_msg(Msg) -> emqx_misc:readable_error_msg(Msg). +err_msg(Msg) -> emqx_utils:readable_error_msg(Msg). get_raw_config() -> RawConfig = emqx:get_raw_config([exhook, servers], []), diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 7f37061ac..4145a92a7 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -173,7 +173,7 @@ stats(#state{ end, ConnStats = emqx_pd:get_counters(?CONN_STATS), ChanStats = ChannMod:stats(Channel), - ProcStats = emqx_misc:proc_stats(), + ProcStats = emqx_utils:proc_stats(), lists:append([SockStats, ConnStats, ChanStats, ProcStats]). call(Pid, Req) -> @@ -297,7 +297,7 @@ init_state(WrappedSock, Peername, Options, FrameMod, ChannMod) -> StatsTimer = emqx_gateway_utils:stats_timer(Options), IdleTimeout = emqx_gateway_utils:idle_timeout(Options), OomPolicy = emqx_gateway_utils:oom_policy(Options), - IdleTimer = emqx_misc:start_timer(IdleTimeout, idle_timeout), + IdleTimer = emqx_utils:start_timer(IdleTimeout, idle_timeout), #state{ socket = WrappedSock, peername = Peername, @@ -327,7 +327,7 @@ run_loop( } ) -> emqx_logger:set_metadata_peername(esockd:format(Peername)), - _ = emqx_misc:tune_heap_size(OomPolicy), + _ = emqx_utils:tune_heap_size(OomPolicy), case activate_socket(State) of {ok, NState} -> hibernate(Parent, NState); @@ -383,14 +383,14 @@ wakeup_from_hib(Parent, State) -> %% Ensure/cancel stats timer ensure_stats_timer(Timeout, State = #state{stats_timer = undefined}) -> - State#state{stats_timer = emqx_misc:start_timer(Timeout, emit_stats)}; + State#state{stats_timer = emqx_utils:start_timer(Timeout, emit_stats)}; ensure_stats_timer(_Timeout, State) -> State. cancel_stats_timer(State = #state{stats_timer = TRef}) when is_reference(TRef) -> - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), State#state{stats_timer = undefined}; cancel_stats_timer(State) -> State. @@ -471,7 +471,7 @@ handle_msg( State = #state{idle_timer = IdleTimer} ) -> IdleTimer /= undefined andalso - emqx_misc:cancel_timer(IdleTimer), + emqx_utils:cancel_timer(IdleTimer), NState = State#state{idle_timer = undefined}, handle_incoming(Packet, NState); handle_msg({outgoing, Data}, State) -> @@ -501,7 +501,7 @@ handle_msg( Deliver = {deliver, _Topic, _Msg}, State = #state{active_n = ActiveN} ) -> - Delivers = [Deliver | emqx_misc:drain_deliver(ActiveN)], + Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); %% Something sent %% TODO: Who will deliver this message? @@ -904,7 +904,7 @@ handle_info(Info, State) -> %% msg => "reach_rate_limit", %% pause => Time %% }), -%% TRef = emqx_misc:start_timer(Time, limit_timeout), +%% TRef = emqx_utils:start_timer(Time, limit_timeout), %% State#state{ %% sockstate = blocked, %% limiter = Limiter1, @@ -928,7 +928,7 @@ run_gc(Stats, State = #state{gc_state = GcSt}) -> end. check_oom(State = #state{oom_policy = OomPolicy}) -> - case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of + case ?ENABLED(OomPolicy) andalso emqx_utils:check_oom(OomPolicy) of {shutdown, Reason} -> %% triggers terminate/2 callback immediately erlang:exit({shutdown, Reason}); diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index e64e918b4..cd387e3bb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -115,7 +115,7 @@ clients(get, #{ fun ?MODULE:format_channel_info/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> QStringWithoutNode = maps:without([<<"node">>], QString), emqx_mgmt_api:node_query( diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 71ec4bf59..5de6da63f 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -802,7 +802,7 @@ handle_info( {'DOWN', _MRef, process, Pid, _Reason}, State = #state{gwname = GwName, chan_pmon = PMon} ) -> - ChanPids = [Pid | emqx_misc:drain_down(?DEFAULT_BATCH_SIZE)], + ChanPids = [Pid | emqx_utils:drain_down(?DEFAULT_BATCH_SIZE)], {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), CmTabs = cmtabs(GwName), diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 5982334b4..9bb5821f5 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -495,7 +495,7 @@ reason2msg( reason2msg( {#{roots := [{gateway, _}]}, [_ | _]} = Error ) -> - Bin = emqx_misc:readable_error_msg(Error), + Bin = emqx_utils:readable_error_msg(Error), <<"Invalid configurations: ", Bin/binary>>; reason2msg(_) -> error. diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 9d80de00e..07185ef42 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -223,7 +223,7 @@ merge_default(Udp, Options) -> case lists:keytake(Key, 1, Options) of {value, {Key, TcpOpts}, Options1} -> [ - {Key, emqx_misc:merge_opts(Default, TcpOpts)} + {Key, emqx_utils:merge_opts(Default, TcpOpts)} | Options1 ]; false -> @@ -482,7 +482,7 @@ frame_options(Options) -> -spec init_gc_state(map()) -> emqx_gc:gc_state() | undefined. init_gc_state(Options) -> - emqx_misc:maybe_apply(fun emqx_gc:init/1, force_gc_policy(Options)). + emqx_utils:maybe_apply(fun emqx_gc:init/1, force_gc_policy(Options)). -spec force_gc_policy(map()) -> emqx_gc:opts() | undefined. force_gc_policy(Options) -> diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index 4cf362d9d..b90fd630d 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -111,7 +111,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_coap_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_coap_session:info/1, Session); info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> ClientId; info(ctx, #channel{ctx = Ctx}) -> @@ -366,7 +366,7 @@ ensure_timer(Name, Time, Msg, #channel{timers = Timers} = Channel) -> end. make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. ensure_keepalive_timer(Channel) -> @@ -710,7 +710,7 @@ process_connection( ) -> Queries = emqx_coap_message:get_option(uri_query, Req), case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun run_conn_hooks/2, diff --git a/apps/emqx_gateway_coap/src/emqx_coap_session.erl b/apps/emqx_gateway_coap/src/emqx_coap_session.erl index 688defcbb..5ae169675 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_session.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_session.erl @@ -81,7 +81,7 @@ %%%------------------------------------------------------------------- -spec new() -> session(). new() -> - _ = emqx_misc:rand_seed(), + _ = emqx_utils:rand_seed(), #session{ transport_manager = emqx_coap_tm:new(), observe_manager = emqx_coap_observe_res:new_manager(), diff --git a/apps/emqx_gateway_coap/src/emqx_coap_tm.erl b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl index 82f616b25..68a7ae237 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_tm.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_tm.erl @@ -272,12 +272,12 @@ cancel_state_timer(#state_machine{timers = Timers} = Machine) -> undefined -> Machine; Ref -> - _ = emqx_misc:cancel_timer(Ref), + _ = emqx_utils:cancel_timer(Ref), Machine#state_machine{timers = maps:remove(state_timer, Timers)} end. process_timer(SeqId, {Type, Interval, Msg}, Timers) -> - Ref = emqx_misc:start_timer(Interval, {state_machine, {SeqId, Type, Msg}}), + Ref = emqx_utils:start_timer(Interval, {state_machine, {SeqId, Type, Msg}}), Timers#{Type => Ref}. -spec delete_machine(manager_key(), manager()) -> manager(). @@ -293,7 +293,7 @@ delete_machine(Id, Manager) -> } -> lists:foreach( fun({_, Ref}) -> - emqx_misc:cancel_timer(Ref) + emqx_utils:cancel_timer(Ref) end, maps:to_list(Timers) ), diff --git a/apps/emqx_gateway_coap/src/emqx_coap_transport.erl b/apps/emqx_gateway_coap/src/emqx_coap_transport.erl index 1948c969d..daea13ba8 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_transport.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_transport.erl @@ -119,7 +119,7 @@ idle(out, #coap_message{type = non} = Msg, _) -> timeouts => [{stop_timeout, ?NON_LIFETIME}] }); idle(out, Msg, Transport) -> - _ = emqx_misc:rand_seed(), + _ = emqx_utils:rand_seed(), Timeout = ?ACK_TIMEOUT + rand:uniform(?ACK_RANDOM_FACTOR), out(Msg, #{ next => wait_ack, diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl index 7234e7a2f..3b2c8d73b 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl @@ -681,14 +681,14 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> ensure_timer(Name, remove_timer_ref(Name, Channel)). cancel_timer(Name, Channel = #channel{timers = Timers}) -> - emqx_misc:cancel_timer(maps:get(Name, Timers, undefined)), + emqx_utils:cancel_timer(maps:get(Name, Timers, undefined)), remove_timer_ref(Name, Channel). remove_timer_ref(Name, Channel = #channel{timers = Timers}) -> @@ -792,4 +792,4 @@ proto_name_to_protocol(ProtoName) when is_binary(ProtoName) -> binary_to_atom(ProtoName). anonymous_clientid() -> - iolist_to_binary(["exproto-", emqx_misc:gen_id()]). + iolist_to_binary(["exproto-", emqx_utils:gen_id()]). diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl index af15ef9d3..34883cdce 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_gcli.erl @@ -50,7 +50,7 @@ start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [] diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl index a424375dd..77652744a 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl @@ -105,7 +105,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_lwm2m_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_lwm2m_session:info/1, Session); info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> ClientId; info(ctx, #channel{ctx = Ctx}) -> @@ -286,7 +286,7 @@ handle_call(discard, _From, Channel) -> % pendings = Pendings}) -> % ok = emqx_session:takeover(Session), % %% TODO: Should not drain deliver here (side effect) -% Delivers = emqx_misc:drain_deliver(), +% Delivers = emqx_utils:drain_deliver(), % AllPendings = lists:append(Delivers, Pendings), % shutdown_and_reply(takenover, AllPendings, Channel); @@ -390,7 +390,7 @@ set_peercert_infos(Peercert, ClientInfo) -> ClientInfo#{dn => DN, cn => CN}. make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. update_life_timer(#channel{session = Session, timers = Timers} = Channel) -> @@ -413,7 +413,7 @@ do_takeover(_DesireId, Msg, Channel) -> do_connect(Req, Result, Channel, Iter) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun check_lwm2m_version/2, fun enrich_conninfo/2, diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index c27c0ba3f..1ccc8b95a 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -218,7 +218,7 @@ info(conn_state, #channel{conn_state = ConnState}) -> info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> - emqx_misc:maybe_apply(fun emqx_session:info/1, Session); + emqx_utils:maybe_apply(fun emqx_session:info/1, Session); info(will_msg, #channel{will_msg = WillMsg}) -> WillMsg; info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> @@ -282,7 +282,7 @@ enrich_clientinfo( feedvar(Override, Packet, ConnInfo, ClientInfo0), ClientInfo0 ), - {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + {ok, NPacket, NClientInfo} = emqx_utils:pipeline( [ fun maybe_assign_clientid/2, %% FIXME: CALL After authentication successfully @@ -414,7 +414,7 @@ process_connect( Channel#channel{session = Session} ); {ok, #{session := Session, present := true, pendings := Pendings}} -> - Pendings1 = lists:usort(lists:append(Pendings, emqx_misc:drain_deliver())), + Pendings1 = lists:usort(lists:append(Pendings, emqx_utils:drain_deliver())), NChannel = Channel#channel{ session = Session, resuming = true, @@ -595,7 +595,7 @@ handle_in( Channel = #channel{conn_state = idle} ) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun run_conn_hooks/2, @@ -718,7 +718,7 @@ handle_in(PubPkt = ?SN_PUBLISH_MSG(_Flags, TopicId0, MsgId, _Data), Channel) -> Id end, case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun check_qos3_enable/2, fun preproc_pub_pkt/2, @@ -877,7 +877,7 @@ handle_in( end; handle_in(SubPkt = ?SN_SUBSCRIBE_MSG(_, MsgId, _), Channel) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun preproc_subs_type/2, fun check_subscribe_authz/2, @@ -911,7 +911,7 @@ handle_in( Channel ) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun preproc_unsub_type/2, fun run_client_unsub_hook/2, @@ -1823,7 +1823,7 @@ handle_call( ) -> ok = emqx_session:takeover(Session), %% TODO: Should not drain deliver here (side effect) - Delivers = emqx_misc:drain_deliver(), + Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), shutdown_and_reply(takenover, AllPendings, Channel); %handle_call(list_authz_cache, _From, Channel) -> @@ -2247,7 +2247,7 @@ ensure_register_timer(Channel) -> ensure_register_timer(RetryTimes, Channel = #channel{timers = Timers}) -> Msg = maps:get(register_timer, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(?REGISTER_TIMEOUT, {Msg, RetryTimes}), + TRef = emqx_utils:start_timer(?REGISTER_TIMEOUT, {Msg, RetryTimes}), Channel#channel{timers = Timers#{register_timer => TRef}}. cancel_timer(Name, Channel = #channel{timers = Timers}) -> @@ -2255,7 +2255,7 @@ cancel_timer(Name, Channel = #channel{timers = Timers}) -> undefined -> Channel; TRef -> - emqx_misc:cancel_timer(TRef), + emqx_utils:cancel_timer(TRef), Channel#channel{timers = maps:without([Name], Timers)} end. @@ -2270,7 +2270,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index 13b70348a..316432dea 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -252,7 +252,7 @@ enrich_clientinfo( feedvar(Override, Packet, ConnInfo, ClientInfo0), ClientInfo0 ), - {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + {ok, NPacket, NClientInfo} = emqx_utils:pipeline( [ fun maybe_assign_clientid/2, fun parse_heartbeat/2, @@ -416,7 +416,7 @@ handle_in( {error, unexpected_connect, Channel}; handle_in(Packet = ?PACKET(?CMD_CONNECT), Channel) -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun enrich_conninfo/2, fun negotiate_version/2, @@ -474,7 +474,7 @@ handle_in( Topic = header(<<"destination">>, Headers), Ack = header(<<"ack">>, Headers, <<"auto">>), case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun parse_topic_filter/2, fun check_subscribed_status/2, @@ -796,7 +796,7 @@ handle_call( reply({error, no_subid}, Channel); SubId -> case - emqx_misc:pipeline( + emqx_utils:pipeline( [ fun parse_topic_filter/2, fun check_subscribed_status/2 @@ -869,7 +869,7 @@ handle_call(discard, _From, Channel) -> % pendings = Pendings}) -> % ok = emqx_session:takeover(Session), % %% TODO: Should not drain deliver here (side effect) -% Delivers = emqx_misc:drain_deliver(), +% Delivers = emqx_utils:drain_deliver(), % AllPendings = lists:append(Delivers, Pendings), % shutdown_and_reply(takenover, AllPendings, Channel); @@ -1289,7 +1289,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Msg), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> diff --git a/apps/emqx_machine/src/emqx_global_gc.erl b/apps/emqx_machine/src/emqx_global_gc.erl index f17ab2d16..121855e68 100644 --- a/apps/emqx_machine/src/emqx_global_gc.erl +++ b/apps/emqx_machine/src/emqx_global_gc.erl @@ -86,7 +86,7 @@ ensure_timer(State) -> disabled -> State; Interval when is_integer(Interval) -> - TRef = emqx_misc:start_timer(Interval, run), + TRef = emqx_utils:start_timer(Interval, run), State#{timer := TRef} end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index cac3edaed..cd4829342 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -644,7 +644,7 @@ list_clients(QString) -> fun ?MODULE:format_channel_info/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> QStringWithoutNode = maps:without([<<"node">>], QString), emqx_mgmt_api:node_query( diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index 409af4d95..1b69835f9 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -149,7 +149,7 @@ subscriptions(get, #{query_string := QString}) -> fun ?MODULE:format/2 ); Node0 -> - case emqx_misc:safe_to_existing_atom(Node0) of + case emqx_utils:safe_to_existing_atom(Node0) of {ok, Node1} -> emqx_mgmt_api:node_query( Node1, diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index a69837eb3..5df641add 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -498,7 +498,7 @@ download_trace_log(get, #{bindings := #{name := Name}, query_string := Query}) - %% We generate a session ID so that we name files %% with unique names. Then we won't cause %% overwrites for concurrent requests. - SessionId = emqx_misc:gen_id(), + SessionId = emqx_utils:gen_id(), ZipDir = filename:join([emqx_trace:zip_dir(), SessionId]), ok = file:make_dir(ZipDir), %% Write files to ZipDir and create an in-memory zip file diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 6f2a27414..12a7a6641 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -174,7 +174,7 @@ create_app(Name, ApiSecret, Enable, ExpiredAt, Desc) -> desc = Desc, created_at = erlang:system_time(second), api_secret_hash = emqx_dashboard_admin:hash(ApiSecret), - api_key = list_to_binary(emqx_misc:gen_id(16)) + api_key = list_to_binary(emqx_utils:gen_id(16)) }, case create_app(App) of {ok, Res} -> @@ -213,7 +213,7 @@ do_force_create_app(App, ApiKey, NamePrefix) -> end. generate_unique_name(NamePrefix) -> - New = list_to_binary(NamePrefix ++ emqx_misc:gen_id(16)), + New = list_to_binary(NamePrefix ++ emqx_utils:gen_id(16)), case mnesia:read(?APP, New) of [] -> New; _ -> generate_unique_name(NamePrefix) @@ -246,7 +246,7 @@ init_bootstrap_file(File) -> {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]), init_bootstrap_file(File, Dev, MP); {error, Reason0} -> - Reason = emqx_misc:explain_posix(Reason0), + Reason = emqx_utils:explain_posix(Reason0), ?SLOG( error, #{ diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index 442d5c7de..253d527ac 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -356,7 +356,7 @@ mnesia(_) -> %% @doc Logger Command log(["set-level", Level]) -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> case emqx_logger:set_log_level(Level1) of ok -> emqx_ctl:print("~ts~n", [Level]); @@ -369,7 +369,7 @@ log(["primary-level"]) -> Level = emqx_logger:get_primary_log_level(), emqx_ctl:print("~ts~n", [Level]); log(["primary-level", Level]) -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> _ = emqx_logger:set_primary_log_level(Level1), ok; @@ -392,7 +392,7 @@ log(["handlers", "list"]) -> ], ok; log(["handlers", "start", HandlerId]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> case emqx_logger:start_log_handler(HandlerId1) of ok -> @@ -406,7 +406,7 @@ log(["handlers", "start", HandlerId]) -> emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(["handlers", "stop", HandlerId]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> case emqx_logger:stop_log_handler(HandlerId1) of ok -> @@ -420,9 +420,9 @@ log(["handlers", "stop", HandlerId]) -> emqx_ctl:print("[error] invalid handler:~ts~n", [HandlerId]) end; log(["handlers", "set-level", HandlerId, Level]) -> - case emqx_misc:safe_to_existing_atom(HandlerId) of + case emqx_utils:safe_to_existing_atom(HandlerId) of {ok, HandlerId1} -> - case emqx_misc:safe_to_existing_atom(Level) of + case emqx_utils:safe_to_existing_atom(Level) of {ok, Level1} -> case emqx_logger:set_log_handler_level(HandlerId1, Level1) of ok -> @@ -628,7 +628,7 @@ listeners([]) -> emqx_listeners:list() ); listeners(["stop", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:stop_listener(ListenerId1) of ok -> @@ -640,7 +640,7 @@ listeners(["stop", ListenerId]) -> emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(["start", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:start_listener(ListenerId1) of ok -> @@ -652,7 +652,7 @@ listeners(["start", ListenerId]) -> emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId]) end; listeners(["restart", ListenerId]) -> - case emqx_misc:safe_to_existing_atom(ListenerId) of + case emqx_utils:safe_to_existing_atom(ListenerId) of {ok, ListenerId1} -> case emqx_listeners:restart_listener(ListenerId1) of ok -> diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 91cdbedd3..85313c181 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -328,7 +328,7 @@ handle_info(Info, State) -> terminate(_Reason, #{stats_timer := StatsTimer} = State) -> emqx_conf:remove_handler([delayed]), - emqx_misc:cancel_timer(StatsTimer), + emqx_utils:cancel_timer(StatsTimer), do_load_or_unload(false, State). code_change(_Vsn, State, _Extra) -> @@ -370,14 +370,14 @@ ensure_publish_timer({Ts, _Id}, State = #{publish_timer := undefined}) -> ensure_publish_timer({Ts, _Id}, State = #{publish_timer := TRef, publish_at := PubAt}) when Ts < PubAt -> - ok = emqx_misc:cancel_timer(TRef), + ok = emqx_utils:cancel_timer(TRef), ensure_publish_timer(Ts, ?NOW, State); ensure_publish_timer(_Key, State) -> State. ensure_publish_timer(Ts, Now, State) -> Interval = max(1, Ts - Now), - TRef = emqx_misc:start_timer(Interval, do_publish), + TRef = emqx_utils:start_timer(Interval, do_publish), State#{publish_timer := TRef, publish_at := Now + Interval}. do_publish(Key, Now) -> @@ -418,7 +418,7 @@ do_load_or_unload(true, State) -> State; do_load_or_unload(false, #{publish_timer := PubTimer} = State) -> emqx_hooks:del('message.publish', {?MODULE, on_message_publish}), - emqx_misc:cancel_timer(PubTimer), + emqx_utils:cancel_timer(PubTimer), ets:delete_all_objects(?TAB), State#{publish_timer := undefined, publish_at := 0}; do_load_or_unload(_, State) -> diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl index 1d49fb3e7..3b27302df 100644 --- a/apps/emqx_modules/src/emqx_telemetry.erl +++ b/apps/emqx_modules/src/emqx_telemetry.erl @@ -161,7 +161,7 @@ handle_call(enable, _From, State) -> FirstReportTimeoutMS = timer:seconds(10), {reply, ok, ensure_report_timer(FirstReportTimeoutMS, State)}; handle_call(disable, _From, State = #state{timer = Timer}) -> - emqx_misc:cancel_timer(Timer), + emqx_utils:cancel_timer(Timer), {reply, ok, State#state{timer = undefined}}; handle_call(get_node_uuid, _From, State = #state{node_uuid = UUID}) -> {reply, {ok, UUID}, State}; @@ -205,7 +205,7 @@ ensure_report_timer(State = #state{report_interval = ReportInterval}) -> ensure_report_timer(ReportInterval, State). ensure_report_timer(ReportInterval, State) -> - State#state{timer = emqx_misc:start_timer(ReportInterval, time_to_report_telemetry_data)}. + State#state{timer = emqx_utils:start_timer(ReportInterval, time_to_report_telemetry_data)}. os_info() -> case erlang:system_info(os_type) of diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl index 9b286f360..a3048e5dd 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl @@ -79,7 +79,7 @@ health_check_ecpool_workers(PoolName, CheckFunc, Timeout) -> false end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of [_ | _] = Status -> lists:all(fun(St) -> St =:= true end, Status); [] -> diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index 40f88a0dc..8844fe586 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -225,7 +225,7 @@ tcp_connectivity(Host, Port) -> ) -> ok | {error, Reason :: term()}. tcp_connectivity(Host, Port, Timeout) -> - case gen_tcp:connect(Host, Port, emqx_misc:ipv6_probe([]), Timeout) of + case gen_tcp:connect(Host, Port, emqx_utils:ipv6_probe([]), Timeout) of {ok, Sock} -> gen_tcp:close(Sock), ok; diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 60d52f58b..05d9508b6 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -144,7 +144,7 @@ terminate(_Reason, _State) -> ok. ensure_timer(Interval) -> - emqx_misc:start_timer(Interval, ?TIMER_MSG). + emqx_utils:start_timer(Interval, ?TIMER_MSG). %%-------------------------------------------------------------------- %% prometheus callbacks diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 0fa4c0bd8..331480064 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1535,7 +1535,7 @@ queue_count(Q) -> disk_queue_dir(Id, Index) -> QDir0 = binary_to_list(Id) ++ ":" ++ integer_to_list(Index), QDir = filename:join([emqx:data_dir(), "bufs", node(), QDir0]), - emqx_misc:safe_filename(QDir). + emqx_utils:safe_filename(QDir). clear_disk_queue_dir(Id, Index) -> ReplayQDir = disk_queue_dir(Id, Index), diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 9fe0fbdf2..7ecf56c18 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -539,7 +539,7 @@ stop_resource(#data{state = ResState, id = ResId} = Data) -> Data#data{status = stopped}. make_test_id() -> - RandId = iolist_to_binary(emqx_misc:gen_id(16)), + RandId = iolist_to_binary(emqx_utils:gen_id(16)), <>. handle_manually_health_check(From, Data) -> @@ -613,7 +613,7 @@ maybe_alarm(_Status, ResId, Error) -> HrError = case Error of undefined -> <<"Unknown reason">>; - _Else -> emqx_misc:readable_error_msg(Error) + _Else -> emqx_utils:readable_error_msg(Error) end, emqx_alarm:activate( ResId, diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index 11013cdd3..7bfc8ee4e 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl index 454a65eb3..32cbba084 100644 --- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl +++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl @@ -91,7 +91,7 @@ worker() -> | ignore. start_link(Pool, Id) -> gen_server:start_link( - {local, emqx_misc:proc_name(?MODULE, Id)}, + {local, emqx_utils:proc_name(?MODULE, Id)}, ?MODULE, [Pool, Id], [{hibernate_after, 1000}] diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 40158d342..a9f24ddcd 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -175,7 +175,7 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) -> flags = Flags, headers = #{ republish_by => RuleId, - properties => emqx_misc:pub_props_to_packet(PubProps) + properties => emqx_utils:pub_props_to_packet(PubProps) }, topic = Topic, payload = Payload, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index c63d6275f..d66f2c1c9 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -343,7 +343,7 @@ param_path_id() -> {200, Result} end; '/rules'(post, #{body := Params0}) -> - case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of + case maps:get(<<"id">>, Params0, list_to_binary(emqx_utils:gen_id(8))) of <<>> -> {400, #{code => 'BAD_REQUEST', message => <<"empty rule id is not allowed">>}}; Id -> @@ -459,11 +459,11 @@ param_path_id() -> %%------------------------------------------------------------------------------ err_msg({RuleError, {_E, Reason, _S}}) -> - emqx_misc:readable_error_msg(encode_nested_error(RuleError, Reason)); + emqx_utils:readable_error_msg(encode_nested_error(RuleError, Reason)); err_msg({Reason, _Details}) -> - emqx_misc:readable_error_msg(Reason); + emqx_utils:readable_error_msg(Reason); err_msg(Msg) -> - emqx_misc:readable_error_msg(Msg). + emqx_utils:readable_error_msg(Msg). encode_nested_error(RuleError, Reason) when is_tuple(Reason) -> encode_nested_error(RuleError, element(1, Reason)); diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 944aa02f8..81435d9ac 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -643,10 +643,10 @@ map(Data) -> emqx_plugin_libs_rule:map(Data). bin2hexstr(Bin) when is_binary(Bin) -> - emqx_misc:bin_to_hexstr(Bin, upper). + emqx_utils:bin_to_hexstr(Bin, upper). hexstr2bin(Str) when is_binary(Str) -> - emqx_misc:hexstr_to_bin(Str). + emqx_utils:hexstr_to_bin(Str). %%------------------------------------------------------------------------------ %% NULL Funcs @@ -944,7 +944,7 @@ sha256(S) when is_binary(S) -> hash(sha256, S). hash(Type, Data) -> - emqx_misc:bin_to_hexstr(crypto:hash(Type, Data), lower). + emqx_utils:bin_to_hexstr(crypto:hash(Type, Data), lower). %%------------------------------------------------------------------------------ %% gzip Funcs diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index e97c45b35..455efe389 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -48,7 +48,7 @@ test(#{sql := Sql, context := Context}) -> end. test_rule(Sql, Select, Context, EventTopics) -> - RuleId = iolist_to_binary(["sql_tester:", emqx_misc:gen_id(16)]), + RuleId = iolist_to_binary(["sql_tester:", emqx_utils:gen_id(16)]), ok = emqx_rule_engine:maybe_add_metrics_for_rule(RuleId), Rule = #{ id => RuleId, diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 9d40c6857..412e0b685 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_statsd, [ {description, "EMQX Statsd"}, - {vsn, "5.0.7"}, + {vsn, "5.0.8"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, [ diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl index c2e1819ac..c5a7fc1c8 100644 --- a/apps/emqx_statsd/src/emqx_statsd.erl +++ b/apps/emqx_statsd/src/emqx_statsd.erl @@ -144,7 +144,7 @@ flush_interval(_FlushInterval, SampleInterval) -> SampleInterval. ensure_timer(State = #{sample_time_interval := SampleTimeInterval}) -> - State#{timer => emqx_misc:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}. + State#{timer => emqx_utils:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}. check_multicall_result({Results, []}) -> case diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx_utils/src/emqx_utils.erl similarity index 99% rename from apps/emqx/src/emqx_misc.erl rename to apps/emqx_utils/src/emqx_utils.erl index d0c7397c8..21dbd339d 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -14,14 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_misc). +-module(emqx_utils). -compile(inline). +%% [TODO] Cleanup so the instruction below is not necessary. -elvis([{elvis_style, god_modules, disable}]). --include("types.hrl"). --include("logger.hrl"). - -export([ merge_opts/2, maybe_apply/2, @@ -71,6 +69,8 @@ -export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]). +-type maybe(T) :: undefined | T. + -dialyzer({nowarn_function, [nolink_apply/2]}). -define(SHORT, 8). @@ -221,6 +221,7 @@ drain_down(Cnt, Acc) -> %% `ok': There is nothing out of the ordinary. %% `shutdown': Some numbers (message queue length hit the limit), %% hence shutdown for greater good (system stability). +%% [FIXME] cross-dependency on `emqx_types`. -spec check_oom(emqx_types:oom_policy()) -> ok | {shutdown, term()}. check_oom(Policy) -> check_oom(self(), Policy). @@ -279,6 +280,7 @@ proc_name(Mod, Id) -> list_to_atom(lists:concat([Mod, "_", Id])). %% Get Proc's Stats. +%% [FIXME] cross-dependency on `emqx_types`. -spec proc_stats() -> emqx_types:stats(). proc_stats() -> proc_stats(self()). diff --git a/apps/emqx_utils/test/emqx_utils_SUITE.erl b/apps/emqx_utils/test/emqx_utils_SUITE.erl index 09a8d085c..598f05a21 100644 --- a/apps/emqx_utils/test/emqx_utils_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -20,29 +20,188 @@ -compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). -all() -> - emqx_common_test_helpers:all(?MODULE). +-define(SOCKOPTS, [ + binary, + {packet, raw}, + {reuseaddr, true}, + {backlog, 512}, + {nodelay, true} +]). -init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([emqx_conf, emqx_utils]), - Config. +all() -> emqx_common_test_helpers:all(?MODULE). -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_conf, emqx_utils]), - ok. +t_merge_opts(_) -> + Opts = emqx_utils:merge_opts(?SOCKOPTS, [ + raw, + binary, + {backlog, 1024}, + {nodelay, false}, + {max_clients, 1024}, + {acceptors, 16} + ]), + ?assertEqual(1024, proplists:get_value(backlog, Opts)), + ?assertEqual(1024, proplists:get_value(max_clients, Opts)), + ?assertEqual( + [ + binary, + raw, + {acceptors, 16}, + {backlog, 1024}, + {max_clients, 1024}, + {nodelay, false}, + {packet, raw}, + {reuseaddr, true} + ], + lists:sort(Opts) + ). -init_per_testcase(TestCase, Config) -> - emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config). +t_maybe_apply(_) -> + ?assertEqual(undefined, emqx_utils:maybe_apply(fun(A) -> A end, undefined)), + ?assertEqual(a, emqx_utils:maybe_apply(fun(A) -> A end, a)). -end_per_testcase(TestCase, Config) -> - emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config). +t_run_fold(_) -> + ?assertEqual(1, emqx_utils:run_fold([], 1, state)), + Add = fun(I, St) -> I + St end, + Mul = fun(I, St) -> I * St end, + ?assertEqual(6, emqx_utils:run_fold([Add, Mul], 1, 2)). -t_fail(init, Config) -> - Config; -t_fail('end', _Config) -> - ok. +t_pipeline(_) -> + ?assertEqual({ok, input, state}, emqx_utils:pipeline([], input, state)), + Funs = [ + fun(_I, _St) -> ok end, + fun(_I, St) -> {ok, St + 1} end, + fun(I, St) -> {ok, I + 1, St + 1} end, + fun(I, St) -> {ok, I * 2, St * 2} end + ], + ?assertEqual({ok, 4, 6}, emqx_utils:pipeline(Funs, 1, 1)), + ?assertEqual( + {error, undefined, 1}, emqx_utils:pipeline([fun(_I) -> {error, undefined} end], 1, 1) + ), + ?assertEqual( + {error, undefined, 2}, + emqx_utils:pipeline([fun(_I, _St) -> {error, undefined, 2} end], 1, 1) + ). -t_fail(_Config) -> - ?assert(false). +t_start_timer(_) -> + TRef = emqx_utils:start_timer(1, tmsg), + timer:sleep(2), + ?assertEqual([{timeout, TRef, tmsg}], drain()), + ok = emqx_utils:cancel_timer(TRef). + +t_cancel_timer(_) -> + Timer = emqx_utils:start_timer(0, foo), + ok = emqx_utils:cancel_timer(Timer), + ?assertEqual([], drain()), + ok = emqx_utils:cancel_timer(undefined). + +t_proc_name(_) -> + ?assertEqual(emqx_pool_1, emqx_utils:proc_name(emqx_pool, 1)). + +t_proc_stats(_) -> + Pid1 = spawn(fun() -> exit(normal) end), + timer:sleep(10), + ?assertEqual([], emqx_utils:proc_stats(Pid1)), + Pid2 = spawn(fun() -> + ?assertMatch([{mailbox_len, 0} | _], emqx_utils:proc_stats()), + timer:sleep(200) + end), + timer:sleep(10), + Pid2 ! msg, + timer:sleep(10), + ?assertMatch([{mailbox_len, 1} | _], emqx_utils:proc_stats(Pid2)). + +t_drain_deliver(_) -> + self() ! {deliver, t1, m1}, + self() ! {deliver, t2, m2}, + ?assertEqual( + [ + {deliver, t1, m1}, + {deliver, t2, m2} + ], + emqx_utils:drain_deliver(2) + ). + +t_drain_down(_) -> + {Pid1, _Ref1} = erlang:spawn_monitor(fun() -> ok end), + {Pid2, _Ref2} = erlang:spawn_monitor(fun() -> ok end), + timer:sleep(100), + ?assertEqual([Pid1, Pid2], lists:sort(emqx_utils:drain_down(2))), + ?assertEqual([], emqx_utils:drain_down(1)). + +t_index_of(_) -> + try emqx_utils:index_of(a, []) of + _ -> ct:fail(should_throw_error) + catch + error:Reason -> + ?assertEqual(badarg, Reason) + end, + ?assertEqual(3, emqx_utils:index_of(a, [b, c, a, e, f])). + +t_check(_) -> + Policy = #{ + max_message_queue_len => 10, + max_heap_size => 1024 * 1024 * 8, + enable => true + }, + [self() ! {msg, I} || I <- lists:seq(1, 5)], + ?assertEqual(ok, emqx_utils:check_oom(Policy)), + [self() ! {msg, I} || I <- lists:seq(1, 6)], + ?assertEqual({shutdown, message_queue_too_long}, emqx_utils:check_oom(Policy)). + +drain() -> + drain([]). + +drain(Acc) -> + receive + Msg -> drain([Msg | Acc]) + after 0 -> + lists:reverse(Acc) + end. + +t_rand_seed(_) -> + ?assert(is_tuple(emqx_utils:rand_seed())). + +t_now_to_secs(_) -> + ?assert(is_integer(emqx_utils:now_to_secs(os:timestamp()))). + +t_now_to_ms(_) -> + ?assert(is_integer(emqx_utils:now_to_ms(os:timestamp()))). + +t_gen_id(_) -> + ?assertEqual(10, length(emqx_utils:gen_id(10))), + ?assertEqual(20, length(emqx_utils:gen_id(20))). + +t_pmap_normal(_) -> + ?assertEqual( + [5, 7, 9], + emqx_utils:pmap( + fun({A, B}) -> A + B end, + [{2, 3}, {3, 4}, {4, 5}] + ) + ). + +t_pmap_timeout(_) -> + ?assertExit( + timeout, + emqx_utils:pmap( + fun + (timeout) -> ct:sleep(1000); + ({A, B}) -> A + B + end, + [{2, 3}, {3, 4}, timeout], + 100 + ) + ). + +t_pmap_exception(_) -> + ?assertError( + foobar, + emqx_utils:pmap( + fun + (error) -> error(foobar); + ({A, B}) -> A + B + end, + [{2, 3}, {3, 4}, error] + ) + ). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index e899b0fcd..133ef0430 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -272,7 +272,7 @@ t_setup_via_config_and_publish(Config) -> {ok, _}, create_bridge(Config) ), - MsgId = emqx_misc:gen_id(), + MsgId = emqx_utils:gen_id(), SentData = #{id => MsgId, payload => ?PAYLOAD}, ?check_trace( begin @@ -309,7 +309,7 @@ t_setup_via_http_api_and_publish(Config) -> {ok, _}, create_bridge_http(PgsqlConfig) ), - MsgId = emqx_misc:gen_id(), + MsgId = emqx_utils:gen_id(), SentData = #{id => MsgId, payload => ?PAYLOAD}, ?check_trace( begin @@ -375,7 +375,7 @@ t_write_failure(Config) -> #{?snk_kind := resource_connected_enter}, 20_000 ), - SentData = #{id => emqx_misc:gen_id(), payload => ?PAYLOAD}, + SentData = #{id => emqx_utils:gen_id(), payload => ?PAYLOAD}, emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> ?assertMatch( {error, {resource_error, #{reason := timeout}}}, send_message(Config, SentData) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 3cc4c75e4..49bc444c6 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -809,7 +809,7 @@ test_publish_success_batch(Config) -> %% making 1-sized batches. also important to note that the pool %% size for the resource (replayq buffering) must be set to 1 to %% avoid further segmentation of batches. - emqx_misc:pmap(fun emqx:publish/1, Messages), + emqx_utils:pmap(fun emqx:publish/1, Messages), DecodedMessages0 = assert_http_request(ServiceAccountJSON), ?assertEqual(BatchSize, length(DecodedMessages0)), DecodedMessages1 = assert_http_request(ServiceAccountJSON), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl index 69b1e26ce..be07a2bb7 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl @@ -615,7 +615,7 @@ t_workload_fits_prepared_statement_limit(Config) -> create_bridge(Config) ), Results = lists:append( - emqx_misc:pmap( + emqx_utils:pmap( fun(_) -> [ begin diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index ce964dd17..86b908038 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -102,7 +102,7 @@ on_start( ?SLOG(info, #{ msg => "starting_cassandra_connector", connector => InstId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), Options = [ diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl index 3749ecc6e..2872e8cf0 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl @@ -139,7 +139,7 @@ on_start( ?SLOG(info, #{ msg => "starting_clickhouse_connector", connector => InstanceID, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), PoolName = emqx_plugin_libs_pool:pool_name(InstanceID), Options = [ @@ -181,7 +181,7 @@ log_start_error(Config, Reason, Stacktrace) -> #{ msg => "clickhouse_connector_start_failed", error_reason => Reason, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }, ?SLOG(info, maps:merge(LogMessage, StacktraceMap)), ?tp( @@ -318,7 +318,7 @@ do_get_status(PoolName, Timeout) -> Error end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of Results -> case [E || {error, _} = E <- Results] of [] -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index ee56e0c5f..85daefbb7 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -95,7 +95,7 @@ on_start( ?SLOG(info, #{ msg => "starting_dynamo_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), {Schema, Server} = get_host_schema(to_str(Url)), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl index a2045b8c5..2295f63ab 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl @@ -86,7 +86,7 @@ on_start( PoolType = random, Transport = tls, TransportOpts = emqx_tls_lib:to_client_opts(#{enable => true, verify => verify_none}), - NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), + NTransportOpts = emqx_utils:ipv6_probe(TransportOpts), PoolOpts = [ {host, Host}, {port, Port}, @@ -587,7 +587,7 @@ do_get_status(InstanceId, PoolName, Timeout) -> false end end, - try emqx_misc:pmap(DoPerWorker, Workers, Timeout) of + try emqx_utils:pmap(DoPerWorker, Workers, Timeout) of [_ | _] = Status -> lists:all(fun(St) -> St =:= true end, Status); [] -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index afe17cae6..700eb2a81 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -216,8 +216,8 @@ start_client(InstId, Config) -> ?SLOG(info, #{ msg => "starting influxdb connector", connector => InstId, - config => emqx_misc:redact(Config), - client_config => emqx_misc:redact(ClientConfig) + config => emqx_utils:redact(Config), + client_config => emqx_utils:redact(ClientConfig) }), try do_start_client(InstId, ClientConfig, Config) @@ -353,7 +353,7 @@ password(_) -> []. redact_auth(Term) -> - emqx_misc:redact(Term, fun is_auth_key/1). + emqx_utils:redact(Term, fun is_auth_key/1). is_auth_key(Key) when is_binary(Key) -> string:equal("authorization", Key, true); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl index d296b0b4f..205359bb8 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_rocketmq.erl @@ -265,7 +265,7 @@ client_id(InstanceId) -> erlang:binary_to_atom(Name, utf8). redact(Msg) -> - emqx_misc:redact(Msg, fun is_sensitive_key/1). + emqx_utils:redact(Msg, fun is_sensitive_key/1). is_sensitive_key(security_token) -> true; diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl index 746ab5485..9b6718882 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl @@ -93,7 +93,7 @@ on_start( ?SLOG(info, #{ msg => "starting_tdengine_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), {Host, Port} = emqx_schema:parse_server(Server, ?TD_HOST_OPTIONS), diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 3c26d1966..43ccd9fc1 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -218,7 +218,7 @@ start_cluster(Cluster) -> || {Name, Opts} <- Cluster ], on_exit(fun() -> - emqx_misc:pmap( + emqx_utils:pmap( fun(N) -> ct:pal("stopping ~p", [N]), ok = emqx_common_test_helpers:stop_slave(N) diff --git a/lib-ee/emqx_license/src/emqx_license_http_api.erl b/lib-ee/emqx_license/src/emqx_license_http_api.erl index 0133cd637..ef0e1ee52 100644 --- a/lib-ee/emqx_license/src/emqx_license_http_api.erl +++ b/lib-ee/emqx_license/src/emqx_license_http_api.erl @@ -95,7 +95,7 @@ sample_license_info_response() -> }. error_msg(Code, Msg) -> - #{code => Code, message => emqx_misc:readable_error_msg(Msg)}. + #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. %% read license info '/license'(get, _Params) -> From d0df086c80bce1c9aa9c3d2ad4de0e98f947cd09 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 14:50:30 +0200 Subject: [PATCH 115/279] refactor: rename emqx_api_lib to emqx_utils_api --- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl | 6 +++--- apps/emqx_management/src/emqx_mgmt_api_nodes.erl | 6 +++--- .../include/emqx_utils_api.hrl} | 0 apps/emqx_utils/src/emqx_utils.app.src | 1 + .../src/emqx_utils_api.erl} | 4 ++-- .../test/emqx_utils_api_SUITE.erl} | 10 +++++----- .../src/emqx_ee_schema_registry_http_api.erl | 2 +- 8 files changed, 16 insertions(+), 15 deletions(-) rename apps/{emqx/include/emqx_api_lib.hrl => emqx_utils/include/emqx_utils_api.hrl} (100%) rename apps/{emqx/src/emqx_api_lib.erl => emqx_utils/src/emqx_utils_api.erl} (97%) rename apps/{emqx/test/emqx_api_lib_SUITE.erl => emqx_utils/test/emqx_utils_api_SUITE.erl} (92%) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 814fcdeb7..a1841b705 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -20,7 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_api_lib.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). -include_lib("emqx_bridge/include/emqx_bridge.hrl"). -import(hoconsc, [mk/2, array/1, enum/1]). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index c2dd4a9dd..c0e162b62 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -122,7 +122,7 @@ fields(sampler_current) -> monitor(get, #{query_string := QS, bindings := Bindings}) -> Latest = maps:get(<<"latest">>, QS, infinity), RawNode = maps:get(node, Bindings, <<"all">>), - emqx_api_lib:with_node_or_cluster(RawNode, dashboard_samplers_fun(Latest)). + emqx_utils_api:with_node_or_cluster(RawNode, dashboard_samplers_fun(Latest)). dashboard_samplers_fun(Latest) -> fun(NodeOrCluster) -> @@ -133,10 +133,10 @@ dashboard_samplers_fun(Latest) -> end. monitor_current(get, #{bindings := []}) -> - emqx_api_lib:with_node_or_cluster(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); + emqx_utils_api:with_node_or_cluster(erlang:node(), fun emqx_dashboard_monitor:current_rate/1); monitor_current(get, #{bindings := Bindings}) -> RawNode = maps:get(node, Bindings, <<"all">>), - emqx_api_lib:with_node_or_cluster(RawNode, fun current_rate/1). + emqx_utils_api:with_node_or_cluster(RawNode, fun current_rate/1). current_rate(Node) -> case emqx_dashboard_monitor:current_rate(Node) of diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index a4173f5b0..ecf465f43 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -247,13 +247,13 @@ nodes(get, _Params) -> list_nodes(#{}). node(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun get_node/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun get_node/1)). node_metrics(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_metrics/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_metrics/1)). node_stats(get, #{bindings := #{node := NodeName}}) -> - emqx_api_lib:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_stats/1)). + emqx_utils_api:with_node(NodeName, to_ok_result_fun(fun emqx_mgmt:get_stats/1)). %%-------------------------------------------------------------------- %% api apply diff --git a/apps/emqx/include/emqx_api_lib.hrl b/apps/emqx_utils/include/emqx_utils_api.hrl similarity index 100% rename from apps/emqx/include/emqx_api_lib.hrl rename to apps/emqx_utils/include/emqx_utils_api.hrl diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index e1f10c139..08a802d1b 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -5,6 +5,7 @@ {vsn, "5.0.0"}, {modules, [ emqx_utils, + emqx_utils_api, emqx_utils_binary, emqx_utils_ets, emqx_utils_json, diff --git a/apps/emqx/src/emqx_api_lib.erl b/apps/emqx_utils/src/emqx_utils_api.erl similarity index 97% rename from apps/emqx/src/emqx_api_lib.erl rename to apps/emqx_utils/src/emqx_utils_api.erl index 5eddf6188..e6bd07272 100644 --- a/apps/emqx/src/emqx_api_lib.erl +++ b/apps/emqx_utils/src/emqx_utils_api.erl @@ -14,14 +14,14 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_api_lib). +-module(emqx_utils_api). -export([ with_node/2, with_node_or_cluster/2 ]). --include("emqx_api_lib.hrl"). +-include("emqx_utils_api.hrl"). -define(NODE_NOT_FOUND(NODE), ?NOT_FOUND(<<"Node not found: ", NODE/binary>>)). diff --git a/apps/emqx/test/emqx_api_lib_SUITE.erl b/apps/emqx_utils/test/emqx_utils_api_SUITE.erl similarity index 92% rename from apps/emqx/test/emqx_api_lib_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_api_SUITE.erl index 29f5c6095..3ed3cd250 100644 --- a/apps/emqx/test/emqx_api_lib_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_api_SUITE.erl @@ -14,12 +14,12 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_api_lib_SUITE). +-module(emqx_utils_api_SUITE). -compile(export_all). -compile(nowarn_export_all). --include("emqx_api_lib.hrl"). +-include("emqx_utils_api.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(DUMMY, dummy_module). @@ -45,14 +45,14 @@ end_per_testcase(_Case, _Config) -> meck:unload(?DUMMY). t_with_node(_) -> - test_with(fun emqx_api_lib:with_node/2, [<<"all">>]). + test_with(fun emqx_utils_api:with_node/2, [<<"all">>]). t_with_node_or_cluster(_) -> - test_with(fun emqx_api_lib:with_node_or_cluster/2, []), + test_with(fun emqx_utils_api:with_node_or_cluster/2, []), meck:reset(?DUMMY), ?assertEqual( ?OK(success), - emqx_api_lib:with_node_or_cluster( + emqx_utils_api:with_node_or_cluster( <<"all">>, fun ?DUMMY:expect_success/1 ) diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl index 897d29e07..c7cc6c99a 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_http_api.erl @@ -8,7 +8,7 @@ -include("emqx_ee_schema_registry.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_api_lib.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). -export([ namespace/0, From 6e8665365baa952dff6161bd66290dd53c1bbaa8 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 14:57:45 +0200 Subject: [PATCH 116/279] refactor: rename emqx_tables to emqx_utils_ets --- apps/emqx/src/emqx_broker.erl | 8 ++++---- apps/emqx/src/emqx_broker_helper.erl | 10 +++++----- apps/emqx/src/emqx_cm.erl | 8 ++++---- apps/emqx/src/emqx_flapping.erl | 2 +- apps/emqx/src/emqx_hooks.erl | 2 +- apps/emqx/src/emqx_metrics.erl | 2 +- apps/emqx/src/emqx_ocsp_cache.erl | 2 +- apps/emqx/src/emqx_sequence.erl | 2 +- apps/emqx/src/emqx_session_router.erl | 10 +++++----- apps/emqx/src/emqx_shared_sub.erl | 8 +++++--- apps/emqx/src/emqx_stats.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_cm.erl | 6 +++--- apps/emqx_gateway/src/emqx_gateway_metrics.erl | 2 +- apps/emqx_modules/src/emqx_topic_metrics.erl | 2 +- .../src/emqx_utils_ets.erl} | 2 +- .../test/emqx_utils_ets_SUITE.erl} | 18 +++++++++--------- .../test/emqx_ee_schema_registry_SUITE.erl | 2 +- 17 files changed, 45 insertions(+), 43 deletions(-) rename apps/{emqx/src/emqx_tables.erl => emqx_utils/src/emqx_utils_ets.erl} (98%) rename apps/{emqx/test/emqx_tables_SUITE.erl => emqx_utils/test/emqx_utils_ets_SUITE.erl} (73%) diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index 7e00c050e..5f7c4aaf5 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -71,7 +71,7 @@ code_change/3 ]). --import(emqx_tables, [lookup_value/2, lookup_value/3]). +-import(emqx_utils_ets, [lookup_value/2, lookup_value/3]). -ifdef(TEST). -compile(export_all). @@ -107,15 +107,15 @@ create_tabs() -> TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}], %% SubOption: {Topic, SubPid} -> SubOption - ok = emqx_tables:new(?SUBOPTION, [ordered_set | TabOpts]), + ok = emqx_utils_ets:new(?SUBOPTION, [ordered_set | TabOpts]), %% Subscription: SubPid -> Topic1, Topic2, Topic3, ... %% duplicate_bag: o(1) insert - ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), + ok = emqx_utils_ets:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... %% bag: o(n) insert:( - ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). + ok = emqx_utils_ets:new(?SUBSCRIBER, [bag | TabOpts]). %%------------------------------------------------------------------------------ %% Subscribe API diff --git a/apps/emqx/src/emqx_broker_helper.erl b/apps/emqx/src/emqx_broker_helper.erl index 87b291361..06f249678 100644 --- a/apps/emqx/src/emqx_broker_helper.erl +++ b/apps/emqx/src/emqx_broker_helper.erl @@ -73,11 +73,11 @@ register_sub(SubPid, SubId) when is_pid(SubPid) -> -spec lookup_subid(pid()) -> maybe(emqx_types:subid()). lookup_subid(SubPid) when is_pid(SubPid) -> - emqx_tables:lookup_value(?SUBMON, SubPid). + emqx_utils_ets:lookup_value(?SUBMON, SubPid). -spec lookup_subpid(emqx_types:subid()) -> maybe(pid()). lookup_subpid(SubId) -> - emqx_tables:lookup_value(?SUBID, SubId). + emqx_utils_ets:lookup_value(?SUBID, SubId). -spec get_sub_shard(pid(), emqx_types:topic()) -> non_neg_integer(). get_sub_shard(SubPid, Topic) -> @@ -105,15 +105,15 @@ reclaim_seq(Topic) -> init([]) -> %% Helper table - ok = emqx_tables:new(?HELPER, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?HELPER, [{read_concurrency, true}]), %% Shards: CPU * 32 true = ets:insert(?HELPER, {shards, emqx_vm:schedulers() * 32}), %% SubSeq: Topic -> SeqId ok = emqx_sequence:create(?SUBSEQ), %% SubId: SubId -> SubPid - ok = emqx_tables:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), %% SubMon: SubPid -> SubId - ok = emqx_tables:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), %% Stats timer ok = emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), {ok, #{pmon => emqx_pmon:new()}}. diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index 346e8dcb5..0290b57d3 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -651,10 +651,10 @@ cast(Msg) -> gen_server:cast(?CM, Msg). init([]) -> TabOpts = [public, {write_concurrency, true}], - ok = emqx_tables:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]), - ok = emqx_tables:new(?CHAN_CONN_TAB, [bag | TabOpts]), - ok = emqx_tables:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]), - ok = emqx_tables:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_TAB, [bag, {read_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_CONN_TAB, [bag | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_INFO_TAB, [ordered_set, compressed | TabOpts]), + ok = emqx_utils_ets:new(?CHAN_LIVE_TAB, [ordered_set, {write_concurrency, true} | TabOpts]), ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0), State = #{chan_pmon => emqx_pmon:new()}, {ok, State}. diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 2cfed0a8e..46193fb16 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -99,7 +99,7 @@ now_diff(TS) -> erlang:system_time(millisecond) - TS. %%-------------------------------------------------------------------- init([]) -> - ok = emqx_tables:new(?FLAPPING_TAB, [ + ok = emqx_utils_ets:new(?FLAPPING_TAB, [ public, set, {keypos, #flapping.clientid}, diff --git a/apps/emqx/src/emqx_hooks.erl b/apps/emqx/src/emqx_hooks.erl index 1784d8ea3..0b8dc0941 100644 --- a/apps/emqx/src/emqx_hooks.erl +++ b/apps/emqx/src/emqx_hooks.erl @@ -229,7 +229,7 @@ lookup(HookPoint) -> %%-------------------------------------------------------------------- init([]) -> - ok = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), {ok, #{}}. handle_call({add, HookPoint, Callback = #callback{action = {M, F, _}}}, _From, State) -> diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index c2e297623..21a114c0f 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -541,7 +541,7 @@ init([]) -> CRef = counters:new(?MAX_SIZE, [write_concurrency]), ok = persistent_term:put(?MODULE, CRef), % Create index mapping table - ok = emqx_tables:new(?TAB, [{keypos, 2}, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{keypos, 2}, {read_concurrency, true}]), Metrics = lists:append([ ?BYTES_METRICS, ?PACKET_METRICS, diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index 7044d992e..ccf3e0c40 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -120,7 +120,7 @@ inject_sni_fun(ListenerID, Conf0) -> init(_Args) -> logger:set_process_metadata(#{domain => [emqx, ocsp, cache]}), - emqx_tables:new(?CACHE_TAB, [ + emqx_utils_ets:new(?CACHE_TAB, [ named_table, public, {heir, whereis(emqx_kernel_sup), none}, diff --git a/apps/emqx/src/emqx_sequence.erl b/apps/emqx/src/emqx_sequence.erl index 60596324a..7acc87256 100644 --- a/apps/emqx/src/emqx_sequence.erl +++ b/apps/emqx/src/emqx_sequence.erl @@ -39,7 +39,7 @@ %% @doc Create a sequence. -spec create(name()) -> ok. create(Name) -> - emqx_tables:new(Name, [public, set, {write_concurrency, true}]). + emqx_utils_ets:new(Name, [public, set, {write_concurrency, true}]). %% @doc Next value of the sequence. -spec nextval(name(), key()) -> seqid(). diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl index 8a21d4d03..0435ddca3 100644 --- a/apps/emqx/src/emqx_session_router.erl +++ b/apps/emqx/src/emqx_session_router.erl @@ -95,7 +95,7 @@ create_table(Tab, Storage) -> %%-------------------------------------------------------------------- create_init_tab() -> - emqx_tables:new(?SESSION_INIT_TAB, [ + emqx_utils_ets:new(?SESSION_INIT_TAB, [ public, {read_concurrency, true}, {write_concurrency, true} @@ -182,7 +182,7 @@ pending(SessionID, MarkerIDs) -> call(pick(SessionID), {pending, SessionID, MarkerIDs}). buffer(SessionID, STopic, Msg) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> ok; Worker -> emqx_session_router_worker:buffer(Worker, STopic, Msg) end. @@ -194,7 +194,7 @@ resume_begin(From, SessionID) when is_pid(From), is_binary(SessionID) -> -spec resume_end(pid(), binary()) -> {'ok', [emqx_types:message()]} | {'error', term()}. resume_end(From, SessionID) when is_pid(From), is_binary(SessionID) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> ?tp(ps_session_not_found, #{sid => SessionID}), {error, not_found}; @@ -249,7 +249,7 @@ handle_cast({delete_routes, SessionID, Subscriptions}, State) -> ok = lists:foreach(Fun, maps:to_list(Subscriptions)), {noreply, State}; handle_cast({resume_end, SessionID, Pid}, State) -> - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> skip; P when P =:= Pid -> ets:delete(?SESSION_INIT_TAB, SessionID); P when is_pid(P) -> skip @@ -283,7 +283,7 @@ init_resume_worker(RemotePid, SessionID, #{pmon := Pmon} = State) -> error; {ok, Pid} -> Pmon1 = emqx_pmon:monitor(Pid, Pmon), - case emqx_tables:lookup_value(?SESSION_INIT_TAB, SessionID) of + case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of undefined -> {ok, Pid, State#{pmon => Pmon1}}; {_, OldPid} -> diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 061f2a42f..d7dc8c5a6 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -399,9 +399,11 @@ init([]) -> ok = mria:wait_for_tables([?TAB]), {ok, _} = mnesia:subscribe({table, ?TAB, simple}), {atomic, PMon} = mria:transaction(?SHARED_SUB_SHARD, fun ?MODULE:init_monitors/0), - ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), - ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), - ok = emqx_tables:new(?SHARED_SUBS_ROUND_ROBIN_COUNTER, [public, set, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?SHARED_SUBS, [protected, bag]), + ok = emqx_utils_ets:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), + ok = emqx_utils_ets:new(?SHARED_SUBS_ROUND_ROBIN_COUNTER, [ + public, set, {write_concurrency, true} + ]), {ok, update_stats(#state{pmon = PMon})}. init_monitors() -> diff --git a/apps/emqx/src/emqx_stats.erl b/apps/emqx/src/emqx_stats.erl index 017b83116..ef9109e33 100644 --- a/apps/emqx/src/emqx_stats.erl +++ b/apps/emqx/src/emqx_stats.erl @@ -201,7 +201,7 @@ cast(Msg) -> gen_server:cast(?SERVER, Msg). %%-------------------------------------------------------------------- init(#{tick_ms := TickMs}) -> - ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [public, set, {write_concurrency, true}]), Stats = lists:append([ ?CONNECTION_STATS, ?CHANNEL_STATS, diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 5de6da63f..837600811 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -766,9 +766,9 @@ init(Options) -> TabOpts = [public, {write_concurrency, true}], {ChanTab, ConnTab, InfoTab} = cmtabs(GwName), - ok = emqx_tables:new(ChanTab, [bag, {read_concurrency, true} | TabOpts]), - ok = emqx_tables:new(ConnTab, [bag | TabOpts]), - ok = emqx_tables:new(InfoTab, [ordered_set, compressed | TabOpts]), + ok = emqx_utils_ets:new(ChanTab, [bag, {read_concurrency, true} | TabOpts]), + ok = emqx_utils_ets:new(ConnTab, [bag | TabOpts]), + ok = emqx_utils_ets:new(InfoTab, [ordered_set, compressed | TabOpts]), %% Start link cm-registry process %% XXX: Should I hang it under a higher level supervisor? diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl index e94510387..0aa2ff210 100644 --- a/apps/emqx_gateway/src/emqx_gateway_metrics.erl +++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl @@ -89,7 +89,7 @@ tabname(GwName) -> init([GwName]) -> TabOpts = [public, {write_concurrency, true}], - ok = emqx_tables:new(tabname(GwName), [set | TabOpts]), + ok = emqx_utils_ets:new(tabname(GwName), [set | TabOpts]), {ok, #state{}}. handle_call(_Request, _From, State) -> diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl index dfc6b07ab..de09e568f 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics.erl @@ -201,7 +201,7 @@ reset() -> init([Opts]) -> erlang:process_flag(trap_exit, true), - ok = emqx_tables:new(?TAB, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB, [{read_concurrency, true}]), erlang:send_after(timer:seconds(?TICKING_INTERVAL), self(), ticking), Fun = fun(#{topic := Topic}, CurrentSpeeds) -> diff --git a/apps/emqx/src/emqx_tables.erl b/apps/emqx_utils/src/emqx_utils_ets.erl similarity index 98% rename from apps/emqx/src/emqx_tables.erl rename to apps/emqx_utils/src/emqx_utils_ets.erl index ffdf7d891..099152675 100644 --- a/apps/emqx/src/emqx_tables.erl +++ b/apps/emqx_utils/src/emqx_utils_ets.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_tables). +-module(emqx_utils_ets). -export([ new/1, diff --git a/apps/emqx/test/emqx_tables_SUITE.erl b/apps/emqx_utils/test/emqx_utils_ets_SUITE.erl similarity index 73% rename from apps/emqx/test/emqx_tables_SUITE.erl rename to apps/emqx_utils/test/emqx_utils_ets_SUITE.erl index ad53e7139..13bf427fd 100644 --- a/apps/emqx/test/emqx_tables_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_ets_SUITE.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_tables_SUITE). +-module(emqx_utils_ets_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -26,19 +26,19 @@ all() -> emqx_common_test_helpers:all(?MODULE). t_new(_) -> - ok = emqx_tables:new(?TAB), - ok = emqx_tables:new(?TAB, [{read_concurrency, true}]), + ok = emqx_utils_ets:new(?TAB), + ok = emqx_utils_ets:new(?TAB, [{read_concurrency, true}]), ?assertEqual(?TAB, ets:info(?TAB, name)). t_lookup_value(_) -> - ok = emqx_tables:new(?TAB, []), + ok = emqx_utils_ets:new(?TAB, []), true = ets:insert(?TAB, {key, val}), - ?assertEqual(val, emqx_tables:lookup_value(?TAB, key)), - ?assertEqual(undefined, emqx_tables:lookup_value(?TAB, badkey)). + ?assertEqual(val, emqx_utils_ets:lookup_value(?TAB, key)), + ?assertEqual(undefined, emqx_utils_ets:lookup_value(?TAB, badkey)). t_delete(_) -> - ok = emqx_tables:new(?TAB, []), + ok = emqx_utils_ets:new(?TAB, []), ?assertEqual(?TAB, ets:info(?TAB, name)), - ok = emqx_tables:delete(?TAB), - ok = emqx_tables:delete(?TAB), + ok = emqx_utils_ets:delete(?TAB), + ok = emqx_utils_ets:delete(?TAB), ?assertEqual(undefined, ets:info(?TAB, name)). diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 43ccd9fc1..325d3d499 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -68,7 +68,7 @@ trace_rule(Data, Envs, _Args) -> make_trace_fn_action() -> persistent_term:put({?MODULE, test_pid}, self()), Fn = <<(atom_to_binary(?MODULE))/binary, ":trace_rule">>, - emqx_tables:new(recorded_actions, [named_table, public, ordered_set]), + emqx_utils_ets:new(recorded_actions, [named_table, public, ordered_set]), #{function => Fn, args => #{}}. create_rule_http(RuleParams) -> From 062ce5f81991cbd0103a081815bd67c64ceed452 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 15:02:41 +0200 Subject: [PATCH 117/279] refactor: rename emqx_map_lib to emqx_utils_maps --- apps/emqx/src/emqx.erl | 18 +++--- apps/emqx/src/emqx_config.erl | 64 +++++++++---------- apps/emqx/src/emqx_config_handler.erl | 18 +++--- .../emqx_limiter/src/emqx_limiter_server.erl | 2 +- apps/emqx/src/emqx_listeners.erl | 6 +- apps/emqx/src/emqx_ocsp_cache.erl | 4 +- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx/src/emqx_tls_lib.erl | 20 +++--- apps/emqx/test/emqx_crl_cache_SUITE.erl | 4 +- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 22 ++++--- apps/emqx/test/emqx_tls_lib_tests.erl | 6 +- apps/emqx_authn/src/emqx_authn_api.erl | 4 +- .../emqx_authz/src/emqx_authz_api_sources.erl | 2 +- apps/emqx_authz/src/emqx_authz_schema.erl | 2 +- apps/emqx_bridge/src/emqx_bridge.erl | 4 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 2 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 4 +- .../src/schema/emqx_bridge_schema.erl | 2 +- .../test/emqx_bridge_api_SUITE.erl | 2 +- .../test/emqx_bridge_mqtt_SUITE.erl | 2 +- .../emqx_bridge_kafka_impl_consumer_SUITE.erl | 6 +- .../test/emqx_bridge_kafka_tests.erl | 4 +- apps/emqx_conf/src/emqx_conf.erl | 28 ++++---- .../src/proto/emqx_conf_proto_v1.erl | 8 +-- .../src/proto/emqx_conf_proto_v2.erl | 8 +-- .../emqx_connector/src/emqx_connector_ssl.erl | 2 +- .../src/mqtt/emqx_connector_mqtt_worker.erl | 2 +- .../src/emqx_dashboard_listener.erl | 11 ++-- .../src/emqx_dashboard_swagger.erl | 2 +- apps/emqx_exhook/src/emqx_exhook_mgr.erl | 8 +-- .../test/emqx_exhook_api_SUITE.erl | 2 +- .../src/emqx_gateway_api_listeners.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_conf.erl | 60 ++++++++--------- apps/emqx_gateway/src/emqx_gateway_http.erl | 4 +- .../test/emqx_gateway_api_SUITE.erl | 12 ++-- .../test/emqx_gateway_authn_SUITE.erl | 2 +- .../test/emqx_gateway_conf_SUITE.erl | 6 +- .../test/emqx_gateway_test_utils.erl | 2 +- .../src/emqx_gateway_exproto.erl | 4 +- .../src/emqx_mgmt_api_clients.erl | 6 +- .../src/emqx_mgmt_api_configs.erl | 4 +- .../src/emqx_mgmt_api_listeners.erl | 2 +- .../test/emqx_mgmt_api_configs_SUITE.erl | 38 ++++++----- .../test/emqx_mgmt_api_listeners_SUITE.erl | 6 +- .../test/emqx_mgmt_api_nodes_SUITE.erl | 6 +- .../test/emqx_delayed_api_SUITE.erl | 2 +- .../src/emqx_retainer_dispatcher.erl | 2 +- .../test/emqx_retainer_api_SUITE.erl | 4 +- .../emqx_rule_engine/src/emqx_rule_engine.erl | 2 +- .../test/emqx_slow_subs_api_SUITE.erl | 2 +- .../src/emqx_utils_maps.erl} | 4 +- .../test/emqx_utils_maps_tests.erl} | 32 +++++----- .../test/emqx_ee_bridge_cassa_SUITE.erl | 2 +- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 2 +- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 4 +- .../test/emqx_ee_bridge_influxdb_SUITE.erl | 4 +- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 2 +- .../test/emqx_ee_bridge_pgsql_SUITE.erl | 2 +- .../test/emqx_ee_bridge_tdengine_SUITE.erl | 2 +- .../src/emqx_ee_schema_registry.erl | 4 +- .../test/emqx_ee_schema_registry_SUITE.erl | 2 +- 61 files changed, 256 insertions(+), 241 deletions(-) rename apps/{emqx/src/emqx_map_lib.erl => emqx_utils/src/emqx_utils_maps.erl} (99%) rename apps/{emqx/test/emqx_map_lib_tests.erl => emqx_utils/test/emqx_utils_maps_tests.erl} (76%) diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 6e4aa9922..ef870685a 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -164,29 +164,29 @@ run_hook(HookPoint, Args) -> run_fold_hook(HookPoint, Args, Acc) -> emqx_hooks:run_fold(HookPoint, Args, Acc). --spec get_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_config(emqx_utils_maps:config_key_path()) -> term(). get_config(KeyPath) -> emqx_config:get(KeyPath). --spec get_config(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_config(KeyPath, Default) -> emqx_config:get(KeyPath, Default). --spec get_raw_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw_config(emqx_utils_maps:config_key_path()) -> term(). get_raw_config(KeyPath) -> emqx_config:get_raw(KeyPath). --spec get_raw_config(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw_config(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw_config(KeyPath, Default) -> emqx_config:get_raw(KeyPath, Default). --spec update_config(emqx_map_lib:config_key_path(), emqx_config:update_request()) -> +-spec update_config(emqx_utils_maps:config_key_path(), emqx_config:update_request()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_config(KeyPath, UpdateReq) -> update_config(KeyPath, UpdateReq, #{}). -spec update_config( - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -198,12 +198,12 @@ update_config([RootName | _] = KeyPath, UpdateReq, Opts) -> {{update, UpdateReq}, Opts} ). --spec remove_config(emqx_map_lib:config_key_path()) -> +-spec remove_config(emqx_utils_maps:config_key_path()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove_config(KeyPath) -> remove_config(KeyPath, #{}). --spec remove_config(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove_config([RootName | _] = KeyPath, Opts) -> emqx_config_handler:update_config( @@ -212,7 +212,7 @@ remove_config([RootName | _] = KeyPath, Opts) -> {remove, Opts} ). --spec reset_config(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset_config(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset_config([RootName | _] = KeyPath, Opts) -> case emqx_config:get_default_value(KeyPath) of diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index bf3134568..b7effa240 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -142,7 +142,7 @@ -type app_envs() :: [proplists:property()]. %% @doc For the given path, get root value enclosed in a single-key map. --spec get_root(emqx_map_lib:config_key_path()) -> map(). +-spec get_root(emqx_utils_maps:config_key_path()) -> map(). get_root([RootName | _]) -> #{RootName => do_get(?CONF, [RootName], #{})}. @@ -153,14 +153,14 @@ get_root_raw([RootName | _]) -> %% @doc Get a config value for the given path. %% The path should at least include root config name. --spec get(emqx_map_lib:config_key_path()) -> term(). +-spec get(emqx_utils_maps:config_key_path()) -> term(). get(KeyPath) -> do_get(?CONF, KeyPath). --spec get(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get(emqx_utils_maps:config_key_path(), term()) -> term(). get(KeyPath, Default) -> do_get(?CONF, KeyPath, Default). --spec find(emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find(emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find([]) -> Ref = make_ref(), case do_get(?CONF, [], Ref) of @@ -170,12 +170,12 @@ find([]) -> find(KeyPath) -> atom_conf_path( KeyPath, - fun(AtomKeyPath) -> emqx_map_lib:deep_find(AtomKeyPath, get_root(KeyPath)) end, + fun(AtomKeyPath) -> emqx_utils_maps:deep_find(AtomKeyPath, get_root(KeyPath)) end, {return, {not_found, KeyPath}} ). --spec find_raw(emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find_raw(emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find_raw([]) -> Ref = make_ref(), case do_get_raw([], Ref) of @@ -183,9 +183,9 @@ find_raw([]) -> Res -> {ok, Res} end; find_raw(KeyPath) -> - emqx_map_lib:deep_find([bin(Key) || Key <- KeyPath], get_root_raw(KeyPath)). + emqx_utils_maps:deep_find([bin(Key) || Key <- KeyPath], get_root_raw(KeyPath)). --spec get_zone_conf(atom(), emqx_map_lib:config_key_path()) -> term(). +-spec get_zone_conf(atom(), emqx_utils_maps:config_key_path()) -> term(). get_zone_conf(Zone, KeyPath) -> case find(?ZONE_CONF_PATH(Zone, KeyPath)) of %% not found in zones, try to find the global config @@ -195,7 +195,7 @@ get_zone_conf(Zone, KeyPath) -> Value end. --spec get_zone_conf(atom(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_zone_conf(atom(), emqx_utils_maps:config_key_path(), term()) -> term(). get_zone_conf(Zone, KeyPath, Default) -> case find(?ZONE_CONF_PATH(Zone, KeyPath)) of %% not found in zones, try to find the global config @@ -205,24 +205,24 @@ get_zone_conf(Zone, KeyPath, Default) -> Value end. --spec put_zone_conf(atom(), emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_zone_conf(atom(), emqx_utils_maps:config_key_path(), term()) -> ok. put_zone_conf(Zone, KeyPath, Conf) -> ?MODULE:put(?ZONE_CONF_PATH(Zone, KeyPath), Conf). --spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> term(). +-spec get_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path()) -> term(). get_listener_conf(Type, Listener, KeyPath) -> ?MODULE:get(?LISTENER_CONF_PATH(Type, Listener, KeyPath)). --spec get_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path(), term()) -> term(). get_listener_conf(Type, Listener, KeyPath, Default) -> ?MODULE:get(?LISTENER_CONF_PATH(Type, Listener, KeyPath), Default). --spec put_listener_conf(atom(), atom(), emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path(), term()) -> ok. put_listener_conf(Type, Listener, KeyPath, Conf) -> ?MODULE:put(?LISTENER_CONF_PATH(Type, Listener, KeyPath), Conf). --spec find_listener_conf(atom(), atom(), emqx_map_lib:config_key_path()) -> - {ok, term()} | {not_found, emqx_map_lib:config_key_path(), term()}. +-spec find_listener_conf(atom(), atom(), emqx_utils_maps:config_key_path()) -> + {ok, term()} | {not_found, emqx_utils_maps:config_key_path(), term()}. find_listener_conf(Type, Listener, KeyPath) -> find(?LISTENER_CONF_PATH(Type, Listener, KeyPath)). @@ -241,20 +241,20 @@ erase(RootName) -> persistent_term:erase(?PERSIS_KEY(?RAW_CONF, bin(RootName))), ok. --spec put(emqx_map_lib:config_key_path(), term()) -> ok. +-spec put(emqx_utils_maps:config_key_path(), term()) -> ok. put(KeyPath, Config) -> Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_put(Path, Map, Value) + emqx_utils_maps:deep_put(Path, Map, Value) end, do_put(?CONF, Putter, KeyPath, Config). %% Puts value into configuration even if path doesn't exist %% For paths of non-existing atoms use force_put(KeyPath, Config, unsafe) --spec force_put(emqx_map_lib:config_key_path(), term()) -> ok. +-spec force_put(emqx_utils_maps:config_key_path(), term()) -> ok. force_put(KeyPath, Config) -> force_put(KeyPath, Config, safe). --spec force_put(emqx_map_lib:config_key_path(), term(), safe | unsafe) -> ok. +-spec force_put(emqx_utils_maps:config_key_path(), term(), safe | unsafe) -> ok. force_put(KeyPath0, Config, Safety) -> KeyPath = case Safety of @@ -262,19 +262,19 @@ force_put(KeyPath0, Config, Safety) -> unsafe -> [unsafe_atom(Key) || Key <- KeyPath0] end, Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_force_put(Path, Map, Value) + emqx_utils_maps:deep_force_put(Path, Map, Value) end, do_put(?CONF, Putter, KeyPath, Config). --spec get_default_value(emqx_map_lib:config_key_path()) -> {ok, term()} | {error, term()}. +-spec get_default_value(emqx_utils_maps:config_key_path()) -> {ok, term()} | {error, term()}. get_default_value([RootName | _] = KeyPath) -> BinKeyPath = [bin(Key) || Key <- KeyPath], case find_raw([RootName]) of {ok, RawConf} -> - RawConf1 = emqx_map_lib:deep_remove(BinKeyPath, #{bin(RootName) => RawConf}), + RawConf1 = emqx_utils_maps:deep_remove(BinKeyPath, #{bin(RootName) => RawConf}), try fill_defaults(get_schema_mod(RootName), RawConf1, #{}) of FullConf -> - case emqx_map_lib:deep_find(BinKeyPath, FullConf) of + case emqx_utils_maps:deep_find(BinKeyPath, FullConf) of {not_found, _, _} -> {error, no_default_value}; {ok, Val} -> {ok, Val} end @@ -285,10 +285,10 @@ get_default_value([RootName | _] = KeyPath) -> {error, {rootname_not_found, RootName}} end. --spec get_raw(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path()) -> term(). get_raw(KeyPath) -> do_get_raw(KeyPath). --spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw(KeyPath, Default) -> do_get_raw(KeyPath, Default). -spec put_raw(map()) -> ok. @@ -301,10 +301,10 @@ put_raw(Config) -> hocon_maps:ensure_plain(Config) ). --spec put_raw(emqx_map_lib:config_key_path(), term()) -> ok. +-spec put_raw(emqx_utils_maps:config_key_path(), term()) -> ok. put_raw(KeyPath, Config) -> Putter = fun(Path, Map, Value) -> - emqx_map_lib:deep_force_put(Path, Map, Value) + emqx_utils_maps:deep_force_put(Path, Map, Value) end, do_put(?RAW_CONF, Putter, KeyPath, Config). @@ -430,7 +430,7 @@ do_check_config(SchemaMod, RawConf, Opts0) -> Opts = maps:merge(Opts0, Opts1), {AppEnvs, CheckedConf} = hocon_tconf:map_translate(SchemaMod, RawConf, Opts), - {AppEnvs, emqx_map_lib:unsafe_atom_key_map(CheckedConf)}. + {AppEnvs, emqx_utils_maps:unsafe_atom_key_map(CheckedConf)}. fill_defaults(RawConf) -> fill_defaults(RawConf, #{}). @@ -645,11 +645,11 @@ do_put(Type, Putter, [RootName | KeyPath], DeepValue) -> do_deep_get(?CONF, KeyPath, Map, Default) -> atom_conf_path( KeyPath, - fun(AtomKeyPath) -> emqx_map_lib:deep_get(AtomKeyPath, Map, Default) end, + fun(AtomKeyPath) -> emqx_utils_maps:deep_get(AtomKeyPath, Map, Default) end, {return, Default} ); do_deep_get(?RAW_CONF, KeyPath, Map, Default) -> - emqx_map_lib:deep_get([bin(Key) || Key <- KeyPath], Map, Default). + emqx_utils_maps:deep_get([bin(Key) || Key <- KeyPath], Map, Default). do_deep_put(?CONF, Putter, KeyPath, Map, Value) -> atom_conf_path( diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index a0a99b62e..b440d2af7 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -232,7 +232,7 @@ process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) -> BinKeyPath = bin_path(ConfKeyPath), case check_permissions(remove, BinKeyPath, OldRawConf, Opts) of allow -> - NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), + NewRawConf = emqx_utils_maps:deep_remove(BinKeyPath, OldRawConf), OverrideConf = remove_from_override_config(BinKeyPath, Opts), {ok, NewRawConf, OverrideConf, Opts}; {deny, Reason} -> @@ -445,7 +445,7 @@ remove_from_override_config(_BinKeyPath, #{persistent := false}) -> undefined; remove_from_override_config(BinKeyPath, Opts) -> OldConf = emqx_config:read_override_conf(Opts), - emqx_map_lib:deep_remove(BinKeyPath, OldConf). + emqx_utils_maps:deep_remove(BinKeyPath, OldConf). %% apply new config on top of override config merge_to_override_config(_RawConf, #{persistent := false}) -> @@ -467,7 +467,7 @@ return_change_result(_ConfKeyPath, {remove, _Opts}) -> return_rawconf(ConfKeyPath, #{rawconf_with_defaults := true}) -> FullRawConf = emqx_config:fill_defaults(emqx_config:get_raw([])), - emqx_map_lib:deep_get(bin_path(ConfKeyPath), FullRawConf); + emqx_utils_maps:deep_get(bin_path(ConfKeyPath), FullRawConf); return_rawconf(ConfKeyPath, _) -> emqx_config:get_raw(ConfKeyPath). @@ -485,16 +485,16 @@ atom(Atom) when is_atom(Atom) -> -dialyzer({nowarn_function, do_remove_handler/2}). do_remove_handler(ConfKeyPath, Handlers) -> - NewHandlers = emqx_map_lib:deep_remove(ConfKeyPath ++ [?MOD], Handlers), + NewHandlers = emqx_utils_maps:deep_remove(ConfKeyPath ++ [?MOD], Handlers), remove_empty_leaf(ConfKeyPath, NewHandlers). remove_empty_leaf([], Handlers) -> Handlers; remove_empty_leaf(KeyPath, Handlers) -> - case emqx_map_lib:deep_find(KeyPath, Handlers) =:= {ok, #{}} of + case emqx_utils_maps:deep_find(KeyPath, Handlers) =:= {ok, #{}} of %% empty leaf true -> - Handlers1 = emqx_map_lib:deep_remove(KeyPath, Handlers), + Handlers1 = emqx_utils_maps:deep_remove(KeyPath, Handlers), SubKeyPath = lists:sublist(KeyPath, length(KeyPath) - 1), remove_empty_leaf(SubKeyPath, Handlers1); false -> @@ -511,7 +511,7 @@ assert_callback_function(Mod) -> end, ok. --spec schema(module(), emqx_map_lib:config_key_path()) -> hocon_schema:schema(). +-spec schema(module(), emqx_utils_maps:config_key_path()) -> hocon_schema:schema(). schema(SchemaModule, [RootKey | _]) -> Roots = hocon_schema:roots(SchemaModule), {Field, Translations} = @@ -550,10 +550,10 @@ save_handlers(Handlers) -> check_permissions(_Action, _ConfKeyPath, _NewRawConf, #{override_to := local}) -> allow; check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) -> - case emqx_map_lib:deep_find(ConfKeyPath, NewRawConf) of + case emqx_utils_maps:deep_find(ConfKeyPath, NewRawConf) of {ok, NewRaw} -> LocalOverride = emqx_config:read_override_conf(#{override_to => local}), - case emqx_map_lib:deep_find(ConfKeyPath, LocalOverride) of + case emqx_utils_maps:deep_find(ConfKeyPath, LocalOverride) of {ok, LocalRaw} -> case is_mutable(Action, NewRaw, LocalRaw) of ok -> diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index 44663ceeb..f1daeaaeb 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -572,7 +572,7 @@ find_limiter_cfg(Type, #{rate := _} = Cfg) -> find_limiter_cfg(Type, Cfg) -> { maps:get(Type, Cfg, undefined), - find_client_cfg(Type, emqx_map_lib:deep_get([client, Type], Cfg, undefined)) + find_client_cfg(Type, emqx_utils_maps:deep_get([client, Type], Cfg, undefined)) }. find_client_cfg(Type, BucketCfg) -> diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 96e28b1de..f82aebe7c 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -427,12 +427,12 @@ pre_config_update([listeners, _Type, _Name], {create, _NewConf}, _RawConf) -> pre_config_update([listeners, _Type, _Name], {update, _Request}, undefined) -> {error, not_found}; pre_config_update([listeners, Type, Name], {update, Request}, RawConf) -> - NewConfT = emqx_map_lib:deep_merge(RawConf, Request), + NewConfT = emqx_utils_maps:deep_merge(RawConf, Request), NewConf = ensure_override_limiter_conf(NewConfT, Request), CertsDir = certs_dir(Type, Name), {ok, convert_certs(CertsDir, NewConf)}; pre_config_update([listeners, _Type, _Name], {action, _Action, Updated}, RawConf) -> - NewConf = emqx_map_lib:deep_merge(RawConf, Updated), + NewConf = emqx_utils_maps:deep_merge(RawConf, Updated), {ok, NewConf}; pre_config_update(_Path, _Request, RawConf) -> {ok, RawConf}. @@ -500,7 +500,7 @@ esockd_opts(ListenerId, Type, Opts0) -> ws_opts(Type, ListenerName, Opts) -> WsPaths = [ - {emqx_map_lib:deep_get([websocket, mqtt_path], Opts, "/mqtt"), emqx_ws_connection, #{ + {emqx_utils_maps:deep_get([websocket, mqtt_path], Opts, "/mqtt"), emqx_ws_connection, #{ zone => zone(Opts), listener => {Type, ListenerName}, limiter => limiter(Opts), diff --git a/apps/emqx/src/emqx_ocsp_cache.erl b/apps/emqx/src/emqx_ocsp_cache.erl index ccf3e0c40..3bb10ee5c 100644 --- a/apps/emqx/src/emqx_ocsp_cache.erl +++ b/apps/emqx/src/emqx_ocsp_cache.erl @@ -110,7 +110,7 @@ register_listener(ListenerID, Opts) -> -spec inject_sni_fun(emqx_listeners:listener_id(), map()) -> map(). inject_sni_fun(ListenerID, Conf0) -> SNIFun = emqx_const_v1:make_sni_fun(ListenerID), - Conf = emqx_map_lib:deep_merge(Conf0, #{ssl_options => #{sni_fun => SNIFun}}), + Conf = emqx_utils_maps:deep_merge(Conf0, #{ssl_options => #{sni_fun => SNIFun}}), ok = ?MODULE:register_listener(ListenerID, Conf), Conf. @@ -149,7 +149,7 @@ handle_call({register_listener, ListenerID, Conf}, _From, State0) -> msg => "registering_ocsp_cache", listener_id => ListenerID }), - RefreshInterval0 = emqx_map_lib:deep_get([ssl_options, ocsp, refresh_interval], Conf), + RefreshInterval0 = emqx_utils_maps:deep_get([ssl_options, ocsp, refresh_interval], Conf), RefreshInterval = max(RefreshInterval0, ?MIN_REFRESH_INTERVAL), State = State0#{{refresh_interval, ListenerID} => RefreshInterval}, %% we need to pass the config along because this might be called diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 96d9aea34..9934e09c9 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2348,7 +2348,7 @@ ocsp_outer_validator(_Conf) -> ok. ocsp_inner_validator(#{enable_ocsp_stapling := _} = Conf) -> - ocsp_inner_validator(emqx_map_lib:binary_key_map(Conf)); + ocsp_inner_validator(emqx_utils_maps:binary_key_map(Conf)); ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := false} = _Conf) -> ok; ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := true} = Conf) -> diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 47797b326..d1c57bf0d 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -317,7 +317,9 @@ ensure_ssl_files(Dir, SSL, Opts) -> ensure_ssl_files(_Dir, SSL, [], _Opts) -> {ok, SSL}; ensure_ssl_files(Dir, SSL, [KeyPath | KeyPaths], Opts) -> - case ensure_ssl_file(Dir, KeyPath, SSL, emqx_map_lib:deep_get(KeyPath, SSL, undefined), Opts) of + case + ensure_ssl_file(Dir, KeyPath, SSL, emqx_utils_maps:deep_get(KeyPath, SSL, undefined), Opts) + of {ok, NewSSL} -> ensure_ssl_files(Dir, NewSSL, KeyPaths, Opts); {error, Reason} -> @@ -332,7 +334,7 @@ delete_ssl_files(Dir, NewOpts0, OldOpts0) -> {ok, OldOpts} = ensure_ssl_files(Dir, OldOpts0, #{dry_run => DryRun}), Get = fun (_KP, undefined) -> undefined; - (KP, Opts) -> emqx_map_lib:deep_get(KP, Opts, undefined) + (KP, Opts) -> emqx_utils_maps:deep_get(KP, Opts, undefined) end, lists:foreach( fun(KeyPath) -> delete_old_file(Get(KeyPath, NewOpts), Get(KeyPath, OldOpts)) end, @@ -372,7 +374,7 @@ do_ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, DryRun) -> true -> case save_pem_file(Dir, KeyPath, MaybePem, DryRun) of {ok, Path} -> - NewSSL = emqx_map_lib:deep_put(KeyPath, SSL, Path), + NewSSL = emqx_utils_maps:deep_put(KeyPath, SSL, Path), {ok, NewSSL}; {error, Reason} -> {error, Reason} @@ -482,9 +484,9 @@ is_valid_pem_file(Path) -> %% so they are forced to upload a cert file, or use an existing file path. -spec drop_invalid_certs(map()) -> map(). drop_invalid_certs(#{enable := False} = SSL) when ?IS_FALSE(False) -> - lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS_A); + lists:foldl(fun emqx_utils_maps:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS_A); drop_invalid_certs(#{<<"enable">> := False} = SSL) when ?IS_FALSE(False) -> - lists:foldl(fun emqx_map_lib:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS); + lists:foldl(fun emqx_utils_maps:deep_remove/2, SSL, ?SSL_FILE_OPT_PATHS); drop_invalid_certs(#{enable := True} = SSL) when ?IS_TRUE(True) -> do_drop_invalid_certs(?SSL_FILE_OPT_PATHS_A, SSL); drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> @@ -493,7 +495,7 @@ drop_invalid_certs(#{<<"enable">> := True} = SSL) when ?IS_TRUE(True) -> do_drop_invalid_certs([], SSL) -> SSL; do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> - case emqx_map_lib:deep_get(KeyPath, SSL, undefined) of + case emqx_utils_maps:deep_get(KeyPath, SSL, undefined) of undefined -> do_drop_invalid_certs(KeyPaths, SSL); PemOrPath -> @@ -501,7 +503,7 @@ do_drop_invalid_certs([KeyPath | KeyPaths], SSL) -> true -> do_drop_invalid_certs(KeyPaths, SSL); {error, _} -> - do_drop_invalid_certs(KeyPaths, emqx_map_lib:deep_remove(KeyPath, SSL)) + do_drop_invalid_certs(KeyPaths, emqx_utils_maps:deep_remove(KeyPath, SSL)) end end. @@ -586,7 +588,9 @@ ensure_ssl_file_key(_SSL, []) -> ok; ensure_ssl_file_key(SSL, RequiredKeyPaths) -> NotFoundRef = make_ref(), - Filter = fun(KeyPath) -> NotFoundRef =:= emqx_map_lib:deep_get(KeyPath, SSL, NotFoundRef) end, + Filter = fun(KeyPath) -> + NotFoundRef =:= emqx_utils_maps:deep_get(KeyPath, SSL, NotFoundRef) + end, case lists:filter(Filter, RequiredKeyPaths) of [] -> ok; Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} diff --git a/apps/emqx/test/emqx_crl_cache_SUITE.erl b/apps/emqx/test/emqx_crl_cache_SUITE.erl index 21bfc39df..dd3eb29e7 100644 --- a/apps/emqx/test/emqx_crl_cache_SUITE.erl +++ b/apps/emqx/test/emqx_crl_cache_SUITE.erl @@ -997,7 +997,7 @@ do_t_update_listener(Config) -> <<"enable_crl_check">> => true } }, - ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, CRLConfig), + ListenerData1 = emqx_utils_maps:deep_merge(ListenerData0, CRLConfig), {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1), ?assertMatch( #{ @@ -1040,7 +1040,7 @@ do_t_validations(_Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ListenerData1 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index a9b80c276..dff8ce5a7 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -143,7 +143,7 @@ init_per_testcase(t_ocsp_responder_error_responses, Config) -> } }, Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_map_lib:binary_key_map(Conf), + ConfBin = emqx_utils_maps:binary_key_map(Conf), hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), snabbkaffe:start_trace(), @@ -184,7 +184,7 @@ init_per_testcase(_TestCase, Config) -> } }, Conf = #{listeners => #{Type => #{Name => ListenerOpts}}}, - ConfBin = emqx_map_lib:binary_key_map(Conf), + ConfBin = emqx_utils_maps:binary_key_map(Conf), hocon_tconf:check_plain(emqx_schema, ConfBin, #{required => false, atom_keys => false}), emqx_config:put_listener_conf(Type, Name, [], ListenerOpts), snabbkaffe:start_trace(), @@ -679,7 +679,7 @@ do_t_update_listener(Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ?assertEqual( undefined, - emqx_map_lib:deep_get([<<"ssl_options">>, <<"ocsp">>], ListenerData0, undefined) + emqx_utils_maps:deep_get([<<"ssl_options">>, <<"ocsp">>], ListenerData0, undefined) ), assert_no_http_get(), @@ -702,7 +702,7 @@ do_t_update_listener(Config) -> } } }, - ListenerData1 = emqx_map_lib:deep_merge(ListenerData0, OCSPConfig), + ListenerData1 = emqx_utils_maps:deep_merge(ListenerData0, OCSPConfig), {ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1), ?assertMatch( #{ @@ -722,14 +722,14 @@ do_t_update_listener(Config) -> %% location ?assertNotEqual( IssuerPemPath, - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [<<"ssl_options">>, <<"ocsp">>, <<"issuer_pem">>], ListenerData2 ) ), ?assertNotEqual( IssuerPem, - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [<<"ssl_options">>, <<"ocsp">>, <<"issuer_pem">>], ListenerData2 ) @@ -818,7 +818,7 @@ do_t_validations(_Config) -> {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), ListenerData1 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -843,7 +843,7 @@ do_t_validations(_Config) -> ), ListenerData2 = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -873,7 +873,7 @@ do_t_validations(_Config) -> ), ListenerData3a = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ListenerData0, #{ <<"ssl_options">> => @@ -886,7 +886,9 @@ do_t_validations(_Config) -> } } ), - ListenerData3 = emqx_map_lib:deep_remove([<<"ssl_options">>, <<"certfile">>], ListenerData3a), + ListenerData3 = emqx_utils_maps:deep_remove( + [<<"ssl_options">>, <<"certfile">>], ListenerData3a + ), {error, {_, _, ResRaw3}} = update_listener_via_api(ListenerId, ListenerData3), #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := MsgRaw3} = emqx_utils_json:decode(ResRaw3, [return_maps]), diff --git a/apps/emqx/test/emqx_tls_lib_tests.erl b/apps/emqx/test/emqx_tls_lib_tests.erl index 5510e4027..ad9598107 100644 --- a/apps/emqx/test/emqx_tls_lib_tests.erl +++ b/apps/emqx/test/emqx_tls_lib_tests.erl @@ -191,7 +191,7 @@ ssl_files_save_delete_test() -> FileKey = maps:get(<<"keyfile">>, SSL), ?assertMatch(<<"/tmp/ssl-test-dir/key-", _:16/binary>>, FileKey), ?assertEqual({ok, bin(test_key())}, file:read_file(FileKey)), - FileIssuerPem = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL), + FileIssuerPem = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL), ?assertMatch(<<"/tmp/ssl-test-dir/ocsp_issuer_pem-", _:16/binary>>, FileIssuerPem), ?assertEqual({ok, bin(test_key())}, file:read_file(FileIssuerPem)), %% no old file to delete @@ -251,8 +251,8 @@ ssl_file_replace_test() -> {ok, SSL3} = emqx_tls_lib:ensure_ssl_files(Dir, SSL1), File1 = maps:get(<<"keyfile">>, SSL2), File2 = maps:get(<<"keyfile">>, SSL3), - IssuerPem1 = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2), - IssuerPem2 = emqx_map_lib:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL3), + IssuerPem1 = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2), + IssuerPem2 = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL3), ?assert(filelib:is_regular(File1)), ?assert(filelib:is_regular(File2)), ?assert(filelib:is_regular(IssuerPem1)), diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index fc026a671..de856f163 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -929,7 +929,7 @@ aggregate_metrics([]) -> aggregate_metrics([HeadMetrics | AllMetrics]) -> ErrorLogger = fun(Reason) -> ?SLOG(info, #{msg => "bad_metrics_value", error => Reason}) end, Fun = fun(ElemMap, AccMap) -> - emqx_map_lib:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) + emqx_utils_maps:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) end, lists:foldl(Fun, HeadMetrics, AllMetrics). @@ -1069,7 +1069,7 @@ update_user(ChainName, AuthenticatorID, UserID, UserInfo0) -> true -> serialize_error({missing_parameter, password}); false -> - UserInfo = emqx_map_lib:safe_atom_key_map(UserInfo0), + UserInfo = emqx_utils_maps:safe_atom_key_map(UserInfo0), case emqx_authentication:update_user(ChainName, AuthenticatorID, UserID, UserInfo) of {ok, User} -> {200, User}; diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 0c2dee340..2220e8f6e 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -403,7 +403,7 @@ aggregate_metrics([]) -> aggregate_metrics([HeadMetrics | AllMetrics]) -> ErrorLogger = fun(Reason) -> ?SLOG(info, #{msg => "bad_metrics_value", error => Reason}) end, Fun = fun(ElemMap, AccMap) -> - emqx_map_lib:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) + emqx_utils_maps:best_effort_recursive_sum(AccMap, ElemMap, ErrorLogger) end, lists:foldl(Fun, HeadMetrics, AllMetrics). diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 6630ed526..f03ae52a8 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -337,7 +337,7 @@ check_ssl_opts(Conf) -> (#{<<"url">> := Url} = Source) -> case emqx_authz_http:parse_url(Url) of {<<"https", _/binary>>, _, _} -> - case emqx_map_lib:deep_find([<<"ssl">>, <<"enable">>], Source) of + case emqx_utils_maps:deep_find([<<"ssl">>, <<"enable">>], Source) of {ok, true} -> true; {ok, false} -> throw({ssl_not_enable, Url}); _ -> throw({ssl_enable_not_found, Url}) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 087bc6a3f..08b8222f2 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -207,7 +207,7 @@ send_message(BridgeId, Message) -> end. query_opts(Config) -> - case emqx_map_lib:deep_get([resource_opts, request_timeout], Config, false) of + case emqx_utils_maps:deep_get([resource_opts, request_timeout], Config, false) of Timeout when is_integer(Timeout) -> %% request_timeout is configured #{timeout => Timeout}; @@ -367,7 +367,7 @@ perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) -> perform_bridge_changes(Tasks, Result). diff_confs(NewConfs, OldConfs) -> - emqx_map_lib:diff_maps( + emqx_utils_maps:diff_maps( flatten_confs(NewConfs), flatten_confs(OldConfs) ). diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index a1841b705..205fcfcf9 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -1029,5 +1029,5 @@ deobfuscate(NewConf, OldConf) -> map_to_json(M) -> emqx_utils_json:encode( - emqx_map_lib:jsonable_map(M, fun(K, V) -> {K, emqx_map_lib:binary_string(V)} end) + emqx_utils_maps:jsonable_map(M, fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end) ). diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 903e86443..347f9d973 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -186,7 +186,7 @@ update(Type, Name, {OldConf, Conf}, Opts0) -> %% without restarting the bridge. %% Opts = override_start_after_created(Conf, Opts0), - case emqx_map_lib:if_only_to_toggle_enable(OldConf, Conf) of + case emqx_utils_maps:if_only_to_toggle_enable(OldConf, Conf) of false -> ?SLOG(info, #{ msg => "update bridge", @@ -238,7 +238,7 @@ recreate(Type, Name, Conf, Opts) -> create_dry_run(Type, Conf0) -> TmpPath0 = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), TmpPath = emqx_utils:safe_filename(TmpPath0), - Conf = emqx_map_lib:safe_atom_key_map(Conf0), + Conf = emqx_utils_maps:safe_atom_key_map(Conf0), case emqx_connector_ssl:convert_certs(TmpPath, Conf) of {error, Reason} -> {error, Reason}; diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 52000ba7f..e5def2d64 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -251,7 +251,7 @@ do_convert_webhook_config( case {MReqTRoot, MReqTResource} of {{ok, ReqTRoot}, {ok, ReqTResource}} -> {_Parsed, ReqTRaw} = max({ReqTRoot, ReqTRootRaw}, {ReqTResource, ReqTResourceRaw}), - Conf1 = emqx_map_lib:deep_merge( + Conf1 = emqx_utils_maps:deep_merge( Conf0, #{ <<"request_timeout">> => ReqTRaw, diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index ef997f7e3..3afe17080 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -1221,7 +1221,7 @@ t_inconsistent_webhook_request_timeouts(Config) -> URL1 = ?URL(Port, "path1"), Name = ?BRIDGE_NAME, BadBridgeParams = - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( ?HTTP_BRIDGE(URL1, Name), #{ <<"request_timeout">> => <<"1s">>, diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index bd6af9323..ab58ce383 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -270,7 +270,7 @@ t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> ?SERVER_CONF(<<"user1">>)#{ <<"type">> => ?TYPE_MQTT, <<"name">> => BridgeName, - <<"ingress">> => emqx_map_lib:deep_merge( + <<"ingress">> => emqx_utils_maps:deep_merge( ?INGRESS_CONF, #{<<"remote">> => #{<<"qos">> => 2}} ) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 254284b75..08fbf5e15 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -673,7 +673,7 @@ create_bridge(Config, Overrides) -> Type = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), emqx_bridge:create(Type, Name, KafkaConfig). delete_bridge(Config) -> @@ -696,7 +696,7 @@ create_bridge_api(Config, Overrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), @@ -719,7 +719,7 @@ update_bridge_api(Config, Overrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kafka_name, Config), KafkaConfig0 = ?config(kafka_config, Config), - KafkaConfig = emqx_map_lib:deep_merge(KafkaConfig0, Overrides), + KafkaConfig = emqx_utils_maps:deep_merge(KafkaConfig0, Overrides), BridgeId = emqx_bridge_resource:bridge_id(TypeBin, Name), Params = KafkaConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges", BridgeId]), diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl index fa352ce89..b16df854f 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl @@ -92,7 +92,7 @@ kafka_consumer_test() -> ), %% Bad: can't repeat kafka topics. - BadConf1 = emqx_map_lib:deep_put( + BadConf1 = emqx_utils_maps:deep_put( [<<"bridges">>, <<"kafka_consumer">>, <<"my_consumer">>, <<"topic_mapping">>], Conf1, [ @@ -121,7 +121,7 @@ kafka_consumer_test() -> ), %% Bad: there must be at least 1 mapping. - BadConf2 = emqx_map_lib:deep_put( + BadConf2 = emqx_utils_maps:deep_put( [<<"bridges">>, <<"kafka_consumer">>, <<"my_consumer">>, <<"topic_mapping">>], Conf1, [] diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index d03cf9c27..246875ab6 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -43,50 +43,50 @@ add_handler(ConfKeyPath, HandlerName) -> remove_handler(ConfKeyPath) -> emqx_config_handler:remove_handler(ConfKeyPath). --spec get(emqx_map_lib:config_key_path()) -> term(). +-spec get(emqx_utils_maps:config_key_path()) -> term(). get(KeyPath) -> emqx:get_config(KeyPath). --spec get(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get(emqx_utils_maps:config_key_path(), term()) -> term(). get(KeyPath, Default) -> emqx:get_config(KeyPath, Default). --spec get_raw(emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path(), term()) -> term(). get_raw(KeyPath, Default) -> emqx_config:get_raw(KeyPath, Default). --spec get_raw(emqx_map_lib:config_key_path()) -> term(). +-spec get_raw(emqx_utils_maps:config_key_path()) -> term(). get_raw(KeyPath) -> emqx_config:get_raw(KeyPath). %% @doc Returns all values in the cluster. --spec get_all(emqx_map_lib:config_key_path()) -> #{node() => term()}. +-spec get_all(emqx_utils_maps:config_key_path()) -> #{node() => term()}. get_all(KeyPath) -> {ResL, []} = emqx_conf_proto_v2:get_all(KeyPath), maps:from_list(ResL). %% @doc Returns the specified node's KeyPath, or exception if not found --spec get_by_node(node(), emqx_map_lib:config_key_path()) -> term(). +-spec get_by_node(node(), emqx_utils_maps:config_key_path()) -> term(). get_by_node(Node, KeyPath) when Node =:= node() -> emqx:get_config(KeyPath); get_by_node(Node, KeyPath) -> emqx_conf_proto_v2:get_config(Node, KeyPath). %% @doc Returns the specified node's KeyPath, or the default value if not found --spec get_by_node(node(), emqx_map_lib:config_key_path(), term()) -> term(). +-spec get_by_node(node(), emqx_utils_maps:config_key_path(), term()) -> term(). get_by_node(Node, KeyPath, Default) when Node =:= node() -> emqx:get_config(KeyPath, Default); get_by_node(Node, KeyPath, Default) -> emqx_conf_proto_v2:get_config(Node, KeyPath, Default). %% @doc Returns the specified node's KeyPath, or config_not_found if key path not found --spec get_node_and_config(emqx_map_lib:config_key_path()) -> term(). +-spec get_node_and_config(emqx_utils_maps:config_key_path()) -> term(). get_node_and_config(KeyPath) -> {node(), emqx:get_config(KeyPath, config_not_found)}. %% @doc Update all value of key path in cluster-override.conf or local-override.conf. -spec update( - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -97,7 +97,7 @@ update(KeyPath, UpdateReq, Opts) -> %% @doc Update the specified node's key path in local-override.conf. -spec update( node(), - emqx_map_lib:config_key_path(), + emqx_utils_maps:config_key_path(), emqx_config:update_request(), emqx_config:update_opts() ) -> @@ -108,13 +108,13 @@ update(Node, KeyPath, UpdateReq, Opts) -> emqx_conf_proto_v2:update(Node, KeyPath, UpdateReq, Opts). %% @doc remove all value of key path in cluster-override.conf or local-override.conf. --spec remove(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove(KeyPath, Opts) -> emqx_conf_proto_v2:remove_config(KeyPath, Opts). %% @doc remove the specified node's key path in local-override.conf. --spec remove(node(), emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec remove(node(), emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. remove(Node, KeyPath, Opts) when Node =:= node() -> emqx:remove_config(KeyPath, Opts#{override_to => local}); @@ -122,13 +122,13 @@ remove(Node, KeyPath, Opts) -> emqx_conf_proto_v2:remove_config(Node, KeyPath, Opts). %% @doc reset all value of key path in cluster-override.conf or local-override.conf. --spec reset(emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset(emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset(KeyPath, Opts) -> emqx_conf_proto_v2:reset(KeyPath, Opts). %% @doc reset the specified node's key path in local-override.conf. --spec reset(node(), emqx_map_lib:config_key_path(), emqx_config:update_opts()) -> +-spec reset(node(), emqx_utils_maps:config_key_path(), emqx_config:update_opts()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. reset(Node, KeyPath, Opts) when Node =:= node() -> emqx:reset_config(KeyPath, Opts#{override_to => local}); diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl index 84687e314..b66307a1b 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v1.erl @@ -38,22 +38,22 @@ -include_lib("emqx/include/bpapi.hrl"). --type update_config_key_path() :: [emqx_map_lib:config_key(), ...]. +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. introduced_in() -> "5.0.0". --spec get_config(node(), emqx_map_lib:config_key_path()) -> +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath) -> rpc:call(Node, emqx, get_config, [KeyPath]). --spec get_config(node(), emqx_map_lib:config_key_path(), _Default) -> +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath, Default) -> rpc:call(Node, emqx, get_config, [KeyPath, Default]). --spec get_all(emqx_map_lib:config_key_path()) -> emqx_rpc:multicall_result(). +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). get_all(KeyPath) -> rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). diff --git a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl index dd8d2fedd..97446ee9f 100644 --- a/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl +++ b/apps/emqx_conf/src/proto/emqx_conf_proto_v2.erl @@ -44,19 +44,19 @@ introduced_in() -> -spec sync_data_from_node(node()) -> {ok, binary()} | emqx_rpc:badrpc(). sync_data_from_node(Node) -> rpc:call(Node, emqx_conf_app, sync_data_from_node, [], 20000). --type update_config_key_path() :: [emqx_map_lib:config_key(), ...]. +-type update_config_key_path() :: [emqx_utils_maps:config_key(), ...]. --spec get_config(node(), emqx_map_lib:config_key_path()) -> +-spec get_config(node(), emqx_utils_maps:config_key_path()) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath) -> rpc:call(Node, emqx, get_config, [KeyPath]). --spec get_config(node(), emqx_map_lib:config_key_path(), _Default) -> +-spec get_config(node(), emqx_utils_maps:config_key_path(), _Default) -> term() | emqx_rpc:badrpc(). get_config(Node, KeyPath, Default) -> rpc:call(Node, emqx, get_config, [KeyPath, Default]). --spec get_all(emqx_map_lib:config_key_path()) -> emqx_rpc:multicall_result(). +-spec get_all(emqx_utils_maps:config_key_path()) -> emqx_rpc:multicall_result(). get_all(KeyPath) -> rpc:multicall(emqx_conf, get_node_and_config, [KeyPath], 5000). diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl index 54dc0e022..e07d95d51 100644 --- a/apps/emqx_connector/src/emqx_connector_ssl.erl +++ b/apps/emqx_connector/src/emqx_connector_ssl.erl @@ -74,7 +74,7 @@ new_ssl_config(Config, _NewSSL) -> normalize_key_to_bin(undefined) -> undefined; normalize_key_to_bin(Map) when is_map(Map) -> - emqx_map_lib:binary_key_map(Map). + emqx_utils_maps:binary_key_map(Map). try_map_get(Key, Map, Default) when is_map(Map) -> maps:get(Key, Map, Default); diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 5197a35df..880a99313 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -410,7 +410,7 @@ handle_disconnect(_Reason) -> ok. maybe_publish_local(Msg, Vars, Props) -> - case emqx_map_lib:deep_get([local, topic], Vars, undefined) of + case emqx_utils_maps:deep_get([local, topic], Vars, undefined) of %% local topic is not set, discard it undefined -> ok; diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl index eac4f845f..01d96bdf0 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl @@ -119,7 +119,7 @@ remove_handler() -> pre_config_update(_Path, UpdateConf0, RawConf) -> UpdateConf = remove_sensitive_data(UpdateConf0), - NewConf = emqx_map_lib:deep_merge(RawConf, UpdateConf), + NewConf = emqx_utils_maps:deep_merge(RawConf, UpdateConf), ensure_ssl_cert(NewConf). -define(SENSITIVE_PASSWORD, <<"******">>). @@ -134,7 +134,7 @@ remove_sensitive_data(Conf0) -> end, case Conf1 of #{<<"listeners">> := #{<<"https">> := #{<<"password">> := ?SENSITIVE_PASSWORD}}} -> - emqx_map_lib:deep_remove([<<"listeners">>, <<"https">>, <<"password">>], Conf1); + emqx_utils_maps:deep_remove([<<"listeners">>, <<"https">>, <<"password">>], Conf1); _ -> Conf1 end. @@ -152,7 +152,7 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) -> ok. get_listener(Type, Conf) -> - emqx_map_lib:deep_get([listeners, Type], Conf, undefined). + emqx_utils_maps:deep_get([listeners, Type], Conf, undefined). diff_listeners(_, Listener, Listener) -> {#{}, #{}}; diff_listeners(Type, undefined, Start) -> {#{}, #{Type => Start}}; @@ -162,13 +162,14 @@ diff_listeners(Type, Stop, Start) -> {#{Type => Stop}, #{Type => Start}}. -define(DIR, <<"dashboard">>). ensure_ssl_cert(#{<<"listeners">> := #{<<"https">> := #{<<"enable">> := true}}} = Conf) -> - Https = emqx_map_lib:deep_get([<<"listeners">>, <<"https">>], Conf, undefined), + Https = emqx_utils_maps:deep_get([<<"listeners">>, <<"https">>], Conf, undefined), Opts = #{required_keys => [[<<"keyfile">>], [<<"certfile">>], [<<"cacertfile">>]]}, case emqx_tls_lib:ensure_ssl_files(?DIR, Https, Opts) of {ok, undefined} -> {error, <<"ssl_cert_not_found">>}; {ok, NewHttps} -> - {ok, emqx_map_lib:deep_merge(Conf, #{<<"listeners">> => #{<<"https">> => NewHttps}})}; + {ok, + emqx_utils_maps:deep_merge(Conf, #{<<"listeners">> => #{<<"https">> => NewHttps}})}; {error, Reason} -> ?SLOG(error, Reason#{msg => "bad_ssl_config"}), {error, Reason} diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 22cf484ff..b2ad69997 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -474,7 +474,7 @@ maybe_add_summary_from_label(Spec, Hocon) -> get_i18n(Key, Struct, Default) -> {ok, #{cache := Cache, lang := Lang}} = emqx_dashboard:get_i18n(), Desc = hocon_schema:resolve_schema(Struct, Cache), - emqx_map_lib:deep_get([Key, Lang], Desc, Default). + emqx_utils_maps:deep_get([Key, Lang], Desc, Default). trans_label(Spec, Hocon, Default) -> Label = diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl index 77937a835..0647c80ea 100644 --- a/apps/emqx_exhook/src/emqx_exhook_mgr.erl +++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl @@ -507,11 +507,11 @@ sort_name_by_order(Names, Orders) -> lists:sort( fun (A, B) when is_binary(A) -> - emqx_map_lib:deep_get([A, order], Orders) < - emqx_map_lib:deep_get([B, order], Orders); + emqx_utils_maps:deep_get([A, order], Orders) < + emqx_utils_maps:deep_get([B, order], Orders); (#{name := A}, #{name := B}) -> - emqx_map_lib:deep_get([A, order], Orders) < - emqx_map_lib:deep_get([B, order], Orders) + emqx_utils_maps:deep_get([A, order], Orders) < + emqx_utils_maps:deep_get([B, order], Orders) end, Names ). diff --git a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl index 58f2f29b4..c03b3f231 100644 --- a/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_api_SUITE.erl @@ -311,7 +311,7 @@ t_update(Cfg) -> decode_json(Data) -> BinJosn = emqx_utils_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJosn). + emqx_utils_maps:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 14b80a500..d90bf3689 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -96,7 +96,7 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> LName = binary_to_atom(maps:get(<<"name">>, LConf)), Path = [listeners, Type, LName], - case emqx_map_lib:deep_get(Path, RunningConf, undefined) of + case emqx_utils_maps:deep_get(Path, RunningConf, undefined) of undefined -> ListenerId = emqx_gateway_utils:listener_id( GwName, Type, LName diff --git a/apps/emqx_gateway/src/emqx_gateway_conf.erl b/apps/emqx_gateway/src/emqx_gateway_conf.erl index 07a4c1e2c..56a3e2068 100644 --- a/apps/emqx_gateway/src/emqx_gateway_conf.erl +++ b/apps/emqx_gateway/src/emqx_gateway_conf.erl @@ -106,7 +106,7 @@ unconvert_listeners(Ls) when is_list(Ls) -> {[Type, Name], Lis1} = maps_key_take([<<"type">>, <<"name">>], Lis), _ = vaildate_listener_name(Name), NLis1 = maps:without([<<"id">>, <<"running">>], Lis1), - emqx_map_lib:deep_merge(Acc, #{Type => #{Name => NLis1}}) + emqx_utils_maps:deep_merge(Acc, #{Type => #{Name => NLis1}}) end, #{}, Ls @@ -160,8 +160,8 @@ gateway(GwName0) -> RawConf = emqx_config:fill_defaults( emqx_config:get_root_raw(Path) ), - Confs = emqx_map_lib:jsonable_map( - emqx_map_lib:deep_get(Path, RawConf) + Confs = emqx_utils_maps:jsonable_map( + emqx_utils_maps:deep_get(Path, RawConf) ), LsConf = maps:get(<<"listeners">>, Confs, #{}), Confs#{<<"listeners">> => convert_listeners(GwName, LsConf)}. @@ -198,8 +198,8 @@ listeners(GwName0) -> RawConf = emqx_config:fill_defaults( emqx_config:get_root_raw([<<"gateway">>]) ), - Listeners = emqx_map_lib:jsonable_map( - emqx_map_lib:deep_get( + Listeners = emqx_utils_maps:jsonable_map( + emqx_utils_maps:deep_get( [<<"gateway">>, GwName, <<"listeners">>], RawConf ) ), @@ -213,12 +213,12 @@ listener(ListenerId) -> ), try Path = [<<"gateway">>, GwName, <<"listeners">>, Type, LName], - LConf = emqx_map_lib:deep_get(Path, RootConf), + LConf = emqx_utils_maps:deep_get(Path, RootConf), Running = emqx_gateway_utils:is_running( binary_to_existing_atom(ListenerId), LConf ), {ok, - emqx_map_lib:jsonable_map( + emqx_utils_maps:jsonable_map( LConf#{ id => ListenerId, type => Type, @@ -305,8 +305,8 @@ ret_ok_err({ok, _}) -> ok; ret_ok_err(Err) -> Err. ret_gw(GwName, {ok, #{raw_config := GwConf}}) -> - GwConf1 = emqx_map_lib:deep_get([bin(GwName)], GwConf), - LsConf = emqx_map_lib:deep_get( + GwConf1 = emqx_utils_maps:deep_get([bin(GwName)], GwConf), + LsConf = emqx_utils_maps:deep_get( [bin(GwName), <<"listeners">>], GwConf, #{} @@ -331,7 +331,7 @@ ret_gw(_GwName, Err) -> Err. ret_authn(GwName, {ok, #{raw_config := GwConf}}) -> - Authn = emqx_map_lib:deep_get( + Authn = emqx_utils_maps:deep_get( [bin(GwName), <<"authentication">>], GwConf ), @@ -340,7 +340,7 @@ ret_authn(_GwName, Err) -> Err. ret_authn(GwName, {LType, LName}, {ok, #{raw_config := GwConf}}) -> - Authn = emqx_map_lib:deep_get( + Authn = emqx_utils_maps:deep_get( [ bin(GwName), <<"listeners">>, @@ -355,7 +355,7 @@ ret_authn(_, _, Err) -> Err. ret_listener_or_err(GwName, {LType, LName}, {ok, #{raw_config := GwConf}}) -> - LConf = emqx_map_lib:deep_get( + LConf = emqx_utils_maps:deep_get( [bin(GwName), <<"listeners">>, bin(LType), bin(LName)], GwConf ), @@ -377,7 +377,7 @@ pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) -> case maps:get(GwName, RawConf, undefined) of undefined -> NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf), - {ok, emqx_map_lib:deep_put([GwName], RawConf, NConf)}; + {ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf)}; _ -> badres_gateway(already_exist, GwName) end; @@ -389,7 +389,7 @@ pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) -> Conf1 = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf), NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf1), NConf1 = maps:merge(GwRawConf, NConf), - {ok, emqx_map_lib:deep_put([GwName], RawConf, NConf1)} + {ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf1)} end; pre_config_update(_, {unload_gateway, GwName}, RawConf) -> _ = tune_gw_certs( @@ -400,7 +400,7 @@ pre_config_update(_, {unload_gateway, GwName}, RawConf) -> {ok, maps:remove(GwName, RawConf)}; pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined ) of @@ -408,7 +408,7 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> NConf = convert_certs(certs_dir(GwName), Conf), NListener = #{LType => #{LName => NConf}}, {ok, - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( RawConf, #{GwName => #{<<"listeners">> => NListener}} )}; @@ -417,7 +417,7 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined ) of @@ -425,7 +425,7 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) - badres_listener(not_found, GwName, LType, LName); OldConf -> NConf = convert_certs(certs_dir(GwName), Conf, OldConf), - NRawConf = emqx_map_lib:deep_put( + NRawConf = emqx_utils_maps:deep_put( [GwName, <<"listeners">>, LType, LName], RawConf, NConf @@ -434,22 +434,22 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) - end; pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) -> Path = [GwName, <<"listeners">>, LType, LName], - case emqx_map_lib:deep_get(Path, RawConf, undefined) of + case emqx_utils_maps:deep_get(Path, RawConf, undefined) of undefined -> {ok, RawConf}; OldConf -> clear_certs(certs_dir(GwName), OldConf), - {ok, emqx_map_lib:deep_remove(Path, RawConf)} + {ok, emqx_utils_maps:deep_remove(Path, RawConf)} end; pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, ?AUTHN_BIN], RawConf, undefined ) of undefined -> {ok, - emqx_map_lib:deep_merge( + emqx_utils_maps:deep_merge( RawConf, #{GwName => #{?AUTHN_BIN => Conf}} )}; @@ -458,7 +458,7 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) -> end; pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined @@ -477,25 +477,25 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) -> #{LType => #{LName => NListener}} } }, - {ok, emqx_map_lib:deep_merge(RawConf, NGateway)}; + {ok, emqx_utils_maps:deep_merge(RawConf, NGateway)}; _ -> badres_listener_authn(already_exist, GwName, LType, LName) end end; pre_config_update(_, {update_authn, GwName, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, ?AUTHN_BIN], RawConf, undefined ) of undefined -> badres_authn(not_found, GwName); _Authn -> - {ok, emqx_map_lib:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf)} + {ok, emqx_utils_maps:deep_put([GwName, ?AUTHN_BIN], RawConf, Conf)} end; pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> case - emqx_map_lib:deep_get( + emqx_utils_maps:deep_get( [GwName, <<"listeners">>, LType, LName], RawConf, undefined @@ -514,7 +514,7 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> Listener ), {ok, - emqx_map_lib:deep_put( + emqx_utils_maps:deep_put( [GwName, <<"listeners">>, LType, LName], RawConf, NListener @@ -523,12 +523,12 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) -> end; pre_config_update(_, {remove_authn, GwName}, RawConf) -> {ok, - emqx_map_lib:deep_remove( + emqx_utils_maps:deep_remove( [GwName, ?AUTHN_BIN], RawConf )}; pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) -> Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN], - {ok, emqx_map_lib:deep_remove(Path, RawConf)}; + {ok, emqx_utils_maps:deep_remove(Path, RawConf)}; pre_config_update(_, UnknownReq, _RawConf) -> logger:error("Unknown configuration update request: ~0p", [UnknownReq]), {error, badreq}. diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 9bb5821f5..7aaaee9cb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -240,7 +240,7 @@ authn(GwName) -> ChainName = emqx_gateway_utils:global_chain(GwName), wrap_chain_name( ChainName, - emqx_map_lib:jsonable_map(emqx:get_raw_config(Path)) + emqx_utils_maps:jsonable_map(emqx:get_raw_config(Path)) ). -spec authn(gateway_name(), binary()) -> map(). @@ -250,7 +250,7 @@ authn(GwName, ListenerId) -> ChainName = emqx_gateway_utils:listener_chain(GwName, Type, Name), wrap_chain_name( ChainName, - emqx_map_lib:jsonable_map(emqx:get_raw_config(Path)) + emqx_utils_maps:jsonable_map(emqx:get_raw_config(Path)) ). wrap_chain_name(ChainName, Conf) -> diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 10adcc428..fb648062a 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -164,7 +164,7 @@ t_gateway_stomp(_) -> {204, _} = request(put, "/gateways/stomp", GwConf), {200, ConfResp} = request(get, "/gateways/stomp"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{frame => #{max_headers => 10}}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{frame => #{max_headers => 10}}), {204, _} = request(put, "/gateways/stomp", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/stomp"), assert_confs(GwConf2, ConfResp2), @@ -186,7 +186,7 @@ t_gateway_mqttsn(_) -> {204, _} = request(put, "/gateways/mqttsn", GwConf), {200, ConfResp} = request(get, "/gateways/mqttsn"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{predefined => []}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{predefined => []}), {204, _} = request(put, "/gateways/mqttsn", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/mqttsn"), assert_confs(GwConf2, ConfResp2), @@ -206,7 +206,7 @@ t_gateway_coap(_) -> {204, _} = request(put, "/gateways/coap", GwConf), {200, ConfResp} = request(get, "/gateways/coap"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{heartbeat => <<"10s">>}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{heartbeat => <<"10s">>}), {204, _} = request(put, "/gateways/coap", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/coap"), assert_confs(GwConf2, ConfResp2), @@ -244,7 +244,7 @@ t_gateway_lwm2m(_) -> {204, _} = request(put, "/gateways/lwm2m", GwConf), {200, ConfResp} = request(get, "/gateways/lwm2m"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{qmode_time_window => <<"10s">>}), {204, _} = request(put, "/gateways/lwm2m", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/lwm2m"), assert_confs(GwConf2, ConfResp2), @@ -264,7 +264,7 @@ t_gateway_exproto(_) -> {204, _} = request(put, "/gateways/exproto", GwConf), {200, ConfResp} = request(get, "/gateways/exproto"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}), + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{server => #{bind => <<"9200">>}}), {204, _} = request(put, "/gateways/exproto", maps:without([name, listeners], GwConf2)), {200, ConfResp2} = request(get, "/gateways/exproto"), assert_confs(GwConf2, ConfResp2), @@ -293,7 +293,7 @@ t_gateway_exproto_with_ssl(_) -> {204, _} = request(put, "/gateways/exproto", GwConf), {200, ConfResp} = request(get, "/gateways/exproto"), assert_confs(GwConf, ConfResp), - GwConf2 = emqx_map_lib:deep_merge(GwConf, #{ + GwConf2 = emqx_utils_maps:deep_merge(GwConf, #{ server => #{ bind => <<"9200">>, ssl_options => SslCliOpts diff --git a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl index 1ab36f7b8..149e6acd6 100644 --- a/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl @@ -282,7 +282,7 @@ t_case_exproto(_) -> disable_authn(GwName, Type, Name) -> RawCfg = emqx_conf:get_raw([gateway, GwName], #{}), - ListenerCfg = emqx_map_lib:deep_get( + ListenerCfg = emqx_utils_maps:deep_get( [<<"listeners">>, atom_to_binary(Type), atom_to_binary(Name)], RawCfg ), {ok, _} = emqx_gateway_conf:update_listener(GwName, {Type, Name}, ListenerCfg#{ diff --git a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl index 33c307770..1e947e793 100644 --- a/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl @@ -413,7 +413,7 @@ t_load_gateway_with_certs_content(_) -> ), {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), - SslConf = emqx_map_lib:deep_get( + SslConf = emqx_utils_maps:deep_get( [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], emqx:get_raw_config([gateway, stomp]) ), @@ -436,7 +436,7 @@ t_load_gateway_with_certs_content(_) -> % ), % {ok, _} = emqx_gateway_conf:load_gateway(<<"stomp">>, StompConf), % assert_confs(StompConf, emqx:get_raw_config([gateway, stomp])), -% SslConf = emqx_map_lib:deep_get( +% SslConf = emqx_utils_maps:deep_get( % [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], % emqx:get_raw_config([gateway, stomp]) % ), @@ -471,7 +471,7 @@ t_add_listener_with_certs_content(_) -> emqx:get_raw_config([gateway, stomp]) ), - SslConf = emqx_map_lib:deep_get( + SslConf = emqx_utils_maps:deep_get( [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>], emqx:get_raw_config([gateway, stomp]) ), diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index 6d80b63d8..bb378ef10 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -159,7 +159,7 @@ do_request(Mth, Req) -> <<>> -> #{}; _ -> - emqx_map_lib:unsafe_atom_key_map( + emqx_utils_maps:unsafe_atom_key_map( emqx_utils_json:decode(Resp, [return_maps]) ) end, diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl index 7860f1220..ff105b931 100644 --- a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.erl @@ -149,7 +149,7 @@ start_grpc_server(GwName, Options = #{bind := ListenOn}) -> } }, SvrOptions = - case emqx_map_lib:deep_get([ssl, enable], Options, false) of + case emqx_utils_maps:deep_get([ssl, enable], Options, false) of false -> []; true -> @@ -201,7 +201,7 @@ start_grpc_client_channel(GwName, Options = #{address := Address}) -> }} ) end, - case emqx_map_lib:deep_get([ssl, enable], Options, false) of + case emqx_utils_maps:deep_get([ssl, enable], Options, false) of false -> SvrAddr = compose_http_uri(http, Host, Port), grpc_client_sup:create_channel_pool(GwName, SvrAddr, #{}); diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index cd4829342..681c851bf 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -860,8 +860,8 @@ format_channel_info(ChannInfo = {_, _ClientInfo, _ClientStats}) -> format_channel_info(WhichNode, {_, ClientInfo0, ClientStats}) -> Node = maps:get(node, ClientInfo0, WhichNode), - ClientInfo1 = emqx_map_lib:deep_remove([conninfo, clientid], ClientInfo0), - ClientInfo2 = emqx_map_lib:deep_remove([conninfo, username], ClientInfo1), + ClientInfo1 = emqx_utils_maps:deep_remove([conninfo, clientid], ClientInfo0), + ClientInfo2 = emqx_utils_maps:deep_remove([conninfo, username], ClientInfo1), StatsMap = maps:without( [memory, next_pkt_id, total_heap_size], maps:from_list(ClientStats) @@ -958,4 +958,4 @@ format_authz_cache({{PubSub, Topic}, {AuthzResult, Timestamp}}) -> to_topic_info(Data) -> M = maps:with([<<"topic">>, <<"qos">>, <<"nl">>, <<"rap">>, <<"rh">>], Data), - emqx_map_lib:safe_atom_key_map(M). + emqx_utils_maps:safe_atom_key_map(M). diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 55cc50597..b37a47be6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -215,7 +215,7 @@ fields(Field) -> %% HTTP API Callbacks config(get, _Params, Req) -> Path = conf_path(Req), - {ok, Conf} = emqx_map_lib:deep_find(Path, get_full_config()), + {ok, Conf} = emqx_utils_maps:deep_find(Path, get_full_config()), {200, Conf}; config(put, #{body := Body}, Req) -> Path = conf_path(Req), @@ -349,7 +349,7 @@ gen_schema(_Conf) -> #{type => string}. with_default_value(Type, Value) -> - Type#{example => emqx_map_lib:binary_string(Value)}. + Type#{example => emqx_utils_maps:binary_string(Value)}. global_zone_roots() -> lists:map(fun({K, _}) -> K end, global_zone_schema()). diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index c126cfe19..d7f3ff321 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -390,7 +390,7 @@ crud_listeners_by_id(put, #{bindings := #{id := Id}, body := Body0}) -> undefined -> {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}}; PrevConf -> - MergeConfT = emqx_map_lib:deep_merge(PrevConf, Conf), + MergeConfT = emqx_utils_maps:deep_merge(PrevConf, Conf), MergeConf = emqx_listeners:ensure_override_limiter_conf(MergeConfT, Conf), case update(Path, MergeConf) of {ok, #{raw_config := _RawConf}} -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index cc200a4d5..e5704b817 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -55,7 +55,7 @@ t_update(_Config) -> %% update ok {ok, SysMon} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort}} = SysMon, - NewSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, not BusyPort), + NewSysMon = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, not BusyPort), {ok, #{}} = update_config(<<"sysmon">>, NewSysMon), {ok, SysMon1} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort1}} = SysMon1, @@ -63,7 +63,7 @@ t_update(_Config) -> assert_busy_port(BusyPort1), %% update failed - ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), + ErrorSysMon = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), ?assertMatch( {error, {"HTTP/1.1", 400, _}}, update_config(<<"sysmon">>, ErrorSysMon) @@ -78,7 +78,7 @@ t_update(_Config) -> assert_busy_port(true), %% reset no_default_value config - NewSysMon1 = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false), + NewSysMon1 = emqx_utils_maps:deep_put([<<"vm">>, <<"busy_port">>], SysMon, false), {ok, #{}} = update_config(<<"sysmon">>, NewSysMon1), ?assertMatch({error, {"HTTP/1.1", 400, _}}, reset_config(<<"sysmon">>, "")), {ok, SysMon4} = get_config(<<"sysmon">>), @@ -94,27 +94,33 @@ t_log(_Config) -> {ok, Log} = get_config("log"), File = "log/emqx-test.log", %% update handler - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), {ok, #{}} = update_config(<<"log">>, Log2), {ok, Log3} = logger:get_handler_config(default), ?assertMatch(#{config := #{file := File}}, Log3), - ErrLog1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, 1), + ErrLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, 1), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog1)), - ErrLog2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enabfe">>], Log, true), + ErrLog2 = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"default">>, <<"enabfe">>], Log, true + ), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_config(<<"log">>, ErrLog2)), %% add new handler File1 = "log/emqx-test1.log", - Handler = emqx_map_lib:deep_get([<<"file_handlers">>, <<"default">>], Log2), - NewLog1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>], Log2, Handler), - NewLog2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>, <<"file">>], NewLog1, File1), + Handler = emqx_utils_maps:deep_get([<<"file_handlers">>, <<"default">>], Log2), + NewLog1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"new">>], Log2, Handler), + NewLog2 = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"new">>, <<"file">>], NewLog1, File1 + ), {ok, #{}} = update_config(<<"log">>, NewLog2), {ok, Log4} = logger:get_handler_config(new), ?assertMatch(#{config := #{file := File1}}, Log4), %% disable new handler - Disable = emqx_map_lib:deep_put([<<"file_handlers">>, <<"new">>, <<"enable">>], NewLog2, false), + Disable = emqx_utils_maps:deep_put( + [<<"file_handlers">>, <<"new">>, <<"enable">>], NewLog2, false + ), {ok, #{}} = update_config(<<"log">>, Disable), ?assertEqual({error, {not_found, new}}, logger:get_handler_config(new)), ok. @@ -125,25 +131,25 @@ t_global_zone(_Config) -> ?assertEqual(lists:usort(ZonesKeys), lists:usort(maps:keys(Zones))), ?assertEqual( emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed]), - emqx_map_lib:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones) + emqx_utils_maps:deep_get([<<"mqtt">>, <<"max_qos_allowed">>], Zones) ), - NewZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), + NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), {ok, #{}} = update_global_zone(NewZones), ?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])), - BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), + BadZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)), %% Remove max_qos_allowed from raw config, but we still get default value(2). Mqtt0 = emqx_conf:get_raw([<<"mqtt">>]), - ?assertEqual(1, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt0)), + ?assertEqual(1, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt0)), Mqtt1 = maps:remove(<<"max_qos_allowed">>, Mqtt0), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt1), Mqtt2 = emqx_conf:get_raw([<<"mqtt">>]), ?assertNot(maps:is_key(<<"max_qos_allowed">>, Mqtt2), Mqtt2), {ok, #{<<"mqtt">> := Mqtt3}} = get_global_zone(), %% the default value is 2 - ?assertEqual(2, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt3)), + ?assertEqual(2, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt3)), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0), ok. diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index 90808d4be..62f689a84 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -193,10 +193,10 @@ t_clear_certs(Config) when is_list(Config) -> }, %% create, make sure the cert files are created - NewConf = emqx_map_lib:deep_put( + NewConf = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"certfile">>], ConfTemp, cert_file("certfile") ), - NewConf2 = emqx_map_lib:deep_put( + NewConf2 = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"keyfile">>], NewConf, cert_file("keyfile") ), @@ -205,7 +205,7 @@ t_clear_certs(Config) when is_list(Config) -> ?assertMatch({ok, [_, _]}, ListResult1), %% update - UpdateConf = emqx_map_lib:deep_put( + UpdateConf = emqx_utils_maps:deep_put( [<<"ssl_options">>, <<"keyfile">>], NewConf2, cert_file("keyfile2") ), _ = request(put, NewPath, [], UpdateConf), diff --git a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl index 7e4ccc541..1f14d075e 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl @@ -34,8 +34,8 @@ init_per_testcase(t_log_path, Config) -> emqx_config_logger:add_handler(), Log = emqx_conf:get_raw([log], #{}), File = "log/emqx-test.log", - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), - Log2 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, true), + Log2 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"file">>], Log1, File), {ok, #{}} = emqx_conf:update([log], Log2, #{rawconf_with_defaults => true}), Config; init_per_testcase(_, Config) -> @@ -43,7 +43,7 @@ init_per_testcase(_, Config) -> end_per_testcase(t_log_path, Config) -> Log = emqx_conf:get_raw([log], #{}), - Log1 = emqx_map_lib:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, false), + Log1 = emqx_utils_maps:deep_put([<<"file_handlers">>, <<"default">>, <<"enable">>], Log, false), {ok, #{}} = emqx_conf:update([log], Log1, #{rawconf_with_defaults => true}), emqx_config_logger:remove_handler(), Config; diff --git a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl index b2d854d22..4ae3dec88 100644 --- a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl @@ -230,7 +230,7 @@ t_large_payload(_) -> decode_json(Data) -> BinJson = emqx_utils_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJson). + emqx_utils_maps:unsafe_atom_key_map(BinJson). clear_all_record() -> ets:delete_all_objects(emqx_delayed). diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl index 32cbba084..7b0a0dc2a 100644 --- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl +++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl @@ -156,7 +156,7 @@ handle_cast({dispatch, Context, Pid, Topic}, #{limiter := Limiter} = State) -> {ok, Limiter2} = dispatch(Context, Pid, Topic, undefined, Limiter), {noreply, State#{limiter := Limiter2}}; handle_cast({refresh_limiter, Conf}, State) -> - BucketCfg = emqx_map_lib:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), + BucketCfg = emqx_utils_maps:deep_get([flow_control, batch_deliver_limiter], Conf, undefined), {ok, Limiter} = emqx_limiter_server:connect(?APP, internal, BucketCfg), {noreply, State#{limiter := Limiter}}; handle_cast(Msg, State) -> diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index 2b92b3f59..61eee0510 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -257,7 +257,7 @@ t_change_storage_type(_Config) -> #{data := Msgs0, meta := _} = decode_json(MsgsJson0), ?assertEqual(1, length(Msgs0)), - ChangedConf = emqx_map_lib:deep_merge( + ChangedConf = emqx_utils_maps:deep_merge( RawConf, #{ <<"backend">> => @@ -312,7 +312,7 @@ t_change_storage_type(_Config) -> %%-------------------------------------------------------------------- decode_json(Data) -> BinJson = emqx_utils_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJson). + emqx_utils_maps:unsafe_atom_key_map(BinJson). %%-------------------------------------------------------------------- %% Internal funcs diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 44b49a75b..ada52c5aa 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -115,7 +115,7 @@ start_link() -> %%------------------------------------------------------------------------------ post_config_update(_, _Req, NewRules, OldRules, _AppEnvs) -> #{added := Added, removed := Removed, changed := Updated} = - emqx_map_lib:diff_maps(NewRules, OldRules), + emqx_utils_maps:diff_maps(NewRules, OldRules), maps_foreach( fun({Id, {_Old, New}}) -> {ok, _} = update_rule(New#{id => bin(Id)}) diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl index ae74348e0..5196868c7 100644 --- a/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl +++ b/apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl @@ -166,7 +166,7 @@ t_settting(_) -> decode_json(Data) -> BinJosn = emqx_utils_json:decode(Data, [return_maps]), - emqx_map_lib:unsafe_atom_key_map(BinJosn). + emqx_utils_maps:unsafe_atom_key_map(BinJosn). request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx_utils/src/emqx_utils_maps.erl similarity index 99% rename from apps/emqx/src/emqx_map_lib.erl rename to apps/emqx_utils/src/emqx_utils_maps.erl index 631c3914d..d64075511 100644 --- a/apps/emqx/src/emqx_map_lib.erl +++ b/apps/emqx_utils/src/emqx_utils_maps.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_map_lib). +-module(emqx_utils_maps). -export([ deep_get/2, @@ -332,7 +332,7 @@ deep_filter(M, F) when is_map(M) -> if_only_to_toggle_enable(OldConf, Conf) -> #{added := Added, removed := Removed, changed := Updated} = - emqx_map_lib:diff_maps(OldConf, Conf), + emqx_utils_maps:diff_maps(OldConf, Conf), case {Added, Removed, Updated} of {Added, Removed, #{enable := _} = Updated} when map_size(Added) =:= 0, diff --git a/apps/emqx/test/emqx_map_lib_tests.erl b/apps/emqx_utils/test/emqx_utils_maps_tests.erl similarity index 76% rename from apps/emqx/test/emqx_map_lib_tests.erl rename to apps/emqx_utils/test/emqx_utils_maps_tests.erl index 894811d7c..506851f0a 100644 --- a/apps/emqx/test/emqx_map_lib_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_maps_tests.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_map_lib_tests). +-module(emqx_utils_maps_tests). -include_lib("eunit/include/eunit.hrl"). best_effort_recursive_sum_test_() -> @@ -22,21 +22,21 @@ best_effort_recursive_sum_test_() -> [ ?_assertEqual( #{foo => 3}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1}, #{foo => 2}, DummyLogger) + emqx_utils_maps:best_effort_recursive_sum(#{foo => 1}, #{foo => 2}, DummyLogger) ), ?_assertEqual( #{foo => 3, bar => 6.0}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => 2.0}, #{foo => 2, bar => 4.0}, DummyLogger ) ), ?_assertEqual( #{foo => 1, bar => 2}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1}, #{bar => 2}, DummyLogger) + emqx_utils_maps:best_effort_recursive_sum(#{foo => 1}, #{bar => 2}, DummyLogger) ), ?_assertEqual( #{foo => #{bar => 42}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 2}}, #{foo => #{bar => 40}}, DummyLogger ) ), @@ -45,7 +45,9 @@ best_effort_recursive_sum_test_() -> Logger = fun(What) -> Self ! {log, What} end, ?assertEqual( #{foo => 1, bar => 2}, - emqx_map_lib:best_effort_recursive_sum(#{foo => 1, bar => 2}, #{bar => bar}, Logger) + emqx_utils_maps:best_effort_recursive_sum( + #{foo => 1, bar => 2}, #{bar => bar}, Logger + ) ), receive {log, Log} -> @@ -55,55 +57,55 @@ best_effort_recursive_sum_test_() -> end, ?_assertEqual( #{}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => foo}, #{foo => bar}, DummyLogger ) ), ?_assertEqual( #{foo => 1}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1}, #{foo => bar}, DummyLogger ) ), ?_assertEqual( #{foo => 1}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => bar}, #{foo => 1}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 1}}, #{foo => 1}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1}, #{foo => #{bar => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => ignored}, #{foo => #{bar => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 2}, bar => #{foo => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => 1, bar => #{foo => 1}}, #{foo => #{bar => 2}, bar => 2}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => 2}, bar => #{foo => 1}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => 2}, bar => 2}, #{foo => 1, bar => #{foo => 1}}, DummyLogger ) ), ?_assertEqual( #{foo => #{bar => #{}}}, - emqx_map_lib:best_effort_recursive_sum( + emqx_utils_maps:best_effort_recursive_sum( #{foo => #{bar => #{foo => []}}}, #{foo => 1}, DummyLogger ) ) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index f37d7fb6a..3e442a926 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -261,7 +261,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(cassa_bridge_type, Config), Name = ?config(cassa_name, Config), BridgeConfig0 = ?config(cassa_config, Config), - BridgeConfig = emqx_map_lib:deep_merge(BridgeConfig0, Overrides), + BridgeConfig = emqx_utils_maps:deep_merge(BridgeConfig0, Overrides), emqx_bridge:create(BridgeType, Name, BridgeConfig). delete_bridge(Config) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 133ef0430..5ebd9a89d 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -193,7 +193,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(dynamo_bridge_type, Config), Name = ?config(dynamo_name, Config), DynamoConfig0 = ?config(dynamo_config, Config), - DynamoConfig = emqx_map_lib:deep_merge(DynamoConfig0, Overrides), + DynamoConfig = emqx_utils_maps:deep_merge(DynamoConfig0, Overrides), emqx_bridge:create(BridgeType, Name, DynamoConfig). delete_all_bridges() -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index 49bc444c6..a785924d4 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -181,7 +181,7 @@ create_bridge(Config, GCPPubSubConfigOverrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(gcp_pubsub_name, Config), GCPPubSubConfig0 = ?config(gcp_pubsub_config, Config), - GCPPubSubConfig = emqx_map_lib:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), + GCPPubSubConfig = emqx_utils_maps:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), ct:pal("creating bridge: ~p", [GCPPubSubConfig]), Res = emqx_bridge:create(TypeBin, Name, GCPPubSubConfig), ct:pal("bridge creation result: ~p", [Res]), @@ -194,7 +194,7 @@ create_bridge_http(Config, GCPPubSubConfigOverrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(gcp_pubsub_name, Config), GCPPubSubConfig0 = ?config(gcp_pubsub_config, Config), - GCPPubSubConfig = emqx_map_lib:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), + GCPPubSubConfig = emqx_utils_maps:deep_merge(GCPPubSubConfig0, GCPPubSubConfigOverrides), Params = GCPPubSubConfig#{<<"type">> => TypeBin, <<"name">> => Name}, Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl index 4891d5d9b..3def35920 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_SUITE.erl @@ -354,7 +354,7 @@ create_bridge(Config, Overrides) -> Type = influxdb_type_bin(?config(influxdb_type, Config)), Name = ?config(influxdb_name, Config), InfluxDBConfig0 = ?config(influxdb_config, Config), - InfluxDBConfig = emqx_map_lib:deep_merge(InfluxDBConfig0, Overrides), + InfluxDBConfig = emqx_utils_maps:deep_merge(InfluxDBConfig0, Overrides), emqx_bridge:create(Type, Name, InfluxDBConfig). delete_bridge(Config) -> @@ -390,7 +390,7 @@ create_rule_and_action_http(Config, Overrides) -> sql => <<"SELECT * FROM \"t/topic\"">>, actions => [BridgeId] }, - Params = emqx_map_lib:deep_merge(Params0, Overrides), + Params = emqx_utils_maps:deep_merge(Params0, Overrides), Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 2d4ee9222..0959e3c78 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -245,7 +245,7 @@ create_bridge(Config, Overrides) -> Type = mongo_type_bin(?config(mongo_type, Config)), Name = ?config(mongo_name, Config), MongoConfig0 = ?config(mongo_config, Config), - MongoConfig = emqx_map_lib:deep_merge(MongoConfig0, Overrides), + MongoConfig = emqx_utils_maps:deep_merge(MongoConfig0, Overrides), ct:pal("creating ~p bridge with config:\n ~p", [Type, MongoConfig]), emqx_bridge:create(Type, Name, MongoConfig). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl index 8f0b9ad9c..d76149b16 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_pgsql_SUITE.erl @@ -229,7 +229,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(pgsql_bridge_type, Config), Name = ?config(pgsql_name, Config), PGConfig0 = ?config(pgsql_config, Config), - PGConfig = emqx_map_lib:deep_merge(PGConfig0, Overrides), + PGConfig = emqx_utils_maps:deep_merge(PGConfig0, Overrides), emqx_bridge:create(BridgeType, Name, PGConfig). delete_bridge(Config) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl index 8e058f2f1..36ed10f38 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl @@ -207,7 +207,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(tdengine_bridge_type, Config), Name = ?config(tdengine_name, Config), TDConfig0 = ?config(tdengine_config, Config), - TDConfig = emqx_map_lib:deep_merge(TDConfig0, Overrides), + TDConfig = emqx_utils_maps:deep_merge(TDConfig0, Overrides), emqx_bridge:create(BridgeType, Name, TDConfig). delete_bridge(Config) -> diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl index 3569b246e..4c00903d5 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -67,7 +67,7 @@ get_schema(SchemaName) -> -spec add_schema(schema_name(), schema()) -> ok | {error, term()}. add_schema(Name, Schema) -> - RawSchema = emqx_map_lib:binary_key_map(Schema), + RawSchema = emqx_utils_maps:binary_key_map(Schema), Res = emqx_conf:update( [?CONF_KEY_ROOT, schemas, Name], RawSchema, @@ -113,7 +113,7 @@ post_config_update( added := Added, changed := Changed0, removed := Removed - } = emqx_map_lib:diff_maps(NewSchemas, OldSchemas), + } = emqx_utils_maps:diff_maps(NewSchemas, OldSchemas), Changed = maps:map(fun(_N, {_Old, New}) -> New end, Changed0), RemovedNames = maps:keys(Removed), case RemovedNames of diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 325d3d499..5bfba34b3 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -344,7 +344,7 @@ t_decode(Config) -> t_fail_rollback(Config) -> SerdeType = ?config(serde_type, Config), - OkSchema = emqx_map_lib:binary_key_map(schema_params(SerdeType)), + OkSchema = emqx_utils_maps:binary_key_map(schema_params(SerdeType)), BrokenSchema = OkSchema#{<<"source">> := <<"{}">>}, %% hopefully, for this small map, the key order is used. Serdes = #{ From 19981757aec4a4cdb215786571ece255c22f6673 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 15:49:12 +0200 Subject: [PATCH 118/279] fix: add is_json --- apps/emqx_utils/src/emqx_utils_json.erl | 16 ++++++++++++---- apps/emqx_utils/test/emqx_utils_json_SUITE.erl | 4 ++++ .../test/props/prop_emqx_utils_json.erl} | 0 3 files changed, 16 insertions(+), 4 deletions(-) rename apps/{emqx/test/props/prop_emqx_json.erl => emqx_utils/test/props/prop_emqx_utils_json.erl} (100%) diff --git a/apps/emqx_utils/src/emqx_utils_json.erl b/apps/emqx_utils/src/emqx_utils_json.erl index 2f0c6f88b..191063d38 100644 --- a/apps/emqx_utils/src/emqx_utils_json.erl +++ b/apps/emqx_utils/src/emqx_utils_json.erl @@ -46,11 +46,15 @@ ]} ). --type encode_options() :: emqx_utils_json:encode_options(). --type decode_options() :: emqx_utils_json:decode_options(). +-export([is_json/1]). + +-compile({inline, [is_json/1]}). + +-type encode_options() :: jiffy:encode_options(). +-type decode_options() :: jiffy:decode_options(). -type json_text() :: iolist() | binary(). --type json_term() :: emqx_utils_json:jiffy_decode_result(). +-type json_term() :: jiffy:jiffy_decode_result(). -export_type([json_text/0, json_term/0]). -export_type([decode_options/0, encode_options/0]). @@ -79,7 +83,7 @@ safe_encode(Term, Opts) -> end. -spec decode(json_text()) -> json_term(). -decode(Json) -> decode(Json, []). +decode(Json) -> decode(Json, [return_maps]). -spec decode(json_text(), decode_options()) -> json_term(). decode(Json, Opts) -> @@ -100,6 +104,10 @@ safe_decode(Json, Opts) -> {error, Reason} end. +-spec is_json(json_text()) -> boolean(). +is_json(Json) -> + element(1, safe_decode(Json)) =:= ok. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- diff --git a/apps/emqx_utils/test/emqx_utils_json_SUITE.erl b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl index 889cde595..ba7b5226a 100644 --- a/apps/emqx_utils/test/emqx_utils_json_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl @@ -136,3 +136,7 @@ safe_encode_decode(Term) -> {ok, {NTerm}} -> NTerm; {ok, NTerm} -> NTerm end. + +t_is_json(_) -> + ?assert(emqx_utils_json:is_json(<<"{}">>)), + ?assert(not emqx_utils_json:is_json(<<"foo">>)). diff --git a/apps/emqx/test/props/prop_emqx_json.erl b/apps/emqx_utils/test/props/prop_emqx_utils_json.erl similarity index 100% rename from apps/emqx/test/props/prop_emqx_json.erl rename to apps/emqx_utils/test/props/prop_emqx_utils_json.erl From 4f80690162e59f69532ba435e3c1355b3ecc65df Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 13 Apr 2023 15:50:34 +0200 Subject: [PATCH 119/279] fix: byebye jsx --- apps/emqx/src/emqx.app.src | 1 - apps/emqx/src/emqx_logger_jsonfmt.erl | 4 ++-- apps/emqx/test/emqx_common_test_helpers.erl | 2 +- apps/emqx_authz/src/emqx_authz_http.erl | 2 +- .../test/emqx_authz_api_cache_SUITE.erl | 4 ++-- .../test/emqx_authz_api_mnesia_SUITE.erl | 22 ++++++++--------- .../test/emqx_authz_api_settings_SUITE.erl | 4 ++-- .../test/emqx_bridge_mqtt_SUITE.erl | 24 +++++++++---------- apps/emqx_conf/src/emqx_conf.erl | 6 ++--- .../test/emqx_dashboard_SUITE.erl | 2 +- .../test/emqx_dashboard_api_test_helpers.erl | 2 +- .../src/emqx_lwm2m_message.erl | 2 +- .../test/emqx_mgmt_api_test_util.erl | 2 +- .../test/emqx_telemetry_api_SUITE.erl | 6 ++--- .../test/emqx_topic_metrics_api_SUITE.erl | 10 ++++---- apps/emqx_utils/src/emqx_utils.app.src | 3 ++- apps/emqx_utils/src/emqx_utils_json.erl | 6 +++-- .../emqx_utils/test/emqx_utils_json_SUITE.erl | 23 +++++++++++------- 18 files changed, 66 insertions(+), 59 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 7014c8381..c4894c933 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -16,7 +16,6 @@ cowboy, sasl, os_mon, - jiffy, lc, hocon ]}, diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index f60f5a716..e231b76dd 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -62,11 +62,11 @@ %% The JSON object is pretty-printed. %% NOTE: do not use this function for logging. best_effort_json(Input) -> - best_effort_json(Input, [space, {indent, 4}]). + best_effort_json(Input, [pretty]). best_effort_json(Input, Opts) -> Config = #{depth => unlimited, single_line => true}, JsonReady = best_effort_json_obj(Input, Config), - jsx:encode(JsonReady, Opts). + emqx_utils_json:encode(JsonReady, Opts). -spec format(logger:log_event(), config()) -> iodata(). format(#{level := Level, msg := Msg, meta := Meta} = Event, Config0) when is_map(Config0) -> diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 85cb46106..6628401cc 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -477,7 +477,7 @@ copy_certs(_, _) -> load_config(SchemaModule, Config, Opts) -> ConfigBin = case is_map(Config) of - true -> jsx:encode(Config); + true -> emqx_utils_json:encode(Config); false -> Config end, ok = emqx_config:delete_override_conf_files(), diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 852a667c8..53378d9c2 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -227,7 +227,7 @@ encode_path(Path) -> lists:flatten(["/" ++ Part || Part <- lists:map(fun uri_encode/1, Parts)]). serialize_body(<<"application/json">>, Body) -> - jsx:encode(Body); + emqx_utils_json:encode(Body); serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> query_string(Body). diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index cfd9f581f..672ddc73c 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -67,12 +67,12 @@ t_clean_cahce(_) -> ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0), {ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(2, length(emqx_utils_json:decode(Result3))), + ?assertEqual(2, maps:size(emqx_utils_json:decode(Result3))), request(delete, uri(["authorization", "cache"])), {ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(0, length(emqx_utils_json:decode(Result4))), + ?assertEqual(0, maps:size(emqx_utils_json:decode(Result4))), ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index 2aa2d9545..3775b9a1c 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -95,7 +95,7 @@ t_api(_) -> <<"page">> := 1, <<"hasnext">> := false } - } = jsx:decode(Request1), + } = emqx_utils_json:decode(Request1), ?assertEqual(3, length(Rules1)), {ok, 200, Request1_1} = @@ -119,7 +119,7 @@ t_api(_) -> <<"hasnext">> => false } }, - jsx:decode(Request1_1) + emqx_utils_json:decode(Request1_1) ), {ok, 200, Request2} = @@ -128,7 +128,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users", "user1"]), [] ), - #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = jsx:decode(Request2), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules1} = emqx_utils_json:decode(Request2), {ok, 204, _} = request( @@ -142,7 +142,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users", "user1"]), [] ), - #{<<"username">> := <<"user1">>, <<"rules">> := Rules2} = jsx:decode(Request3), + #{<<"username">> := <<"user1">>, <<"rules">> := Rules2} = emqx_utils_json:decode(Request3), ?assertEqual(0, length(Rules2)), {ok, 204, _} = @@ -202,8 +202,8 @@ t_api(_) -> <<"data">> := [#{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3}], <<"meta">> := #{<<"count">> := 1, <<"limit">> := 100, <<"page">> := 1} } = - jsx:decode(Request4), - #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = jsx:decode(Request5), + emqx_utils_json:decode(Request4), + #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = emqx_utils_json:decode(Request5), ?assertEqual(3, length(Rules3)), {ok, 204, _} = @@ -218,7 +218,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "clients", "client1"]), [] ), - #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules4} = jsx:decode(Request6), + #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules4} = emqx_utils_json:decode(Request6), ?assertEqual(0, length(Rules4)), {ok, 204, _} = @@ -252,7 +252,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "all"]), [] ), - #{<<"rules">> := Rules5} = jsx:decode(Request7), + #{<<"rules">> := Rules5} = emqx_utils_json:decode(Request7), ?assertEqual(3, length(Rules5)), {ok, 204, _} = @@ -267,7 +267,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "all"]), [] ), - #{<<"rules">> := Rules6} = jsx:decode(Request8), + #{<<"rules">> := Rules6} = emqx_utils_json:decode(Request8), ?assertEqual(0, length(Rules6)), {ok, 204, _} = @@ -285,7 +285,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "users?page=2&limit=5"]), [] ), - #{<<"data">> := Data1} = jsx:decode(Request9), + #{<<"data">> := Data1} = emqx_utils_json:decode(Request9), ?assertEqual(5, length(Data1)), {ok, 204, _} = @@ -303,7 +303,7 @@ t_api(_) -> uri(["authorization", "sources", "built_in_database", "rules", "clients?limit=5"]), [] ), - #{<<"data">> := Data2} = jsx:decode(Request10), + #{<<"data">> := Data2} = emqx_utils_json:decode(Request10), ?assertEqual(5, length(Data2)), {ok, 400, Msg1} = diff --git a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl index 41eba109e..e3412e169 100644 --- a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl @@ -76,7 +76,7 @@ t_api(_) -> {ok, 200, Result1} = request(put, uri(["authorization", "settings"]), Settings1), {ok, 200, Result1} = request(get, uri(["authorization", "settings"]), []), - ?assertEqual(Settings1, jsx:decode(Result1)), + ?assertEqual(Settings1, emqx_utils_json:decode(Result1)), Settings2 = #{ <<"no_match">> => <<"allow">>, @@ -90,7 +90,7 @@ t_api(_) -> {ok, 200, Result2} = request(put, uri(["authorization", "settings"]), Settings2), {ok, 200, Result2} = request(get, uri(["authorization", "settings"]), []), - ?assertEqual(Settings2, jsx:decode(Result2)), + ?assertEqual(Settings2, emqx_utils_json:decode(Result2)), ok. diff --git a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl index ab58ce383..bd5cda3f0 100644 --- a/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl @@ -201,7 +201,7 @@ t_mqtt_conn_bridge_ingress(_) -> #{ <<"type">> := ?TYPE_MQTT, <<"name">> := ?BRIDGE_NAME_INGRESS - } = jsx:decode(Bridge), + } = emqx_utils_json:decode(Bridge), BridgeIDIngress = emqx_bridge_resource:bridge_id(?TYPE_MQTT, ?BRIDGE_NAME_INGRESS), @@ -313,7 +313,7 @@ t_mqtt_conn_bridge_ingress_no_payload_template(_) -> emqx:publish(emqx_message:make(RemoteTopic, Payload)), %% we should receive a message on the local broker, with specified topic Msg = assert_mqtt_msg_received(LocalTopic), - ?assertMatch(#{<<"payload">> := Payload}, jsx:decode(Msg#message.payload)), + ?assertMatch(#{<<"payload">> := Payload}, emqx_utils_json:decode(Msg#message.payload)), %% verify the metrics of the bridge ?assertMetrics( @@ -402,7 +402,7 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> Msg = assert_mqtt_msg_received(RemoteTopic), %% the MapMsg is all fields outputed by Rule-Engine. it's a binary coded json here. ?assertMatch(<>, Msg#message.from), - ?assertMatch(#{<<"payload">> := Payload}, jsx:decode(Msg#message.payload)), + ?assertMatch(#{<<"payload">> := Payload}, emqx_utils_json:decode(Msg#message.payload)), %% verify the metrics of the bridge ?retry( @@ -545,7 +545,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> <<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">> } ), - #{<<"id">> := RuleId} = jsx:decode(Rule), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule), %% we now test if the bridge works as expected @@ -562,7 +562,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> %% and also the rule should be matched, with matched + 1: {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), {ok, 200, Metrics} = request(get, uri(["rules", RuleId, "metrics"]), []), - ?assertMatch(#{<<"id">> := RuleId}, jsx:decode(Rule1)), + ?assertMatch(#{<<"id">> := RuleId}, emqx_utils_json:decode(Rule1)), ?assertMatch( #{ <<"metrics">> := #{ @@ -581,7 +581,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> <<"actions.failed.unknown">> := 0 } }, - jsx:decode(Metrics) + emqx_utils_json:decode(Metrics) ), %% we also check if the actions of the rule is triggered @@ -630,7 +630,7 @@ t_egress_mqtt_bridge_with_rules(_) -> <<"sql">> => <<"SELECT * from \"t/1\"">> } ), - #{<<"id">> := RuleId} = jsx:decode(Rule), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule), %% we now test if the bridge works as expected LocalTopic = <>, @@ -653,7 +653,7 @@ t_egress_mqtt_bridge_with_rules(_) -> timer:sleep(100), emqx:publish(emqx_message:make(RuleTopic, Payload2)), {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []), - ?assertMatch(#{<<"id">> := RuleId, <<"name">> := _}, jsx:decode(Rule1)), + ?assertMatch(#{<<"id">> := RuleId, <<"name">> := _}, emqx_utils_json:decode(Rule1)), {ok, 200, Metrics} = request(get, uri(["rules", RuleId, "metrics"]), []), ?assertMatch( #{ @@ -673,7 +673,7 @@ t_egress_mqtt_bridge_with_rules(_) -> <<"actions.failed.unknown">> := 0 } }, - jsx:decode(Metrics) + emqx_utils_json:decode(Metrics) ), %% we should receive a message on the "remote" broker, with specified topic @@ -911,17 +911,17 @@ create_bridge(Config = #{<<"type">> := Type, <<"name">> := Name}) -> <<"type">> := Type, <<"name">> := Name }, - jsx:decode(Bridge) + emqx_utils_json:decode(Bridge) ), emqx_bridge_resource:bridge_id(Type, Name). request_bridge(BridgeID) -> {ok, 200, Bridge} = request(get, uri(["bridges", BridgeID]), []), - jsx:decode(Bridge). + emqx_utils_json:decode(Bridge). request_bridge_metrics(BridgeID) -> {ok, 200, BridgeMetrics} = request(get, uri(["bridges", BridgeID, "metrics"]), []), - jsx:decode(BridgeMetrics). + emqx_utils_json:decode(BridgeMetrics). request(Method, Url, Body) -> request(<<"connector_admin">>, Method, Url, Body). diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 246875ab6..1ecda913d 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -166,7 +166,7 @@ gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> io:format(user, "===< Including fields from importance level: ~p~n", [IncludeImportance]), Opts = #{desc_file => I18nFile, lang => Lang, include_importance_up_from => IncludeImportance}, JsonMap = hocon_schema_json:gen(SchemaModule, Opts), - IoData = jsx:encode(JsonMap, [space, {indent, 4}]), + IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]), ok = file:write_file(SchemaJsonFile, IoData). gen_api_schema_json(Dir, I18nFile, Lang) -> @@ -268,13 +268,13 @@ do_gen_api_schema_json(File, SchemaMod, SchemaInfo) -> ApiSpec0 ), Components = lists:foldl(fun(M, Acc) -> maps:merge(M, Acc) end, #{}, Components0), - IoData = jsx:encode( + IoData = emqx_utils_json:encode( #{ info => SchemaInfo, paths => ApiSpec, components => #{schemas => Components} }, - [space, {indent, 4}] + [pretty, force_utf8] ), file:write_file(File, IoData). diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index ee850ec17..1f14b02c0 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -142,7 +142,7 @@ t_swagger_json(_Config) -> %% with auth Auth = auth_header_(<<"admin">>, <<"public_www1">>), {ok, 200, Body1} = request_api(get, Url, Auth), - ?assert(jsx:is_json(Body1)), + ?assert(emqx_utils_json:is_json(Body1)), %% without auth {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, Body2}} = httpc:request(get, {Url, []}, [], [{body_format, binary}]), diff --git a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl index 8df130897..91c7729d3 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl @@ -82,7 +82,7 @@ request(Username, Method, Url, Body) -> -> {Url, [auth_header(Username)]}; _ -> - {Url, [auth_header(Username)], "application/json", jsx:encode(Body)} + {Url, [auth_header(Username)], "application/json", emqx_utils_json:encode(Body)} end, ct:pal("Method: ~p, Request: ~p", [Method, Request]), case httpc:request(Method, Request, [], [{body_format, binary}]) of diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl index c65fff441..8b9ba2491 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_message.erl @@ -152,7 +152,7 @@ object_resource_id(BaseName) -> value(Value, ResourceId, ObjDefinition) -> case emqx_lwm2m_xml_object:get_resource_type(ResourceId, ObjDefinition) of "String" -> - % keep binary type since it is same as a string for jsx + % keep binary type since it is same as a string for emqx_utils_json Value; "Integer" -> Size = byte_size(Value) * 8, diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 5fe4ca937..985b95d5b 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -157,7 +157,7 @@ api_path_without_base_path(Parts) -> %% %% Usage with RequestData: %% Payload = [{upload_type, <<"user_picture">>}], -%% PayloadContent = jsx:encode(Payload), +%% PayloadContent = emqx_utils_json:encode(Payload), %% RequestData = [ %% {<<"payload">>, PayloadContent} %% ] diff --git a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl index 16f942bc0..ac6d12039 100644 --- a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl @@ -113,7 +113,7 @@ t_status(_) -> ?assertEqual( #{<<"enable">> => false}, - jsx:decode(Result0) + emqx_utils_json:decode(Result0) ), ?assertMatch( @@ -139,7 +139,7 @@ t_status(_) -> ?assertEqual( #{<<"enable">> => true}, - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -180,7 +180,7 @@ t_data(_) -> <<"uuid">> := _, <<"vm_specs">> := _ }, - jsx:decode(Result) + emqx_utils_json:decode(Result) ), {ok, 200, _} = diff --git a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl index ea85d1fe9..5d64f123d 100644 --- a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl @@ -74,7 +74,7 @@ t_mqtt_topic_metrics_collection(_) -> ?assertEqual( [], - jsx:decode(Result0) + emqx_utils_json:decode(Result0) ), {ok, 200, _} = request( @@ -95,7 +95,7 @@ t_mqtt_topic_metrics_collection(_) -> <<"metrics">> := #{} } ], - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -150,7 +150,7 @@ t_mqtt_topic_metrics(_) -> uri(["mqtt", "topic_metrics"]) ), - ?assertMatch([_], jsx:decode(Result0)), + ?assertMatch([_], emqx_utils_json:decode(Result0)), {ok, 200, Result1} = request( get, @@ -162,7 +162,7 @@ t_mqtt_topic_metrics(_) -> <<"topic">> := <<"topic/1/2">>, <<"metrics">> := #{} }, - jsx:decode(Result1) + emqx_utils_json:decode(Result1) ), ?assertMatch( @@ -288,7 +288,7 @@ t_node_aggregation(_) -> <<"topic">> := <<"topic/1/2">>, <<"metrics">> := #{<<"messages.dropped.count">> := 3} }, - jsx:decode(Result) + emqx_utils_json:decode(Result) ), meck:unload(emqx_topic_metrics_proto_v1). diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index 08a802d1b..b0dedbd72 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -14,7 +14,8 @@ {registered, []}, {applications, [ kernel, - stdlib + stdlib, + jiffy ]}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_utils/src/emqx_utils_json.erl b/apps/emqx_utils/src/emqx_utils_json.erl index 191063d38..df7388c94 100644 --- a/apps/emqx_utils/src/emqx_utils_json.erl +++ b/apps/emqx_utils/src/emqx_utils_json.erl @@ -65,7 +65,7 @@ encode(Term) -> -spec encode(json_term(), encode_options()) -> json_text(). encode(Term, Opts) -> - to_binary(emqx_utils_json:encode(to_ejson(Term), Opts)). + to_binary(jiffy:encode(to_ejson(Term), Opts)). -spec safe_encode(json_term()) -> {ok, json_text()} | {error, Reason :: term()}. @@ -87,7 +87,7 @@ decode(Json) -> decode(Json, [return_maps]). -spec decode(json_text(), decode_options()) -> json_term(). decode(Json, Opts) -> - from_ejson(emqx_utils_json:decode(Json, Opts)). + from_ejson(jiffy:decode(Json, Opts)). -spec safe_decode(json_text()) -> {ok, json_term()} | {error, Reason :: term()}. @@ -125,6 +125,8 @@ to_ejson([{_, _} | _] = L) -> {[{K, to_ejson(V)} || {K, V} <- L]}; to_ejson(L) when is_list(L) -> [to_ejson(E) || E <- L]; +to_ejson(M) when is_map(M) -> + maps:map(fun(_K, V) -> to_ejson(V) end, M); to_ejson(T) -> T. diff --git a/apps/emqx_utils/test/emqx_utils_json_SUITE.erl b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl index ba7b5226a..daf31b440 100644 --- a/apps/emqx_utils/test/emqx_utils_json_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_json_SUITE.erl @@ -84,10 +84,10 @@ t_decode_encode(_) -> 1.25 = decode(encode(1.25)), [] = decode(encode([])), [true, 1] = decode(encode([true, 1])), - [{}] = decode(encode([{}])), - [{<<"foo">>, <<"bar">>}] = decode(encode([{foo, bar}])), - [{<<"foo">>, <<"bar">>}] = decode(encode([{<<"foo">>, <<"bar">>}])), - [[{<<"foo">>, <<"bar">>}]] = decode(encode([[{<<"foo">>, <<"bar">>}]])), + [{}] = decode(encode([{}]), []), + [{<<"foo">>, <<"bar">>}] = decode(encode([{foo, bar}]), []), + [{<<"foo">>, <<"bar">>}] = decode(encode([{<<"foo">>, <<"bar">>}]), []), + [[{<<"foo">>, <<"bar">>}]] = decode(encode([[{<<"foo">>, <<"bar">>}]]), []), [ [ {<<"foo">>, <<"bar">>}, @@ -101,7 +101,8 @@ t_decode_encode(_) -> {<<"a">>, <<"b">>} ], [{<<"x">>, <<"y">>}] - ]) + ]), + [] ), #{<<"foo">> := <<"bar">>} = decode(encode(#{<<"foo">> => <<"bar">>}), [return_maps]), JsonText = <<"{\"bool\":true,\"int\":10,\"foo\":\"bar\"}">>, @@ -110,8 +111,12 @@ t_decode_encode(_) -> <<"int">> => 10, <<"foo">> => <<"bar">> }, - ?assertEqual(JsonText, encode({decode(JsonText)})), - ?assertEqual(JsonMaps, decode(JsonText, [return_maps])). + ?assertEqual(JsonText, encode({decode(JsonText, [])})), + ?assertEqual(JsonMaps, decode(JsonText, [return_maps])), + ?assertEqual( + #{<<"foo">> => #{<<"bar">> => <<"baz">>}}, + decode(encode(#{<<"foo">> => [{<<"bar">>, <<"baz">>}]})) + ). t_safe_decode_encode(_) -> safe_encode_decode(null), @@ -123,7 +128,7 @@ t_safe_decode_encode(_) -> 1.25 = safe_encode_decode(1.25), [] = safe_encode_decode([]), [true, 1] = safe_encode_decode([true, 1]), - [{}] = decode(encode([{}])), + [{}] = decode(encode([{}]), []), [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{foo, bar}]), [{<<"foo">>, <<"bar">>}] = safe_encode_decode([{<<"foo">>, <<"bar">>}]), [[{<<"foo">>, <<"bar">>}]] = safe_encode_decode([[{<<"foo">>, <<"bar">>}]]), @@ -132,7 +137,7 @@ t_safe_decode_encode(_) -> safe_encode_decode(Term) -> {ok, Json} = emqx_utils_json:safe_encode(Term), - case emqx_utils_json:safe_decode(Json) of + case emqx_utils_json:safe_decode(Json, []) of {ok, {NTerm}} -> NTerm; {ok, NTerm} -> NTerm end. From 90520a5382a3b871e97de066ff469135b2af9322 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 11:43:53 +0200 Subject: [PATCH 120/279] docs: add an actual README --- apps/emqx_utils/README.md | 55 +++++++++++++-------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/apps/emqx_utils/README.md b/apps/emqx_utils/README.md index 482b7a947..046b66321 100644 --- a/apps/emqx_utils/README.md +++ b/apps/emqx_utils/README.md @@ -1,43 +1,24 @@ -# [Application Name] - [Mandatory] -> 0. App overview introduction -> 1. let people know what your project can do specifically. Is it a base -> library dependency, or what kind of functionality is provided to the user? -> 2. Provide context and add a link to any reference visitors might be -> unfamiliar with. -> 3. Design details, implementation technology architecture, Roadmap, etc. +# emqx_utils - Erlang utility library for EMQ X -# [Features] - [Optional] -> A List of features your application provided. If the feature is quite simple, just -> list in the previous section. +## Overview -# [Limitation] - [Optional] -> Explain the limitations of the implementation used in your application. +`emqx_utils` is a collection of utility functions for EMQ X, organized into +several modules. It provides various functionalities to make it easier to work +with EMQ X, such as binary manipulations, maps, JSON en- and decoding, ets table +handling, data conversions, and more. -# [Documention links] - [Mandatory] -> You can use the official docs link to provide a detailed explanation of -> the concept and functions of this application. +## Features -# [Installation] - [Optional] -> How users can obtain this application. In most cases, this section is unnecessary. -> Most of the applications are released with the EMQX distribution package. -> Otherwise, it is necessary to provide step-by-step instructions on how -> to install this application. +- `emqx_utils`: unsorted helper functions, formerly known as `emqx_misc` - NEEDS WORK +- `emqx_utils_api`: collection of helper functions for API responses +- `emqx_utils_binary`: binary reverse, join, trim etc +- `emqx_utils_ets`: convenience functions for creating and looking up data in ets tables. +- `emqx_utils_json`: JSON encoding and decoding +- `emqx_utils_maps`: convenience functions for map lookup and manipulation like + deep_get etc. -# [Usage] - [Optional] -> This section explains how users can use the features provided by the application. +## Contributing -# [Basic Usage Guide] - [Optional] -> Simple and reproducible introduction on how to use/start it. - -# [Configurations] - [Mandatory] -> Most important configurations that this application depends on. -> It would be best to attach official documentation if it's available. - -# [HTTP APIs] - [Optional] -> Same as configuration. - -# [Other] - [Optional] -> Other topics that users may need to know can be placed here. - -# Contributing - [Mandatory] -Please see our [contributing.md](../../CONTRIBUTING.md). +Please see our [contributing guidelines](../../CONTRIBUTING.md) for information +on how to contribute to `emqx_utils`. We welcome bug reports, feature requests, +and pull requests. From d98f7222ff2ccdba5f6830a720961f1ea649015c Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 11:52:21 +0200 Subject: [PATCH 121/279] style: add comment to binary_string/1 --- apps/emqx_utils/src/emqx_utils_maps.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_utils/src/emqx_utils_maps.erl b/apps/emqx_utils/src/emqx_utils_maps.erl index d64075511..6bec32ae3 100644 --- a/apps/emqx_utils/src/emqx_utils_maps.erl +++ b/apps/emqx_utils/src/emqx_utils_maps.erl @@ -210,6 +210,7 @@ binary_string_kv(K, V, JsonableFun) -> {K1, V1} -> {binary_string(K1), V1} end. +%% [FIXME] this doesn't belong here binary_string([]) -> []; binary_string(Val) when is_list(Val) -> From badf96280019f1b89482cdb9de96cbca418fcd01 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 11:53:05 +0200 Subject: [PATCH 122/279] fix: stale call to emqx_misc --- apps/emqx_utils/include/emqx_utils_api.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_utils/include/emqx_utils_api.hrl b/apps/emqx_utils/include/emqx_utils_api.hrl index 549b0f94c..bfc8e0a53 100644 --- a/apps/emqx_utils/include/emqx_utils_api.hrl +++ b/apps/emqx_utils/include/emqx_utils_api.hrl @@ -17,7 +17,7 @@ -ifndef(EMQX_API_LIB_HRL). -define(EMQX_API_LIB_HRL, true). --define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_misc:readable_error_msg(REASON)}). +-define(ERROR_MSG(CODE, REASON), #{code => CODE, message => emqx_utils:readable_error_msg(REASON)}). -define(OK(CONTENT), {200, CONTENT}). From 0f162fb50a4cc38a8a599785ca9dde95c8f7aa0b Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 11:54:02 +0200 Subject: [PATCH 123/279] test: add tests for emqx_utils_binary from original site --- .../test/emqx_utils_binary_tests.erl | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 apps/emqx_utils/test/emqx_utils_binary_tests.erl diff --git a/apps/emqx_utils/test/emqx_utils_binary_tests.erl b/apps/emqx_utils/test/emqx_utils_binary_tests.erl new file mode 100644 index 000000000..514e0874a --- /dev/null +++ b/apps/emqx_utils/test/emqx_utils_binary_tests.erl @@ -0,0 +1,212 @@ +-module(emqx_utils_binary_tests). + +-import(emqx_utils_binary, [ + trim/1, + ltrim/1, + rtrim/1, + trim/2, + ltrim/2, + rtrim/2, + reverse/1, + inverse/1, + join/2, + suffix/2, + prefix/2, + duplicate/2, + union/2, + intersection/2, + subtract/2, + optimize_patterns/1 +]). + +-include_lib("eunit/include/eunit.hrl"). + +-ifdef(TEST). + +trim1_test_() -> + [ + ?_assertEqual(trim(<<>>), <<>>), + ?_assertEqual(trim(<<0, 0, 0>>), <<>>), + ?_assertEqual(trim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(trim(<<0, 1, 2>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 0, 1, 2>>), <<1, 2>>), + ?_assertEqual(trim(<<1, 2, 0, 0>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 1, 2, 0>>), <<1, 2>>), + ?_assertEqual(trim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<1, 2>>) + ]. + +ltrim1_test_() -> + [ + ?_assertEqual(ltrim(<<>>), <<>>), + ?_assertEqual(ltrim(<<0, 0, 0>>), <<>>), + ?_assertEqual(ltrim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(ltrim(<<0, 1, 2>>), <<1, 2>>), + ?_assertEqual(ltrim(<<0, 0, 1, 2>>), <<1, 2>>), + ?_assertEqual(ltrim(<<1, 2, 0, 0>>), <<1, 2, 0, 0>>), + ?_assertEqual(ltrim(<<0, 1, 2, 0>>), <<1, 2, 0>>), + ?_assertEqual(ltrim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<1, 2, 0, 0, 0>>) + ]. + +rtrim1_test_() -> + [ + ?_assertEqual(rtrim(<<>>), <<>>), + ?_assertEqual(rtrim(<<1, 2, 3>>), <<1, 2, 3>>), + ?_assertEqual(rtrim(<<0, 0, 0>>), <<>>), + ?_assertEqual(rtrim(<<0, 1, 2>>), <<0, 1, 2>>), + ?_assertEqual(rtrim(<<0, 0, 1, 2>>), <<0, 0, 1, 2>>), + ?_assertEqual(rtrim(<<1, 2, 0, 0>>), <<1, 2>>), + ?_assertEqual(rtrim(<<0, 1, 2, 0>>), <<0, 1, 2>>), + ?_assertEqual(rtrim(<<0, 0, 0, 1, 2, 0, 0, 0>>), <<0, 0, 0, 1, 2>>) + ]. + +trim2_test_() -> + [ + ?_assertEqual(trim(<<5>>, 5), <<>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, 5), <<1, 2>>), + ?_assertEqual(trim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<1, 2, 0, 0, 0>>) + ]. + +ltrim2_test_() -> + [ + ?_assertEqual(ltrim(<<5>>, 5), <<>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, 5), <<1, 2, 5>>), + ?_assertEqual(ltrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<1, 2, 0, 0, 0>>) + ]. + +rtrim2_test_() -> + [ + ?_assertEqual(rtrim(<<5>>, 5), <<>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, 5), <<5, 1, 2>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, 5), <<5, 5, 5, 1, 2, 0, 0, 0>>) + ]. + +mtrim2_test_() -> + [ + ?_assertEqual(trim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, [1, 5]), <<2>>), + ?_assertEqual(trim(<<5, 1, 2, 5>>, [1, 2, 5]), <<>>), + ?_assertEqual(trim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<2, 0, 0, 0>>) + ]. + +mltrim2_test_() -> + [ + ?_assertEqual(ltrim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, [1, 5]), <<2, 5>>), + ?_assertEqual(ltrim(<<5, 1, 2, 5>>, [2, 5]), <<1, 2, 5>>), + ?_assertEqual(ltrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<2, 0, 0, 0>>) + ]. + +mrtrim2_test_() -> + [ + ?_assertEqual(rtrim(<<5>>, [1, 5]), <<>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, [1, 5]), <<5, 1, 2>>), + ?_assertEqual(rtrim(<<5, 1, 2, 5>>, [2, 5]), <<5, 1>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [1, 5]), <<5, 5, 5, 1, 2, 0, 0, 0>>), + ?_assertEqual(rtrim(<<5, 5, 5, 1, 2, 0, 0, 0>>, [0, 5]), <<5, 5, 5, 1, 2>>) + ]. + +reverse_test_() -> + [?_assertEqual(reverse(<<0, 1, 2>>), <<2, 1, 0>>)]. + +join_test_() -> + [ + ?_assertEqual(join([<<1, 2>>, <<3, 4>>, <<5, 6>>], <<0>>), <<1, 2, 0, 3, 4, 0, 5, 6>>), + ?_assertEqual( + join([<<"abc">>, <<"def">>, <<"xyz">>], <<"|">>), + <<"abc|def|xyz">> + ), + ?_assertEqual( + join([<<>>, <<"|">>, <<"x|z">>], <<"|">>), + <<"|||x|z">> + ), + ?_assertEqual( + join([<<"abc">>, <<"def">>, <<"xyz">>], <<>>), + <<"abcdefxyz">> + ), + ?_assertEqual(join([], <<"|">>), <<>>) + ]. + +duplicate_test_() -> + [ + ?_assertEqual(duplicate(5, <<1, 2>>), <<1, 2, 1, 2, 1, 2, 1, 2, 1, 2>>), + ?_assertEqual(duplicate(50, <<0>>), <<0:400>>) + ]. + +suffix_test_() -> + [ + ?_assertEqual(suffix(<<1, 2, 3, 4, 5>>, 2), <<4, 5>>), + ?_assertError(badarg, prefix(<<1, 2, 3, 4, 5>>, 25)) + ]. + +prefix_test_() -> + [ + ?_assertEqual(prefix(<<1, 2, 3, 4, 5>>, 2), <<1, 2>>), + ?_assertError(badarg, prefix(<<1, 2, 3, 4, 5>>, 25)) + ]. + +union_test_() -> + [ + ?_assertEqual( + union( + <<2#0011011:7>>, + <<2#1011110:7>> + ), + <<2#1011111:7>> + ) + ]. + +inverse_test_() -> + [ + ?_assertEqual(inverse(inverse(<<0, 1, 2>>)), <<0, 1, 2>>), + ?_assertEqual(inverse(<<0>>), <<255>>), + ?_assertEqual(inverse(<<2#1:1>>), <<2#0:1>>), + ?_assertEqual(inverse(<<2#0:1>>), <<2#1:1>>), + ?_assertEqual( + inverse(<<2#01:2>>), + <<2#10:2>> + ), + ?_assertEqual( + inverse(<<2#0011011:7>>), + <<2#1100100:7>> + ) + ]. + +intersection_test_() -> + [ + ?_assertEqual( + intersection( + <<2#0011011>>, + <<2#1011110>> + ), + <<2#0011010>> + ) + ]. + +subtract_test_() -> + [ + ?_assertEqual( + subtract( + <<2#0011011>>, + <<2#1011110>> + ), + <<2#0000001>> + ) + ]. + +optimize_patterns_test_() -> + [ + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"t">>, <<"test">>]) + ), + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"t">>, <<"t">>, <<"test">>]) + ), + ?_assertEqual( + [<<"t">>], + optimize_patterns([<<"test">>, <<"t">>, <<"t">>]) + ) + ]. + +-endif. From 92ca2f66f5a648799a29838a2db7fcab2a94f252 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 12:06:52 +0200 Subject: [PATCH 124/279] style: add original copyright header --- apps/emqx_utils/src/emqx_utils_binary.erl | 4 ++-- apps/emqx_utils/test/emqx_utils_binary_tests.erl | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/emqx_utils/src/emqx_utils_binary.erl b/apps/emqx_utils/src/emqx_utils_binary.erl index d9e504f46..26976496d 100644 --- a/apps/emqx_utils/src/emqx_utils_binary.erl +++ b/apps/emqx_utils/src/emqx_utils_binary.erl @@ -1,4 +1,6 @@ %%-------------------------------------------------------------------- +%% Original file taken from https://github.com/arcusfelis/binary2 +%% Copyright (c) 2016 Michael Uvarov %% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,8 +18,6 @@ -module(emqx_utils_binary). -%% copied from https://github.com/arcusfelis/binary2 - %% Bytes -export([ reverse/1, diff --git a/apps/emqx_utils/test/emqx_utils_binary_tests.erl b/apps/emqx_utils/test/emqx_utils_binary_tests.erl index 514e0874a..3709fb014 100644 --- a/apps/emqx_utils/test/emqx_utils_binary_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_binary_tests.erl @@ -1,3 +1,8 @@ +%%-------------------------------------------------------------------- +%% Original file taken from https://github.com/arcusfelis/binary2 +%% Copyright (c) 2016 Michael Uvarov +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% -module(emqx_utils_binary_tests). -import(emqx_utils_binary, [ From 1bad6ca67db338c939193b539aced8ac6f41cce6 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 12:13:29 +0200 Subject: [PATCH 125/279] fix: jiffy pretty print is a bit different from jsx --- apps/emqx/src/emqx_logger_jsonfmt.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_logger_jsonfmt.erl b/apps/emqx/src/emqx_logger_jsonfmt.erl index e231b76dd..0e72fd2b5 100644 --- a/apps/emqx/src/emqx_logger_jsonfmt.erl +++ b/apps/emqx/src/emqx_logger_jsonfmt.erl @@ -62,7 +62,7 @@ %% The JSON object is pretty-printed. %% NOTE: do not use this function for logging. best_effort_json(Input) -> - best_effort_json(Input, [pretty]). + best_effort_json(Input, [pretty, force_utf8]). best_effort_json(Input, Opts) -> Config = #{depth => unlimited, single_line => true}, JsonReady = best_effort_json_obj(Input, Config), @@ -378,15 +378,15 @@ p_config() -> best_effort_json_test() -> ?assertEqual( - <<"{}">>, + <<"{\n \n}">>, emqx_logger_jsonfmt:best_effort_json([]) ), ?assertEqual( - <<"{\n \"key\": []\n}">>, + <<"{\n \"key\" : [\n \n ]\n}">>, emqx_logger_jsonfmt:best_effort_json(#{key => []}) ), ?assertEqual( - <<"[\n {\n \"key\": []\n }\n]">>, + <<"[\n {\n \"key\" : [\n \n ]\n }\n]">>, emqx_logger_jsonfmt:best_effort_json([#{key => []}]) ), ok. From b5a496aacdc4b81619f74e16f73efbc65b3b1cd4 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 14:02:56 +0200 Subject: [PATCH 126/279] fix: include emqx_utils in distribution --- rebar.config.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/rebar.config.erl b/rebar.config.erl index 2ae6ffad4..b6b6a7c18 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -391,6 +391,7 @@ relx_apps(ReleaseType, Edition) -> {covertool, load}, % started by emqx_machine {system_monitor, load}, + emqx_utils, emqx_http_lib, emqx_resource, emqx_connector, From a295d0f1341e17fe73fbddb44ce29894f18f308e Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 14:29:25 +0200 Subject: [PATCH 127/279] fix: add rebar3_path_deps plugin --- apps/emqx/rebar.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index a3eff6c16..bf509f059 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -18,7 +18,7 @@ ]}. %% Deps here may duplicate with emqx.git root level rebar.config -%% but there not be any descrpancy. +%% but there may not be any discrepancy. %% This rebar.config is necessary because the app may be used as a %% `git_subdir` dependency in other projects. {deps, [ @@ -36,7 +36,7 @@ {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} ]}. -{plugins, [{rebar3_proper, "0.12.1"}]}. +{plugins, [{rebar3_proper, "0.12.1"}, rebar3_path_deps]}. {extra_src_dirs, [{"etc", [recursive]}]}. {profiles, [ {test, [ From 4de13d280079725d3822883b55f5eaff83ea6a1f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 09:31:33 -0300 Subject: [PATCH 128/279] feat(buffer_worker): change default max queue bytes to 256 MB --- apps/emqx_resource/include/emqx_resource.hrl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 7f88e1440..904eeffa5 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -88,8 +88,8 @@ -define(DEFAULT_QUEUE_SEG_SIZE, 10 * 1024 * 1024). -define(DEFAULT_QUEUE_SEG_SIZE_RAW, <<"10MB">>). --define(DEFAULT_QUEUE_SIZE, 100 * 1024 * 1024). --define(DEFAULT_QUEUE_SIZE_RAW, <<"100MB">>). +-define(DEFAULT_QUEUE_SIZE, 256 * 1024 * 1024). +-define(DEFAULT_QUEUE_SIZE_RAW, <<"256MB">>). -define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(15)). From 55661faac66a2787e29db0156fd2ad8cabf571dc Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 14:44:54 +0200 Subject: [PATCH 129/279] fix: make elixir happy: --- mix.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/mix.exs b/mix.exs index 582860039..299f23ce6 100644 --- a/mix.exs +++ b/mix.exs @@ -303,6 +303,7 @@ defmodule EMQXUmbrella.MixProject do tools: :load, covertool: :load, system_monitor: :load, + emqx_utils: :permanent, emqx_http_lib: :permanent, emqx_resource: :permanent, emqx_connector: :permanent, From 180f5717659228a46617fa31f6f7666ce58ccd5f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 3 Apr 2023 09:12:23 +0800 Subject: [PATCH 130/279] feat: configuration priority ENV > emqx.conf > API --- apps/emqx/src/config/emqx_config_logger.erl | 23 +-- apps/emqx/src/emqx_config.erl | 189 ++++++++++++++++-- apps/emqx/src/emqx_config_handler.erl | 14 +- apps/emqx/test/emqx_config_handler_SUITE.erl | 15 +- apps/emqx_conf/src/emqx_conf_schema.erl | 4 +- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 8 +- .../src/emqx_mgmt_api_configs.erl | 138 +++++-------- .../test/emqx_mgmt_api_configs_SUITE.erl | 53 +++-- 8 files changed, 289 insertions(+), 155 deletions(-) diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index 15e4d3959..b0fc1ca67 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -32,25 +32,15 @@ remove_handler() -> ok = emqx_config_handler:remove_handler(?LOG), ok. -%% refresh logger config when booting, the override config may have changed after node start. +%% refresh logger config when booting, the cluster config may have changed after node start. %% Kernel's app env is confirmed before the node starts, -%% but we only copy cluster-override.conf from other node after this node starts, +%% but we only copy cluster.conf from other node after this node starts, %% so we need to refresh the logger config after this node starts. -%% It will not affect the logger config when cluster-override.conf is unchanged. +%% It will not affect the logger config when cluster.conf is unchanged. refresh_config() -> - Overrides = emqx_config:read_override_confs(), - refresh_config(Overrides). - -refresh_config(#{<<"log">> := _}) -> %% read the checked config LogConfig = emqx:get_config(?LOG, undefined), - Conf = #{log => LogConfig}, - ok = do_refresh_config(Conf); -refresh_config(_) -> - %% No config override found for 'log', do nothing - %% because the 'kernel' app should already be configured - %% from the base configs. i.e. emqx.conf + env vars - ok. + do_refresh_config(#{log => LogConfig}). %% this call is shared between initial config refresh at boot %% and dynamic config update from HTTP API @@ -61,10 +51,9 @@ do_refresh_config(Conf) -> ok = maybe_update_log_level(Level), ok. +%% always refresh config when the override config is changed post_config_update(?LOG, _Req, NewConf, _OldConf, _AppEnvs) -> - ok = do_refresh_config(#{log => NewConf}); -post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) -> - ok. + do_refresh_config(#{log => NewConf}). maybe_update_log_level(NewLevel) -> OldLevel = emqx_logger:get_primary_log_level(), diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index bf3134568..41e38f184 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -24,17 +24,19 @@ init_load/2, init_load/3, read_override_conf/1, - read_override_confs/0, delete_override_conf_files/0, check_config/2, fill_defaults/1, fill_defaults/2, fill_defaults/3, - save_configs/5, + save_configs/6, save_to_app_env/1, save_to_config_map/2, save_to_override_conf/2 ]). +-export([raw_conf_with_default/4]). +-export([remove_default_conf/2]). +-export([merge_envs/2]). -export([ get_root/1, @@ -329,23 +331,15 @@ init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> init_load(SchemaMod, parse_hocon(Conf), Opts); init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), + RootNames = get_root_names(), %% Merge environment variable overrides on top RawConfWithEnvs = merge_envs(SchemaMod, RawConf), - Overrides = read_override_confs(), - RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), - RootNames = get_root_names(), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), %% check configs against the schema {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), save_to_app_env(AppEnvs), ok = save_to_config_map(CheckedConf, RawConfAll). -%% @doc Read merged cluster + local overrides. -read_override_confs() -> - ClusterOverrides = read_override_conf(#{override_to => cluster}), - LocalOverrides = read_override_conf(#{override_to => local}), - hocon:deep_merge(ClusterOverrides, LocalOverrides). - %% keep the raw and non-raw conf has the same keys to make update raw conf easier. raw_conf_with_default(SchemaMod, RootNames, RawConf, #{raw_with_default := true}) -> Fun = fun(Name, Acc) -> @@ -375,6 +369,9 @@ schema_default(Schema) -> end. parse_hocon(Conf) -> + %% merge cluster-override.conf to local-override.conf + %% cluster-override.conf is deprecated, now is cluster.conf + merge_deprecated_cluster_override_to_local_override(), IncDirs = include_dirs(), case do_parse_hocon(Conf, IncDirs) of {ok, HoconMap} -> @@ -393,8 +390,12 @@ parse_hocon(Conf) -> do_parse_hocon(Conf, IncDirs) -> Opts = #{format => map, include_dirs => IncDirs}, case is_binary(Conf) of - true -> hocon:binary(Conf, Opts); - false -> hocon:files(Conf, Opts) + true -> + hocon:binary(Conf, Opts); + false -> + LocalFile = override_conf_file(#{override_to => local}), + ClusterFile = override_conf_file(#{override_to => cluster}), + hocon:files([ClusterFile] ++ Conf ++ [LocalFile], Opts) end. include_dirs() -> @@ -483,6 +484,13 @@ read_override_conf(#{} = Opts) -> File = override_conf_file(Opts), load_hocon_file(File, map). +read_deprecated_override_conf() -> + ClusterFile = override_conf_file(#{override_to => cluster}), + DeprecatedFile = filename:join(filename:dirname(ClusterFile), "cluster-override.conf"), + Conf = load_hocon_file(DeprecatedFile, map), + _ = file:delete(DeprecatedFile), + Conf. + override_conf_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of @@ -522,14 +530,54 @@ get_schema_mod(RootName) -> get_root_names() -> maps:get(names, persistent_term:get(?PERSIS_SCHEMA_MODS, #{names => []})). --spec save_configs(app_envs(), config(), raw_config(), raw_config(), update_opts()) -> ok. -save_configs(AppEnvs, Conf, RawConf, OverrideConf, Opts) -> - %% We first try to save to override.conf, because saving to files is more error prone +-spec save_configs( + emqx_map_lib:config_key_path(), app_envs(), config(), raw_config(), raw_config(), update_opts() +) -> ok. + +save_configs(Paths0, AppEnvs, Conf, RawConf, OverrideConf, Opts) -> + Default = init_default(Paths0), + OverrideConf1 = remove_default_conf(OverrideConf, Default), + %% We first try to save to files, because saving to files is more error prone %% than saving into memory. - ok = save_to_override_conf(OverrideConf, Opts), + ok = save_to_override_conf(OverrideConf1, Opts), save_to_app_env(AppEnvs), save_to_config_map(Conf, RawConf). +init_default(Paths0) -> + [Root | _] = [bin(Key) || Key <- Paths0], + SchemaMod = get_schema_mod(Root), + {_, {_, Schema}} = lists:keyfind(Root, 1, hocon_schema:roots(SchemaMod)), + fill_defaults(#{Root => schema_default(Schema)}). + +remove_default_conf(undefined, _) -> + undefined; +remove_default_conf(Conf, DefaultConf) when is_map(Conf) andalso is_map(DefaultConf) -> + maps:fold( + fun(Key, Value, Acc) -> + case maps:find(Key, DefaultConf) of + {ok, DefaultValue} -> + remove_default_conf(Value, DefaultValue, Key, Acc); + error -> + Acc + end + end, + Conf, + Conf + ). + +remove_default_conf(Value, Value, Key, Conf) -> + maps:remove(Key, Conf); +remove_default_conf(Value = #{}, DefaultValue = #{}, Key, Conf) -> + case remove_default_conf(Value, DefaultValue) of + SubValue when SubValue =:= #{} -> maps:remove(Key, Conf); + SubValue -> maps:put(Key, SubValue, Conf) + end; +remove_default_conf(Value, DefaultValue, Key, Conf) -> + case try_bin(DefaultValue) =:= try_bin(Value) of + true -> maps:remove(Key, Conf); + false -> Conf + end. + %% we ignore kernel app env, %% because the old app env will be used in emqx_config_logger:post_config_update/5 -define(IGNORE_APPS, [kernel]). @@ -678,6 +726,16 @@ atom(Str) when is_list(Str) -> atom(Atom) when is_atom(Atom) -> Atom. +try_bin(Bin) when is_binary(Bin) -> Bin; +try_bin([Bin | _] = List) when is_binary(Bin) -> List; +try_bin([Atom | _] = List) when is_atom(Atom) -> [atom_to_binary(A) || A <- List]; +try_bin([Map | _] = Maps) when is_map(Map) -> Maps; +try_bin(Str) when is_list(Str) -> list_to_binary(Str); +try_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); +try_bin(Int) when is_integer(Int) -> integer_to_binary(Int); +try_bin(Float) when is_float(Float) -> float_to_binary(Float); +try_bin(Term) -> Term. + bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). @@ -699,3 +757,98 @@ atom_conf_path(Path, ExpFun, OnFail) -> error(Err) end end. + +merge_deprecated_cluster_override_to_local_override() -> + case read_deprecated_override_conf() of + DeprecatedConf when DeprecatedConf =/= #{} -> + LocalOverrides = read_override_conf(#{override_to => local}), + MergedConf = hocon:deep_merge(LocalOverrides, DeprecatedConf), + _ = save_to_override_conf(MergedConf, #{override_to => local}), + ok; + _ -> + ok + end. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +remove_default_conf_test() -> + ?assertEqual( + #{}, + remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => 100}) + ), + ?assertEqual( + #{}, + remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => <<"100">>}) + ), + ?assertEqual( + #{<<"def">> => 100}, + remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => #{<<"bar">> => 100}}) + ), + ?assertEqual( + #{<<"def">> => #{<<"edf">> => 321}}, + remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => 321}}, #{ + <<"def">> => #{<<"abc">> => 100, <<"edf">> => 123} + }) + ), + ?assertEqual( + #{}, + remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => 321}}, #{ + <<"def">> => #{<<"abc">> => <<"100">>, <<"edf">> => 321} + }) + ), + ?assertEqual( + #{}, + remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => <<"true">>}}, #{ + <<"def">> => #{<<"abc">> => <<"100">>, <<"edf">> => true} + }) + ), + ?assertEqual( + #{}, + remove_default_conf( + #{ + <<"bytes_in">> => + #{ + <<"capacity">> => infinity, + <<"initial">> => 0, + <<"rate">> => infinity + } + }, + #{ + <<"bytes_in">> => + #{ + <<"capacity">> => <<"infinity">>, + <<"initial">> => <<"0">>, + <<"rate">> => <<"infinity">> + } + } + ) + ), + ?assertEqual( + #{}, + remove_default_conf( + #{ + <<"limiter">> => #{ + <<"connection">> => + #{ + <<"capacity">> => 1000, + <<"initial">> => <<"0">>, + <<"rate">> => <<"1000/s">> + } + } + }, + #{ + <<"limiter">> => #{ + <<"connection">> => + #{ + <<"capacity">> => <<"1000">>, + <<"initial">> => <<"0">>, + <<"rate">> => <<"1000/s">> + } + } + } + ) + ), + ok. + +-endif. diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index a0a99b62e..ee22c297e 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -271,8 +271,10 @@ do_update_config( SubOldRawConf = get_sub_config(ConfKeyBin, OldRawConf), SubHandlers = get_sub_handlers(ConfKey, Handlers), case do_update_config(SubConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq, ConfKeyPath) of - {ok, NewUpdateReq} -> merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf); - Error -> Error + {ok, NewUpdateReq} -> + merge_to_old_config(#{ConfKeyBin => NewUpdateReq}, OldRawConf); + Error -> + Error end. check_and_save_configs( @@ -289,7 +291,9 @@ check_and_save_configs( OldConf = emqx_config:get_root(ConfKeyPath), case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of {ok, Result0} -> - ok = emqx_config:save_configs(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts), + ok = emqx_config:save_configs( + ConfKeyPath, AppEnvs, NewConf, NewRawConf, OverrideConf, Opts + ), Result1 = return_change_result(ConfKeyPath, UpdateArgs), {ok, Result1#{post_config_update => Result0}}; Error -> @@ -560,11 +564,11 @@ check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) -> allow; {error, Error} -> ?SLOG(error, #{ - msg => "prevent_remove_local_override_conf", + msg => "prevent_remove_local_conf", config_key_path => ConfKeyPath, error => Error }), - {deny, "Disable changed from local-override.conf"} + {deny, "Disable changed from local conf"} end; {not_found, _, _} -> allow diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index 8126b35c6..6e2dbfef1 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -21,8 +21,8 @@ -define(MOD, {mod}). -define(WKEY, '?'). --define(LOCAL_CONF, "/tmp/local-override.conf"). --define(CLUSTER_CONF, "/tmp/cluster-override.conf"). +-define(LOCAL_CONF, "/tmp/local.conf"). +-define(CLUSTER_CONF, "/tmp/cluster.conf"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -221,12 +221,6 @@ t_local_override_update_remove(_Config) -> ), ?assertMatch(0.7, emqx:get_config(KeyPath)), - KeyPath2 = [sysmon, os, cpu_low_watermark], - ok = emqx_config_handler:add_handler(KeyPath2, ?MODULE), - ?assertMatch( - {error, {permission_denied, _}}, emqx:update_config(KeyPath2, <<"40%">>, ClusterOpts) - ), - %% remove ?assertMatch({error, {permission_denied, _}}, emqx:remove_config(KeyPath)), ?assertEqual( @@ -251,7 +245,6 @@ t_local_override_update_remove(_Config) -> ?assert(length(OSKey1) > 1), ok = emqx_config_handler:remove_handler(KeyPath), - ok = emqx_config_handler:remove_handler(KeyPath2), application:unset_env(emqx, local_override_conf_file), application:unset_env(emqx, cluster_override_conf_file), ok. @@ -426,9 +419,9 @@ wait_for_new_pid() -> callback_error(FailedPath, Update, Error) -> Opts = #{rawconf_with_defaults => true}, ok = emqx_config_handler:add_handler(FailedPath, ?MODULE), - Old = emqx:get_raw_config(FailedPath), + Old = emqx:get_raw_config(FailedPath, undefined), ?assertEqual(Error, emqx:update_config(FailedPath, Update, Opts)), - New = emqx:get_raw_config(FailedPath), + New = emqx:get_raw_config(FailedPath, undefined), ?assertEqual(Old, New), ok = emqx_config_handler:remove_handler(FailedPath), ok. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index f689997c2..ed246b4ae 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1076,10 +1076,10 @@ tr_config_files(_Conf) -> end. tr_cluster_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "cluster-override.conf"). + tr_override_conf_file(Conf, "cluster.conf"). tr_local_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "local-override.conf"). + tr_override_conf_file(Conf, "local.conf"). tr_override_conf_file(Conf, Filename) -> DataDir = conf_get("node.data_dir", Conf), diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index c34eb9dc3..eb67490bc 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -85,7 +85,7 @@ create_data_dir() -> ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)), ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)), Telemetry = <<"telemetry.enable = false">>, - ok = file:write_file(Node ++ "/configs/cluster-override.conf", Telemetry). + ok = file:write_file(Node ++ "/configs/cluster.hocon", Telemetry). set_data_dir_env() -> Node = atom_to_list(node()), @@ -100,14 +100,14 @@ set_data_dir_env() -> ok = file:write_file(NewConfigFile, DataDir, [append]), application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), - application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"), + application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster.hocon"), ok. assert_data_copy_done([First0 | Rest]) -> First = atom_to_list(First0), {ok, FakeCertFile} = file:read_file(First ++ "/certs/fake-cert"), {ok, FakeAuthzFile} = file:read_file(First ++ "/authz/fake-authz"), - {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster-override.conf"), + {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster.hocon"), lists:foreach( fun(Node0) -> Node = atom_to_list(Node0), @@ -118,7 +118,7 @@ assert_data_copy_done([First0 | Rest]) -> ), ?assertEqual( {ok, FakeOverrideFile}, - file:read_file(Node ++ "/configs/cluster-override.conf"), + file:read_file(Node ++ "/configs/cluster.hocon"), #{node => Node} ), ?assertEqual( diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 55cc50597..56572b41e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -31,41 +31,23 @@ global_zone_configs/3 ]). --export([gen_schema/1]). - -define(PREFIX, "/configs/"). -define(PREFIX_RESET, "/configs_reset/"). -define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))). -define(OPTS, #{rawconf_with_defaults => true, override_to => cluster}). -define(TAGS, ["Configs"]). --define(EXCLUDES, - [ - <<"exhook">>, - <<"gateway">>, - <<"plugins">>, - <<"bridges">>, - <<"rule_engine">>, - <<"authorization">>, - <<"authentication">>, - <<"rpc">>, - <<"connectors">>, - <<"slow_subs">>, - <<"psk_authentication">>, - <<"topic_metrics">>, - <<"rewrite">>, - <<"auto_subscribe">>, - <<"retainer">>, - <<"statsd">>, - <<"delayed">>, - <<"event_message">>, - <<"prometheus">>, - <<"telemetry">>, - <<"listeners">>, - <<"license">>, - <<"api_key">> - ] ++ global_zone_roots() -). +-define(ROOT_KEYS, [ + <<"dashboard">>, + <<"alarm">>, + <<"sys_topics">>, + <<"sysmon">>, + <<"limiter">>, + <<"trace">>, + <<"log">>, + <<"persistent_session_store">>, + <<"zones">> +]). api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -119,7 +101,6 @@ schema("/configs_reset/:rootname") -> "- For a config entry that has default value, this resets it to the default value;\n" "- For a config entry that has no default value, an error 400 will be returned" >>, - summary => <<"Reset config entry">>, %% We only return "200" rather than the new configs that has been changed, as %% the schema of the changed configs is depends on the request parameter %% `conf_path`, it cannot be defined here. @@ -214,12 +195,11 @@ fields(Field) -> %%%============================================================================================== %% HTTP API Callbacks config(get, _Params, Req) -> + [Path] = conf_path(Req), + {200, get_raw_config(Path)}; +config(put, #{body := NewConf}, Req) -> Path = conf_path(Req), - {ok, Conf} = emqx_map_lib:deep_find(Path, get_full_config()), - {200, Conf}; -config(put, #{body := Body}, Req) -> - Path = conf_path(Req), - case emqx_conf:update(Path, Body, ?OPTS) of + case emqx_conf:update(Path, NewConf, ?OPTS) of {ok, #{raw_config := RawConf}} -> {200, RawConf}; {error, {permission_denied, Reason}} -> @@ -229,28 +209,29 @@ config(put, #{body := Body}, Req) -> end. global_zone_configs(get, _Params, _Req) -> - Paths = global_zone_roots(), - Zones = lists:foldl( - fun(Path, Acc) -> maps:merge(Acc, get_config_with_default(Path)) end, - #{}, - Paths - ), - {200, Zones}; + {200, get_zones()}; global_zone_configs(put, #{body := Body}, _Req) -> + PrevZones = get_zones(), Res = maps:fold( fun(Path, Value, Acc) -> - case emqx_conf:update([Path], Value, ?OPTS) of - {ok, #{raw_config := RawConf}} -> - Acc#{Path => RawConf}; - {error, Reason} -> - ?SLOG(error, #{ - msg => "update global zone failed", - reason => Reason, - path => Path, - value => Value - }), - Acc + PrevValue = maps:get(Path, PrevZones), + case Value =/= PrevValue of + true -> + case emqx_conf:update([Path], Value, ?OPTS) of + {ok, #{raw_config := RawConf}} -> + Acc#{Path => RawConf}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "update global zone failed", + reason => Reason, + path => Path, + value => Value + }), + Acc + end; + false -> + Acc#{Path => Value} end end, #{}, @@ -298,13 +279,30 @@ conf_path_reset(Req) -> get_full_config() -> emqx_config:fill_defaults( - maps:without( - ?EXCLUDES, + maps:with( + ?ROOT_KEYS, emqx:get_raw_config([]) ), #{obfuscate_sensitive_values => true} ). +get_raw_config(Path) -> + #{Path := Conf} = + emqx_config:fill_defaults( + #{Path => emqx:get_raw_config([Path])}, + #{obfuscate_sensitive_values => true} + ), + Conf. + +get_zones() -> + lists:foldl( + fun(Path, Acc) -> + maps:merge(Acc, get_config_with_default(Path)) + end, + #{}, + global_zone_roots() + ). + get_config_with_default(Path) -> emqx_config:fill_defaults(#{Path => emqx:get_raw_config([Path])}). @@ -317,40 +315,12 @@ conf_path_from_querystr(Req) -> config_list() -> Mod = emqx_conf:schema_module(), Roots = hocon_schema:roots(Mod), - lists:foldl(fun(Key, Acc) -> lists:keydelete(Key, 1, Acc) end, Roots, ?EXCLUDES). + lists:foldl(fun(Key, Acc) -> [lists:keyfind(Key, 1, Roots) | Acc] end, [], ?ROOT_KEYS). conf_path(Req) -> <<"/api/v5", ?PREFIX, Path/binary>> = cowboy_req:path(Req), string:lexemes(Path, "/ "). -%% TODO: generate from hocon schema -gen_schema(Conf) when is_boolean(Conf) -> - with_default_value(#{type => boolean}, Conf); -gen_schema(Conf) when is_binary(Conf); is_atom(Conf) -> - with_default_value(#{type => string}, Conf); -gen_schema(Conf) when is_number(Conf) -> - with_default_value(#{type => number}, Conf); -gen_schema(Conf) when is_list(Conf) -> - case io_lib:printable_unicode_list(Conf) of - true -> - gen_schema(unicode:characters_to_binary(Conf)); - false -> - #{type => array, items => gen_schema(hd(Conf))} - end; -gen_schema(Conf) when is_map(Conf) -> - #{ - type => object, - properties => - maps:map(fun(_K, V) -> gen_schema(V) end, Conf) - }; -gen_schema(_Conf) -> - %% the conf is not of JSON supported type, it may have been converted - %% by the hocon schema - #{type => string}. - -with_default_value(Type, Value) -> - Type#{example => emqx_map_lib:binary_string(Value)}. - global_zone_roots() -> lists:map(fun({K, _}) -> K end, global_zone_schema()). diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 2d24bce99..f5ee4e5be 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -55,12 +55,17 @@ t_update(_Config) -> %% update ok {ok, SysMon} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort}} = SysMon, - NewSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, not BusyPort), + NewSysMon = #{<<"vm">> => #{<<"busy_port">> => not BusyPort}}, {ok, #{}} = update_config(<<"sysmon">>, NewSysMon), {ok, SysMon1} = get_config(<<"sysmon">>), #{<<"vm">> := #{<<"busy_port">> := BusyPort1}} = SysMon1, ?assertEqual(BusyPort, not BusyPort1), assert_busy_port(BusyPort1), + %% Make sure the override config is updated, and remove the default value. + ?assertEqual( + #{<<"vm">> => #{<<"busy_port">> => BusyPort1}}, + maps:get(<<"sysmon">>, emqx_config:read_override_conf(#{override_to => cluster})) + ), %% update failed ErrorSysMon = emqx_map_lib:deep_put([<<"vm">>, <<"busy_port">>], SysMon, "123"), @@ -130,6 +135,8 @@ t_global_zone(_Config) -> NewZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), {ok, #{}} = update_global_zone(NewZones), ?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])), + %% Make sure the override config is updated, and remove the default value. + ?assertEqual(#{<<"max_qos_allowed">> => 1}, read_conf(<<"mqtt">>)), BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)), @@ -145,6 +152,10 @@ t_global_zone(_Config) -> %% the default value is 2 ?assertEqual(2, emqx_map_lib:deep_get([<<"max_qos_allowed">>], Mqtt3)), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0), + + DefaultZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2), + {ok, #{}} = update_global_zone(DefaultZones), + ?assertEqual(undefined, read_conf(<<"mqtt">>)), ok. get_global_zone() -> @@ -169,7 +180,7 @@ t_dashboard(_Config) -> Https1 = #{enable => true, bind => 18084}, ?assertMatch( {error, {"HTTP/1.1", 400, _}}, - update_config("dashboard", Dashboard#{<<"https">> => Https1}) + update_config("dashboard", Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https1}}) ), Https2 = #{ @@ -179,35 +190,41 @@ t_dashboard(_Config) -> cacertfile => "etc/certs/badcacert.pem", certfile => "etc/certs/badcert.pem" }, - Dashboard2 = Dashboard#{listeners => Listeners#{https => Https2}}, + Dashboard2 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https2}}, ?assertMatch( {error, {"HTTP/1.1", 400, _}}, update_config("dashboard", Dashboard2) ), - Keyfile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "key.pem"])), - Certfile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "cert.pem"])), - Cacertfile = emqx_common_test_helpers:app_path( + KeyFile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "key.pem"])), + CertFile = emqx_common_test_helpers:app_path(emqx, filename:join(["etc", "certs", "cert.pem"])), + CacertFile = emqx_common_test_helpers:app_path( emqx, filename:join(["etc", "certs", "cacert.pem"]) ), Https3 = #{ - enable => true, - bind => 18084, - keyfile => Keyfile, - cacertfile => Cacertfile, - certfile => Certfile + <<"enable">> => true, + <<"bind">> => 18084, + <<"keyfile">> => list_to_binary(KeyFile), + <<"cacertfile">> => list_to_binary(CacertFile), + <<"certfile">> => list_to_binary(CertFile) }, - Dashboard3 = Dashboard#{listeners => Listeners#{https => Https3}}, + Dashboard3 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => Https3}}, ?assertMatch({ok, _}, update_config("dashboard", Dashboard3)), - Dashboard4 = Dashboard#{listeners => Listeners#{https => #{enable => false}}}, + Dashboard4 = Dashboard#{<<"listeners">> => Listeners#{<<"https">> => #{<<"enable">> => false}}}, ?assertMatch({ok, _}, update_config("dashboard", Dashboard4)), + {ok, Dashboard41} = get_config("dashboard"), + ?assertEqual( + Https3#{<<"enable">> => false}, + read_conf([<<"dashboard">>, <<"listeners">>, <<"https">>]), + Dashboard41 + ), ?assertMatch({ok, _}, update_config("dashboard", Dashboard)), {ok, Dashboard1} = get_config("dashboard"), ?assertNotEqual(Dashboard, Dashboard1), - timer:sleep(1000), + timer:sleep(1500), ok. t_configs_node({'init', Config}) -> @@ -288,3 +305,11 @@ reset_config(Name, Key) -> {ok, []} -> ok; Error -> Error end. + +read_conf(RootKeys) when is_list(RootKeys) -> + case emqx_config:read_override_conf(#{override_to => cluster}) of + undefined -> undefined; + Conf -> emqx_map_lib:deep_get(RootKeys, Conf, undefined) + end; +read_conf(RootKey) -> + read_conf([RootKey]). From 5f4ea3b6d8621ac667b7af121fac452617cea6c7 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 7 Apr 2023 09:17:54 +0800 Subject: [PATCH 131/279] feat: deprecated cluster-override.conf --- apps/emqx/src/emqx_config.erl | 142 ++++++++++++++++-------- apps/emqx_conf/src/emqx_conf_app.erl | 29 +++-- apps/emqx_conf/src/emqx_conf_schema.erl | 16 ++- 3 files changed, 127 insertions(+), 60 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 41e38f184..68a8563d8 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -24,6 +24,7 @@ init_load/2, init_load/3, read_override_conf/1, + has_deprecated_conf/0, delete_override_conf_files/0, check_config/2, fill_defaults/1, @@ -32,7 +33,7 @@ save_configs/6, save_to_app_env/1, save_to_config_map/2, - save_to_override_conf/2 + save_to_override_conf/3 ]). -export([raw_conf_with_default/4]). -export([remove_default_conf/2]). @@ -331,14 +332,35 @@ init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> init_load(SchemaMod, parse_hocon(Conf), Opts); init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), - RootNames = get_root_names(), - %% Merge environment variable overrides on top - RawConfWithEnvs = merge_envs(SchemaMod, RawConf), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), - %% check configs against the schema - {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), - save_to_app_env(AppEnvs), - ok = save_to_config_map(CheckedConf, RawConfAll). + case has_deprecated_conf() of + false -> + RootNames = get_root_names(), + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll); + true -> + %% deprecated conf will be removed in 5.1 + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + Overrides = read_override_confs(), + RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), + RootNames = get_root_names(), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll) + end. + +%% @doc Read merged cluster + local overrides. +read_override_confs() -> + ClusterOverrides = read_override_conf(#{override_to => cluster}), + LocalOverrides = read_override_conf(#{override_to => local}), + hocon:deep_merge(ClusterOverrides, LocalOverrides). %% keep the raw and non-raw conf has the same keys to make update raw conf easier. raw_conf_with_default(SchemaMod, RootNames, RawConf, #{raw_with_default := true}) -> @@ -369,11 +391,9 @@ schema_default(Schema) -> end. parse_hocon(Conf) -> - %% merge cluster-override.conf to local-override.conf - %% cluster-override.conf is deprecated, now is cluster.conf - merge_deprecated_cluster_override_to_local_override(), + HasDeprecatedConf = has_deprecated_conf(), IncDirs = include_dirs(), - case do_parse_hocon(Conf, IncDirs) of + case do_parse_hocon(HasDeprecatedConf, Conf, IncDirs) of {ok, HoconMap} -> HoconMap; {error, Reason} -> @@ -387,14 +407,20 @@ parse_hocon(Conf) -> error(failed_to_load_hocon_conf) end. -do_parse_hocon(Conf, IncDirs) -> +do_parse_hocon(true, Conf, IncDirs) -> + Opts = #{format => map, include_dirs => IncDirs}, + case is_binary(Conf) of + true -> hocon:binary(Conf, Opts); + false -> hocon:files(Conf, Opts) + end; +do_parse_hocon(false, Conf, IncDirs) -> Opts = #{format => map, include_dirs => IncDirs}, case is_binary(Conf) of true -> hocon:binary(Conf, Opts); false -> - LocalFile = override_conf_file(#{override_to => local}), - ClusterFile = override_conf_file(#{override_to => cluster}), + LocalFile = deprecated_override_conf_file(#{override_to => local}), + ClusterFile = deprecated_override_conf_file(#{override_to => cluster}), hocon:files([ClusterFile] ++ Conf ++ [LocalFile], Opts) end. @@ -467,10 +493,14 @@ fill_defaults(SchemaMod, RawConf, Opts0) -> %% Delete override config files. -spec delete_override_conf_files() -> ok. delete_override_conf_files() -> - F1 = override_conf_file(#{override_to => local}), - F2 = override_conf_file(#{override_to => cluster}), + F1 = deprecated_override_conf_file(#{override_to => local}), + F2 = deprecated_override_conf_file(#{override_to => cluster}), + F3 = conf_file(#{override_to => local}), + F4 = conf_file(#{override_to => cluster}), ok = ensure_file_deleted(F1), - ok = ensure_file_deleted(F2). + ok = ensure_file_deleted(F2), + ok = ensure_file_deleted(F3), + ok = ensure_file_deleted(F4). ensure_file_deleted(F) -> case file:delete(F) of @@ -481,24 +511,39 @@ ensure_file_deleted(F) -> -spec read_override_conf(map()) -> raw_config(). read_override_conf(#{} = Opts) -> - File = override_conf_file(Opts), + File = + case has_deprecated_conf() of + true -> deprecated_override_conf_file(Opts); + false -> conf_file(Opts) + end, load_hocon_file(File, map). -read_deprecated_override_conf() -> - ClusterFile = override_conf_file(#{override_to => cluster}), - DeprecatedFile = filename:join(filename:dirname(ClusterFile), "cluster-override.conf"), - Conf = load_hocon_file(DeprecatedFile, map), - _ = file:delete(DeprecatedFile), - Conf. +has_deprecated_conf() -> + DeprecatedFile = deprecated_cluster_conf_file(), + filelib:is_regular(DeprecatedFile). -override_conf_file(Opts) when is_map(Opts) -> +deprecated_cluster_conf_file() -> + ClusterFile = deprecated_override_conf_file(#{override_to => cluster}), + filename:join(filename:dirname(ClusterFile), "cluster-override.conf"). + +deprecated_override_conf_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of local -> local_override_conf_file; cluster -> cluster_override_conf_file end, application:get_env(emqx, Key, undefined); -override_conf_file(Which) when is_atom(Which) -> +deprecated_override_conf_file(Which) when is_atom(Which) -> + application:get_env(emqx, Which, undefined). + +conf_file(Opts) when is_map(Opts) -> + Key = + case maps:get(override_to, Opts, cluster) of + local -> local_conf_file; + cluster -> cluster_conf_file + end, + application:get_env(emqx, Key, undefined); +conf_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). -spec save_schema_mod_and_names(module()) -> ok. @@ -539,7 +584,8 @@ save_configs(Paths0, AppEnvs, Conf, RawConf, OverrideConf, Opts) -> OverrideConf1 = remove_default_conf(OverrideConf, Default), %% We first try to save to files, because saving to files is more error prone %% than saving into memory. - ok = save_to_override_conf(OverrideConf1, Opts), + HasDeprecated = has_deprecated_conf(), + ok = save_to_override_conf(HasDeprecated, OverrideConf1, Opts), save_to_app_env(AppEnvs), save_to_config_map(Conf, RawConf). @@ -592,11 +638,12 @@ save_to_config_map(Conf, RawConf) -> ?MODULE:put(Conf), ?MODULE:put_raw(RawConf). --spec save_to_override_conf(raw_config(), update_opts()) -> ok | {error, term()}. -save_to_override_conf(undefined, _) -> +-spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}. +save_to_override_conf(_, undefined, _) -> ok; -save_to_override_conf(RawConf, Opts) -> - case override_conf_file(Opts) of +%% TODO: Remove deprecated override conf file when 5.1 +save_to_override_conf(true, RawConf, Opts) -> + case deprecated_override_conf_file(Opts) of undefined -> ok; FileName -> @@ -612,6 +659,24 @@ save_to_override_conf(RawConf, Opts) -> }), {error, Reason} end + end; +save_to_override_conf(false, RawConf, Opts) -> + case conf_file(Opts) of + undefined -> + ok; + FileName -> + ok = filelib:ensure_dir(FileName), + case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of + ok -> + ok; + {error, Reason} -> + ?SLOG(error, #{ + msg => "failed_to_save_conf_file", + filename => FileName, + reason => Reason + }), + {error, Reason} + end end. add_handlers() -> @@ -758,17 +823,6 @@ atom_conf_path(Path, ExpFun, OnFail) -> end end. -merge_deprecated_cluster_override_to_local_override() -> - case read_deprecated_override_conf() of - DeprecatedConf when DeprecatedConf =/= #{} -> - LocalOverrides = read_override_conf(#{override_to => local}), - MergedConf = hocon:deep_merge(LocalOverrides, DeprecatedConf), - _ = save_to_override_conf(MergedConf, #{override_to => local}), - ok; - _ -> - ok - end. - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 51fc5c2e2..45631619b 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -60,7 +60,14 @@ get_override_config_file() -> TnxId = emqx_cluster_rpc:get_node_tnx_id(Node), WallClock = erlang:statistics(wall_clock), Conf = emqx_config_handler:get_raw_cluster_override_conf(), - #{wall_clock => WallClock, conf => Conf, tnx_id => TnxId, node => Node} + HasDeprecateConf = emqx_config:has_deprecated_conf(), + #{ + wall_clock => WallClock, + conf => Conf, + tnx_id => TnxId, + node => Node, + has_deprecated_conf => HasDeprecateConf + } end, case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of {atomic, Res} -> {ok, Res}; @@ -153,10 +160,10 @@ copy_override_conf_from_core_node() -> {ok, ?DEFAULT_INIT_TXN_ID}; false -> %% retry in some time - Jitter = rand:uniform(2_000), - Timeout = 10_000 + Jitter, + Jitter = rand:uniform(2000), + Timeout = 10000 + Jitter, ?SLOG(info, #{ - msg => "copy_override_conf_from_core_node_retry", + msg => "copy_cluster_conf_from_core_node_retry", timeout => Timeout, nodes => Nodes, failed => Failed, @@ -168,18 +175,16 @@ copy_override_conf_from_core_node() -> _ -> [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready), #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info, + HasDeprecatedConf = maps:get(has_deprecated_conf, Info, false), ?SLOG(debug, #{ - msg => "copy_override_conf_from_core_node_success", + msg => "copy_cluster_conf_from_core_node_success", node => Node, - cluster_override_conf_file => application:get_env( - emqx, cluster_override_conf_file - ), - local_override_conf_file => application:get_env( - emqx, local_override_conf_file - ), - data_dir => emqx:data_dir() + has_deprecated_conf => HasDeprecatedConf, + data_dir => emqx:data_dir(), + tnx_id => TnxId }), ok = emqx_config:save_to_override_conf( + HasDeprecatedConf, RawOverrideConf, #{override_to => cluster} ), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index ed246b4ae..a67d2a100 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1028,7 +1028,9 @@ translation("emqx") -> [ {"config_files", fun tr_config_files/1}, {"cluster_override_conf_file", fun tr_cluster_override_conf_file/1}, - {"local_override_conf_file", fun tr_local_override_conf_file/1} + {"local_override_conf_file", fun tr_local_override_conf_file/1}, + {"cluster_conf_file", fun tr_cluster_conf_file/1}, + {"local_conf_file", fun tr_local_conf_file/1} ]; translation("gen_rpc") -> [{"default_client_driver", fun tr_default_config_driver/1}]; @@ -1076,12 +1078,18 @@ tr_config_files(_Conf) -> end. tr_cluster_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "cluster.conf"). + tr_conf_file(Conf, "cluster-override.conf"). tr_local_override_conf_file(Conf) -> - tr_override_conf_file(Conf, "local.conf"). + tr_conf_file(Conf, "local-overide.conf"). -tr_override_conf_file(Conf, Filename) -> +tr_cluster_conf_file(Conf) -> + tr_conf_file(Conf, "cluster.hocon"). + +tr_local_conf_file(Conf) -> + tr_conf_file(Conf, "local.hocon"). + +tr_conf_file(Conf, Filename) -> DataDir = conf_get("node.data_dir", Conf), %% assert, this config is not nullable [_ | _] = DataDir, From a271ba5ff9ec53741a9f3ac85dd46fe6bc9e4bae Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 10 Apr 2023 14:52:49 +0800 Subject: [PATCH 132/279] feat: don't check_permission on local.conf --- apps/emqx/src/emqx_config_handler.erl | 117 +------------------ apps/emqx/test/emqx_config_handler_SUITE.erl | 49 -------- 2 files changed, 5 insertions(+), 161 deletions(-) diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index ee22c297e..3e18a172e 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -43,7 +43,6 @@ terminate/2, code_change/3 ]). --export([is_mutable/3]). -define(MOD, {mod}). -define(WKEY, '?'). @@ -230,26 +229,15 @@ process_update_request([_], _Handlers, {remove, _Opts}) -> process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), BinKeyPath = bin_path(ConfKeyPath), - case check_permissions(remove, BinKeyPath, OldRawConf, Opts) of - allow -> - NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), - OverrideConf = remove_from_override_config(BinKeyPath, Opts), - {ok, NewRawConf, OverrideConf, Opts}; - {deny, Reason} -> - {error, {permission_denied, Reason}} - end; + NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), + OverrideConf = remove_from_override_config(BinKeyPath, Opts), + {ok, NewRawConf, OverrideConf, Opts}; process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of {ok, NewRawConf} -> - BinKeyPath = bin_path(ConfKeyPath), - case check_permissions(update, BinKeyPath, NewRawConf, Opts) of - allow -> - OverrideConf = merge_to_override_config(NewRawConf, Opts), - {ok, NewRawConf, OverrideConf, Opts}; - {deny, Reason} -> - {error, {permission_denied, Reason}} - end; + OverrideConf = merge_to_override_config(NewRawConf, Opts), + {ok, NewRawConf, OverrideConf, Opts}; Error -> Error end. @@ -550,98 +538,3 @@ load_prev_handlers() -> save_handlers(Handlers) -> application:set_env(emqx, ?MODULE, Handlers). - -check_permissions(_Action, _ConfKeyPath, _NewRawConf, #{override_to := local}) -> - allow; -check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) -> - case emqx_map_lib:deep_find(ConfKeyPath, NewRawConf) of - {ok, NewRaw} -> - LocalOverride = emqx_config:read_override_conf(#{override_to => local}), - case emqx_map_lib:deep_find(ConfKeyPath, LocalOverride) of - {ok, LocalRaw} -> - case is_mutable(Action, NewRaw, LocalRaw) of - ok -> - allow; - {error, Error} -> - ?SLOG(error, #{ - msg => "prevent_remove_local_conf", - config_key_path => ConfKeyPath, - error => Error - }), - {deny, "Disable changed from local conf"} - end; - {not_found, _, _} -> - allow - end; - {not_found, _, _} -> - allow - end. - -is_mutable(Action, NewRaw, LocalRaw) -> - try - KeyPath = [], - is_mutable(KeyPath, Action, NewRaw, LocalRaw) - catch - throw:Error -> Error - end. - --define(REMOVE_FAILED, "remove_failed"). --define(UPDATE_FAILED, "update_failed"). - -is_mutable(KeyPath, Action, New = #{}, Local = #{}) -> - maps:foreach( - fun(Key, SubLocal) -> - case maps:find(Key, New) of - error -> ok; - {ok, SubNew} -> is_mutable(KeyPath ++ [Key], Action, SubNew, SubLocal) - end - end, - Local - ); -is_mutable(KeyPath, remove, Update, Origin) -> - throw({error, {?REMOVE_FAILED, KeyPath, Update, Origin}}); -is_mutable(_KeyPath, update, Val, Val) -> - ok; -is_mutable(KeyPath, update, Update, Origin) -> - throw({error, {?UPDATE_FAILED, KeyPath, Update, Origin}}). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -is_mutable_update_test() -> - Action = update, - ?assertEqual(ok, is_mutable(Action, #{}, #{})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a => #{b => #{c => #{}}}})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})), - ?assertEqual( - {error, {?UPDATE_FAILED, [a, b, c], 1, 2}}, - is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}}) - ), - ?assertEqual( - {error, {?UPDATE_FAILED, [a, b, d], 2, 3}}, - is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}}) - ), - ok. - -is_mutable_remove_test() -> - Action = remove, - ?assertEqual(ok, is_mutable(Action, #{}, #{})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a1 => #{b => #{c => #{}}}})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b1 => #{c => 1}}})), - ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c1 => 1}}})), - - ?assertEqual( - {error, {?REMOVE_FAILED, [a, b, c], 1, 1}}, - is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}}) - ), - ?assertEqual( - {error, {?REMOVE_FAILED, [a, b, c], 1, 2}}, - is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}}) - ), - ?assertEqual( - {error, {?REMOVE_FAILED, [a, b, c], 1, 1}}, - is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}}) - ), - ok. - --endif. diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index 6e2dbfef1..cea956825 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -200,55 +200,6 @@ t_sub_key_update_remove(_Config) -> ok = emqx_config_handler:remove_handler(KeyPath2), ok. -t_local_override_update_remove(_Config) -> - application:set_env(emqx, local_override_conf_file, ?LOCAL_CONF), - application:set_env(emqx, cluster_override_conf_file, ?CLUSTER_CONF), - KeyPath = [sysmon, os, cpu_high_watermark], - ok = emqx_config_handler:add_handler(KeyPath, ?MODULE), - LocalOpts = #{override_to => local}, - {ok, Res} = emqx:update_config(KeyPath, <<"70%">>, LocalOpts), - ?assertMatch( - #{ - config := 0.7, - post_config_update := #{}, - raw_config := <<"70%">> - }, - Res - ), - ClusterOpts = #{override_to => cluster}, - ?assertMatch( - {error, {permission_denied, _}}, emqx:update_config(KeyPath, <<"71%">>, ClusterOpts) - ), - ?assertMatch(0.7, emqx:get_config(KeyPath)), - - %% remove - ?assertMatch({error, {permission_denied, _}}, emqx:remove_config(KeyPath)), - ?assertEqual( - {ok, #{post_config_update => #{}}}, - emqx:remove_config(KeyPath, #{override_to => local}) - ), - ?assertEqual( - {ok, #{post_config_update => #{}}}, - emqx:remove_config(KeyPath) - ), - ?assertError({config_not_found, KeyPath}, emqx:get_raw_config(KeyPath)), - OSKey = maps:keys(emqx:get_raw_config([sysmon, os])), - ?assertEqual(false, lists:member(<<"cpu_high_watermark">>, OSKey)), - ?assert(length(OSKey) > 0), - - ?assertEqual( - {ok, #{config => 0.8, post_config_update => #{}, raw_config => <<"80%">>}}, - emqx:reset_config(KeyPath, ClusterOpts) - ), - OSKey1 = maps:keys(emqx:get_raw_config([sysmon, os])), - ?assertEqual(true, lists:member(<<"cpu_high_watermark">>, OSKey1)), - ?assert(length(OSKey1) > 1), - - ok = emqx_config_handler:remove_handler(KeyPath), - application:unset_env(emqx, local_override_conf_file), - application:unset_env(emqx, cluster_override_conf_file), - ok. - t_check_failed(_Config) -> KeyPath = [sysmon, os, cpu_check_interval], Opts = #{rawconf_with_defaults => true}, From 7f870257b2e29484e82d0281ebc3ffa9eb9e8fe9 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 10 Apr 2023 17:26:17 +0800 Subject: [PATCH 133/279] test: fix failed ct --- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index eb67490bc..e6b15ad13 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -100,7 +100,7 @@ set_data_dir_env() -> ok = file:write_file(NewConfigFile, DataDir, [append]), application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), - application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster.hocon"), + application:set_env(emqx, cluster_conf_file, Node ++ "/configs/cluster.hocon"), ok. assert_data_copy_done([First0 | Rest]) -> @@ -108,6 +108,7 @@ assert_data_copy_done([First0 | Rest]) -> {ok, FakeCertFile} = file:read_file(First ++ "/certs/fake-cert"), {ok, FakeAuthzFile} = file:read_file(First ++ "/authz/fake-authz"), {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster.hocon"), + {ok, ExpectFake} = hocon:binary(FakeOverrideFile), lists:foreach( fun(Node0) -> Node = atom_to_list(Node0), @@ -117,8 +118,8 @@ assert_data_copy_done([First0 | Rest]) -> #{node => Node} ), ?assertEqual( - {ok, FakeOverrideFile}, - file:read_file(Node ++ "/configs/cluster.hocon"), + {ok, ExpectFake}, + hocon:files([Node ++ "/configs/cluster.hocon"]), #{node => Node} ), ?assertEqual( From fa753cf333664b04d0a1ed60c68328027a43f477 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 13 Apr 2023 14:25:44 +0800 Subject: [PATCH 134/279] fix: failed ct test --- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 53 +++++++++++++++++---- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index e6b15ad13..21e22519e 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -27,7 +27,7 @@ all() -> t_copy_conf_override_on_restarts(_Config) -> ct:timetrap({seconds, 120}), snabbkaffe:fix_ct_logging(), - Cluster = cluster([core, core, core]), + Cluster = cluster([cluster_spec({core, 1}), cluster_spec({core, 2}), cluster_spec({core, 3})]), %% 1. Start all nodes Nodes = start_cluster(Cluster), @@ -41,7 +41,7 @@ t_copy_conf_override_on_restarts(_Config) -> %% crash and eventually all nodes should be ready. start_cluster_async(Cluster), - timer:sleep(15_000), + timer:sleep(15000), assert_config_load_done(Nodes), @@ -50,11 +50,11 @@ t_copy_conf_override_on_restarts(_Config) -> stop_cluster(Nodes) end. -t_copy_data_dir(_Config) -> +t_copy_new_data_dir(_Config) -> net_kernel:start(['master1@127.0.0.1', longnames]), ct:timetrap({seconds, 120}), snabbkaffe:fix_ct_logging(), - Cluster = cluster([{core, copy1}, {core, copy2}, {core, copy3}]), + Cluster = cluster([cluster_spec({core, 4}), cluster_spec({core, 5}), cluster_spec({core, 6})]), %% 1. Start all nodes [First | Rest] = Nodes = start_cluster(Cluster), @@ -66,7 +66,30 @@ t_copy_data_dir(_Config) -> ok = rpc:call(First, application, start, [emqx_conf]), {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]), - assert_data_copy_done(Nodes), + assert_data_copy_done(Nodes, "/configs/cluster.hocon"), + stop_cluster(Nodes), + ok + after + stop_cluster(Nodes) + end. + +t_copy_deprecated_data_dir(_Config) -> + net_kernel:start(['master2@127.0.0.1', longnames]), + ct:timetrap({seconds, 120}), + snabbkaffe:fix_ct_logging(), + Cluster = cluster([cluster_spec({core, 7}), cluster_spec({core, 8}), cluster_spec({core, 9})]), + + %% 1. Start all nodes + [First | Rest] = Nodes = start_cluster(Cluster), + try + assert_config_load_done(Nodes), + rpc:call(First, ?MODULE, create_deprecated_data_dir, []), + {[ok, ok, ok], []} = rpc:multicall(Nodes, application, stop, [emqx_conf]), + {[ok, ok, ok], []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []), + ok = rpc:call(First, application, start, [emqx_conf]), + {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]), + + assert_data_copy_done(Nodes, "/configs/cluster-override.conf"), stop_cluster(Nodes), ok after @@ -87,6 +110,16 @@ create_data_dir() -> Telemetry = <<"telemetry.enable = false">>, ok = file:write_file(Node ++ "/configs/cluster.hocon", Telemetry). +create_deprecated_data_dir() -> + Node = atom_to_list(node()), + ok = filelib:ensure_dir(Node ++ "/certs/"), + ok = filelib:ensure_dir(Node ++ "/authz/"), + ok = filelib:ensure_dir(Node ++ "/configs/"), + ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)), + ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)), + Telemetry = <<"telemetry.enable = false">>, + ok = file:write_file(Node ++ "/configs/cluster-override.conf", Telemetry). + set_data_dir_env() -> Node = atom_to_list(node()), %% will create certs and authz dir @@ -101,13 +134,14 @@ set_data_dir_env() -> application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), application:set_env(emqx, cluster_conf_file, Node ++ "/configs/cluster.hocon"), + application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"), ok. -assert_data_copy_done([First0 | Rest]) -> +assert_data_copy_done([First0 | Rest], File) -> First = atom_to_list(First0), {ok, FakeCertFile} = file:read_file(First ++ "/certs/fake-cert"), {ok, FakeAuthzFile} = file:read_file(First ++ "/authz/fake-authz"), - {ok, FakeOverrideFile} = file:read_file(First ++ "/configs/cluster.hocon"), + {ok, FakeOverrideFile} = file:read_file(First ++ File), {ok, ExpectFake} = hocon:binary(FakeOverrideFile), lists:foreach( fun(Node0) -> @@ -119,7 +153,7 @@ assert_data_copy_done([First0 | Rest]) -> ), ?assertEqual( {ok, ExpectFake}, - hocon:files([Node ++ "/configs/cluster.hocon"]), + hocon:files([Node ++ File]), #{node => Node} ), ?assertEqual( @@ -174,3 +208,6 @@ cluster(Specs) -> ok end} ]). + +cluster_spec({Type, Num}) -> + {Type, list_to_atom(atom_to_list(?MODULE) ++ integer_to_list(Num))}. From c21744c2601cd8607b71455bc4441037ae67de9c Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 17:09:24 +0800 Subject: [PATCH 135/279] fix: hocon_pp crash when atom_key --- apps/emqx/rebar.config | 2 +- apps/emqx/test/emqx_authentication_SUITE.erl | 8 ++++++-- apps/emqx/test/emqx_common_test_helpers.erl | 7 +++++++ mix.exs | 2 +- rebar.config | 2 +- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 9079322eb..bb89dfe52 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index 2c83162ed..0190ab936 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -95,13 +95,17 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + LogLevel = emqx_logger:get_primary_log_level(), + ok = emqx_logger:set_log_level(debug), application:set_env(ekka, strict_mode, true), emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([]), - Config. + [{log_level, LogLevel} | Config]. -end_per_suite(_) -> +end_per_suite(Config) -> emqx_common_test_helpers:stop_apps([]), + LogLevel = ?config(log_level), + emqx_logger:set_log_level(LogLevel), ok. init_per_testcase(Case, Config) -> diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 406183094..bc9da2a9e 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -314,6 +314,8 @@ stop_apps(Apps) -> ok = emqx_config:delete_override_conf_files(), application:unset_env(emqx, local_override_conf_file), application:unset_env(emqx, cluster_override_conf_file), + application:unset_env(emqx, local_conf_file), + application:unset_env(emqx, cluster_conf_file), application:unset_env(gen_rpc, port_discovery), ok. @@ -462,6 +464,11 @@ force_set_config_file_paths(emqx_conf, [Path] = Paths) -> ok = file:write_file(Path, Bin, [append]), application:set_env(emqx, config_files, Paths); force_set_config_file_paths(emqx, Paths) -> + %% we need init cluster conf, so we can save the cluster conf to the file + application:set_env(emqx, local_override_conf_file, "local_override.conf"), + application:set_env(emqx, cluster_override_conf_file, "cluster_override.conf"), + application:set_env(emqx, local_conf_file, "local.hocon"), + application:set_env(emqx, cluster_conf_file, "cluster.hocon"), application:set_env(emqx, config_files, Paths); force_set_config_file_paths(_, _) -> ok. diff --git a/mix.exs b/mix.exs index 582860039..55036d498 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.38.0", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.38.1", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 9b67b6cce..7e783b56d 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From 4f2f0ebb6a25817e468681bcec03357948d6f477 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 14:58:19 +0200 Subject: [PATCH 136/279] fix: bad ref to emqx_misc after rebase --- lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index dc9759341..f11441a3b 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -188,7 +188,7 @@ on_start( ?SLOG(info, #{ msg => "starting_sqlserver_connector", connector => InstanceId, - config => emqx_misc:redact(Config) + config => emqx_utils:redact(Config) }), ODBCDir = code:priv_dir(odbc), From c0e6e79bcddca8eef85dd8a10081f1608789f29f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 20:51:40 +0800 Subject: [PATCH 137/279] feat: don't remove default value when save config --- apps/emqx/src/emqx_config.erl | 250 +++++-------------- apps/emqx/src/emqx_config_handler.erl | 4 +- apps/emqx/test/emqx_common_test_helpers.erl | 4 +- apps/emqx/test/emqx_config_handler_SUITE.erl | 2 - apps/emqx_conf/src/emqx_conf_app.erl | 10 +- apps/emqx_conf/src/emqx_conf_schema.erl | 10 +- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 1 + 7 files changed, 67 insertions(+), 214 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 68a8563d8..75396d779 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -24,19 +24,18 @@ init_load/2, init_load/3, read_override_conf/1, - has_deprecated_conf/0, + has_deprecated_file/0, delete_override_conf_files/0, check_config/2, fill_defaults/1, fill_defaults/2, fill_defaults/3, - save_configs/6, + save_configs/5, save_to_app_env/1, save_to_config_map/2, save_to_override_conf/3 ]). -export([raw_conf_with_default/4]). --export([remove_default_conf/2]). -export([merge_envs/2]). -export([ @@ -329,32 +328,32 @@ init_load(SchemaMod, ConfFiles) -> %% in the rear of the list overrides prior values. -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok. init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> - init_load(SchemaMod, parse_hocon(Conf), Opts); -init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) -> + HasDeprecatedFile = has_deprecated_file(), + RawConf = parse_hocon(HasDeprecatedFile, Conf), + init_load(HasDeprecatedFile, SchemaMod, RawConf, Opts). + +init_load(true, SchemaMod, RawConf, Opts) when is_map(RawConf) -> + %% deprecated conf will be removed in 5.1 + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + Overrides = read_override_confs(), + RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), + RootNames = get_root_names(), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll); +init_load(false, SchemaMod, RawConf, Opts) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), - case has_deprecated_conf() of - false -> - RootNames = get_root_names(), - %% Merge environment variable overrides on top - RawConfWithEnvs = merge_envs(SchemaMod, RawConf), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), - %% check configs against the schema - {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), - save_to_app_env(AppEnvs), - ok = save_to_config_map(CheckedConf, RawConfAll); - true -> - %% deprecated conf will be removed in 5.1 - %% Merge environment variable overrides on top - RawConfWithEnvs = merge_envs(SchemaMod, RawConf), - Overrides = read_override_confs(), - RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), - RootNames = get_root_names(), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), - %% check configs against the schema - {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), - save_to_app_env(AppEnvs), - ok = save_to_config_map(CheckedConf, RawConfAll) - end. + RootNames = get_root_names(), + %% Merge environment variable overrides on top + RawConfWithEnvs = merge_envs(SchemaMod, RawConf), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithEnvs, Opts), + %% check configs against the schema + {AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + save_to_app_env(AppEnvs), + ok = save_to_config_map(CheckedConf, RawConfAll). %% @doc Read merged cluster + local overrides. read_override_confs() -> @@ -390,21 +389,20 @@ schema_default(Schema) -> #{} end. -parse_hocon(Conf) -> - HasDeprecatedConf = has_deprecated_conf(), +parse_hocon(HasDeprecatedFile, Conf) -> IncDirs = include_dirs(), - case do_parse_hocon(HasDeprecatedConf, Conf, IncDirs) of + case do_parse_hocon(HasDeprecatedFile, Conf, IncDirs) of {ok, HoconMap} -> HoconMap; {error, Reason} -> ?SLOG(error, #{ - msg => "failed_to_load_hocon_conf", + msg => "failed_to_load_hocon_file", reason => Reason, pwd => file:get_cwd(), include_dirs => IncDirs, config_file => Conf }), - error(failed_to_load_hocon_conf) + error(failed_to_load_hocon_file) end. do_parse_hocon(true, Conf, IncDirs) -> @@ -416,12 +414,12 @@ do_parse_hocon(true, Conf, IncDirs) -> do_parse_hocon(false, Conf, IncDirs) -> Opts = #{format => map, include_dirs => IncDirs}, case is_binary(Conf) of + %% only use in test true -> hocon:binary(Conf, Opts); false -> - LocalFile = deprecated_override_conf_file(#{override_to => local}), - ClusterFile = deprecated_override_conf_file(#{override_to => cluster}), - hocon:files([ClusterFile] ++ Conf ++ [LocalFile], Opts) + ClusterFile = cluster_hocon_file(#{override_to => cluster}), + hocon:files([ClusterFile | Conf], Opts) end. include_dirs() -> @@ -493,14 +491,12 @@ fill_defaults(SchemaMod, RawConf, Opts0) -> %% Delete override config files. -spec delete_override_conf_files() -> ok. delete_override_conf_files() -> - F1 = deprecated_override_conf_file(#{override_to => local}), - F2 = deprecated_override_conf_file(#{override_to => cluster}), - F3 = conf_file(#{override_to => local}), - F4 = conf_file(#{override_to => cluster}), + F1 = deprecated_conf_file(#{override_to => local}), + F2 = deprecated_conf_file(#{override_to => cluster}), + F3 = cluster_hocon_file(#{override_to => cluster}), ok = ensure_file_deleted(F1), ok = ensure_file_deleted(F2), - ok = ensure_file_deleted(F3), - ok = ensure_file_deleted(F4). + ok = ensure_file_deleted(F3). ensure_file_deleted(F) -> case file:delete(F) of @@ -512,38 +508,35 @@ ensure_file_deleted(F) -> -spec read_override_conf(map()) -> raw_config(). read_override_conf(#{} = Opts) -> File = - case has_deprecated_conf() of - true -> deprecated_override_conf_file(Opts); - false -> conf_file(Opts) + case has_deprecated_file() of + true -> deprecated_conf_file(Opts); + false -> cluster_hocon_file(Opts) end, load_hocon_file(File, map). -has_deprecated_conf() -> - DeprecatedFile = deprecated_cluster_conf_file(), +has_deprecated_file() -> + DeprecatedFile = deprecated_conf_file(#{override_to => cluster}), filelib:is_regular(DeprecatedFile). -deprecated_cluster_conf_file() -> - ClusterFile = deprecated_override_conf_file(#{override_to => cluster}), - filename:join(filename:dirname(ClusterFile), "cluster-override.conf"). - -deprecated_override_conf_file(Opts) when is_map(Opts) -> +deprecated_conf_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of local -> local_override_conf_file; cluster -> cluster_override_conf_file end, application:get_env(emqx, Key, undefined); -deprecated_override_conf_file(Which) when is_atom(Which) -> +deprecated_conf_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). -conf_file(Opts) when is_map(Opts) -> +cluster_hocon_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of - local -> local_conf_file; - cluster -> cluster_conf_file + %% no local config file support + local -> undefined; + cluster -> cluster_hocon_file end, application:get_env(emqx, Key, undefined); -conf_file(Which) when is_atom(Which) -> +cluster_hocon_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). -spec save_schema_mod_and_names(module()) -> ok. @@ -576,54 +569,17 @@ get_root_names() -> maps:get(names, persistent_term:get(?PERSIS_SCHEMA_MODS, #{names => []})). -spec save_configs( - emqx_map_lib:config_key_path(), app_envs(), config(), raw_config(), raw_config(), update_opts() + app_envs(), config(), raw_config(), raw_config(), update_opts() ) -> ok. -save_configs(Paths0, AppEnvs, Conf, RawConf, OverrideConf, Opts) -> - Default = init_default(Paths0), - OverrideConf1 = remove_default_conf(OverrideConf, Default), +save_configs(AppEnvs, Conf, RawConf, OverrideConf, Opts) -> %% We first try to save to files, because saving to files is more error prone %% than saving into memory. - HasDeprecated = has_deprecated_conf(), - ok = save_to_override_conf(HasDeprecated, OverrideConf1, Opts), + HasDeprecatedFile = has_deprecated_file(), + ok = save_to_override_conf(HasDeprecatedFile, OverrideConf, Opts), save_to_app_env(AppEnvs), save_to_config_map(Conf, RawConf). -init_default(Paths0) -> - [Root | _] = [bin(Key) || Key <- Paths0], - SchemaMod = get_schema_mod(Root), - {_, {_, Schema}} = lists:keyfind(Root, 1, hocon_schema:roots(SchemaMod)), - fill_defaults(#{Root => schema_default(Schema)}). - -remove_default_conf(undefined, _) -> - undefined; -remove_default_conf(Conf, DefaultConf) when is_map(Conf) andalso is_map(DefaultConf) -> - maps:fold( - fun(Key, Value, Acc) -> - case maps:find(Key, DefaultConf) of - {ok, DefaultValue} -> - remove_default_conf(Value, DefaultValue, Key, Acc); - error -> - Acc - end - end, - Conf, - Conf - ). - -remove_default_conf(Value, Value, Key, Conf) -> - maps:remove(Key, Conf); -remove_default_conf(Value = #{}, DefaultValue = #{}, Key, Conf) -> - case remove_default_conf(Value, DefaultValue) of - SubValue when SubValue =:= #{} -> maps:remove(Key, Conf); - SubValue -> maps:put(Key, SubValue, Conf) - end; -remove_default_conf(Value, DefaultValue, Key, Conf) -> - case try_bin(DefaultValue) =:= try_bin(Value) of - true -> maps:remove(Key, Conf); - false -> Conf - end. - %% we ignore kernel app env, %% because the old app env will be used in emqx_config_logger:post_config_update/5 -define(IGNORE_APPS, [kernel]). @@ -643,7 +599,7 @@ save_to_override_conf(_, undefined, _) -> ok; %% TODO: Remove deprecated override conf file when 5.1 save_to_override_conf(true, RawConf, Opts) -> - case deprecated_override_conf_file(Opts) of + case deprecated_conf_file(Opts) of undefined -> ok; FileName -> @@ -661,7 +617,7 @@ save_to_override_conf(true, RawConf, Opts) -> end end; save_to_override_conf(false, RawConf, Opts) -> - case conf_file(Opts) of + case cluster_hocon_file(Opts) of undefined -> ok; FileName -> @@ -791,16 +747,6 @@ atom(Str) when is_list(Str) -> atom(Atom) when is_atom(Atom) -> Atom. -try_bin(Bin) when is_binary(Bin) -> Bin; -try_bin([Bin | _] = List) when is_binary(Bin) -> List; -try_bin([Atom | _] = List) when is_atom(Atom) -> [atom_to_binary(A) || A <- List]; -try_bin([Map | _] = Maps) when is_map(Map) -> Maps; -try_bin(Str) when is_list(Str) -> list_to_binary(Str); -try_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); -try_bin(Int) when is_integer(Int) -> integer_to_binary(Int); -try_bin(Float) when is_float(Float) -> float_to_binary(Float); -try_bin(Term) -> Term. - bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). @@ -822,87 +768,3 @@ atom_conf_path(Path, ExpFun, OnFail) -> error(Err) end end. - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -remove_default_conf_test() -> - ?assertEqual( - #{}, - remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => 100}) - ), - ?assertEqual( - #{}, - remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => <<"100">>}) - ), - ?assertEqual( - #{<<"def">> => 100}, - remove_default_conf(#{<<"def">> => 100}, #{<<"def">> => #{<<"bar">> => 100}}) - ), - ?assertEqual( - #{<<"def">> => #{<<"edf">> => 321}}, - remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => 321}}, #{ - <<"def">> => #{<<"abc">> => 100, <<"edf">> => 123} - }) - ), - ?assertEqual( - #{}, - remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => 321}}, #{ - <<"def">> => #{<<"abc">> => <<"100">>, <<"edf">> => 321} - }) - ), - ?assertEqual( - #{}, - remove_default_conf(#{<<"def">> => #{<<"abc">> => 100, <<"edf">> => <<"true">>}}, #{ - <<"def">> => #{<<"abc">> => <<"100">>, <<"edf">> => true} - }) - ), - ?assertEqual( - #{}, - remove_default_conf( - #{ - <<"bytes_in">> => - #{ - <<"capacity">> => infinity, - <<"initial">> => 0, - <<"rate">> => infinity - } - }, - #{ - <<"bytes_in">> => - #{ - <<"capacity">> => <<"infinity">>, - <<"initial">> => <<"0">>, - <<"rate">> => <<"infinity">> - } - } - ) - ), - ?assertEqual( - #{}, - remove_default_conf( - #{ - <<"limiter">> => #{ - <<"connection">> => - #{ - <<"capacity">> => 1000, - <<"initial">> => <<"0">>, - <<"rate">> => <<"1000/s">> - } - } - }, - #{ - <<"limiter">> => #{ - <<"connection">> => - #{ - <<"capacity">> => <<"1000">>, - <<"initial">> => <<"0">>, - <<"rate">> => <<"1000/s">> - } - } - } - ) - ), - ok. - --endif. diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 3e18a172e..d5dc72acc 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -279,9 +279,7 @@ check_and_save_configs( OldConf = emqx_config:get_root(ConfKeyPath), case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of {ok, Result0} -> - ok = emqx_config:save_configs( - ConfKeyPath, AppEnvs, NewConf, NewRawConf, OverrideConf, Opts - ), + ok = emqx_config:save_configs(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts), Result1 = return_change_result(ConfKeyPath, UpdateArgs), {ok, Result1#{post_config_update => Result0}}; Error -> diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index bc9da2a9e..6e56c85ed 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -314,8 +314,7 @@ stop_apps(Apps) -> ok = emqx_config:delete_override_conf_files(), application:unset_env(emqx, local_override_conf_file), application:unset_env(emqx, cluster_override_conf_file), - application:unset_env(emqx, local_conf_file), - application:unset_env(emqx, cluster_conf_file), + application:unset_env(emqx, cluster_hocon_file), application:unset_env(gen_rpc, port_discovery), ok. @@ -467,7 +466,6 @@ force_set_config_file_paths(emqx, Paths) -> %% we need init cluster conf, so we can save the cluster conf to the file application:set_env(emqx, local_override_conf_file, "local_override.conf"), application:set_env(emqx, cluster_override_conf_file, "cluster_override.conf"), - application:set_env(emqx, local_conf_file, "local.hocon"), application:set_env(emqx, cluster_conf_file, "cluster.hocon"), application:set_env(emqx, config_files, Paths); force_set_config_file_paths(_, _) -> diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl index cea956825..deeee8d62 100644 --- a/apps/emqx/test/emqx_config_handler_SUITE.erl +++ b/apps/emqx/test/emqx_config_handler_SUITE.erl @@ -21,7 +21,6 @@ -define(MOD, {mod}). -define(WKEY, '?'). --define(LOCAL_CONF, "/tmp/local.conf"). -define(CLUSTER_CONF, "/tmp/cluster.conf"). -include_lib("eunit/include/eunit.hrl"). @@ -38,7 +37,6 @@ end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([]). init_per_testcase(_Case, Config) -> - _ = file:delete(?LOCAL_CONF), _ = file:delete(?CLUSTER_CONF), Config. diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 45631619b..35a79ea6e 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -60,13 +60,13 @@ get_override_config_file() -> TnxId = emqx_cluster_rpc:get_node_tnx_id(Node), WallClock = erlang:statistics(wall_clock), Conf = emqx_config_handler:get_raw_cluster_override_conf(), - HasDeprecateConf = emqx_config:has_deprecated_conf(), + HasDeprecateFile = emqx_config:has_deprecated_file(), #{ wall_clock => WallClock, conf => Conf, tnx_id => TnxId, node => Node, - has_deprecated_conf => HasDeprecateConf + has_deprecated_file => HasDeprecateFile } end, case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of @@ -175,16 +175,16 @@ copy_override_conf_from_core_node() -> _ -> [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready), #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info, - HasDeprecatedConf = maps:get(has_deprecated_conf, Info, false), + HasDeprecatedFile = maps:get(has_deprecated_file, Info, false), ?SLOG(debug, #{ msg => "copy_cluster_conf_from_core_node_success", node => Node, - has_deprecated_conf => HasDeprecatedConf, + has_deprecated_file => HasDeprecatedFile, data_dir => emqx:data_dir(), tnx_id => TnxId }), ok = emqx_config:save_to_override_conf( - HasDeprecatedConf, + HasDeprecatedFile, RawOverrideConf, #{override_to => cluster} ), diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index a67d2a100..228462e73 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1029,8 +1029,7 @@ translation("emqx") -> {"config_files", fun tr_config_files/1}, {"cluster_override_conf_file", fun tr_cluster_override_conf_file/1}, {"local_override_conf_file", fun tr_local_override_conf_file/1}, - {"cluster_conf_file", fun tr_cluster_conf_file/1}, - {"local_conf_file", fun tr_local_conf_file/1} + {"cluster_hocon_file", fun tr_cluster_hocon_file/1} ]; translation("gen_rpc") -> [{"default_client_driver", fun tr_default_config_driver/1}]; @@ -1081,14 +1080,11 @@ tr_cluster_override_conf_file(Conf) -> tr_conf_file(Conf, "cluster-override.conf"). tr_local_override_conf_file(Conf) -> - tr_conf_file(Conf, "local-overide.conf"). + tr_conf_file(Conf, "local-override.conf"). -tr_cluster_conf_file(Conf) -> +tr_cluster_hocon_file(Conf) -> tr_conf_file(Conf, "cluster.hocon"). -tr_local_conf_file(Conf) -> - tr_conf_file(Conf, "local.hocon"). - tr_conf_file(Conf, Filename) -> DataDir = conf_get("node.data_dir", Conf), %% assert, this config is not nullable diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index 21e22519e..825f01441 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -133,6 +133,7 @@ set_data_dir_env() -> ok = file:write_file(NewConfigFile, DataDir, [append]), application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), + %% We set env both cluster.hocon and cluster-override.conf, but only one will be used application:set_env(emqx, cluster_conf_file, Node ++ "/configs/cluster.hocon"), application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"), ok. From e5b85916b62ac274f4f028a52e08d82df48bc3c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Mon, 10 Apr 2023 22:00:56 +0800 Subject: [PATCH 138/279] feat: hidden stats config --- apps/emqx/src/emqx_schema.erl | 13 +++++---- apps/emqx/src/emqx_zone_schema.erl | 27 ++++++++++++++++++- .../src/emqx_mgmt_api_configs.erl | 5 ++-- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ed20848d2..f03ba07c5 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -204,7 +204,9 @@ roots(low) -> {"stats", sc( ref("stats"), - #{} + #{ + importance => ?IMPORTANCE_HIDDEN + } )}, {"sysmon", sc( @@ -339,6 +341,7 @@ fields("stats") -> boolean(), #{ default => true, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(stats_enable) } )} @@ -609,8 +612,7 @@ fields("mqtt") -> )} ]; fields("zone") -> - Fields = emqx_zone_schema:roots(), - [{F, ref(emqx_zone_schema, F)} || F <- Fields]; + emqx_zone_schema:zone(); fields("flapping_detect") -> [ {"enable", @@ -1971,10 +1973,7 @@ desc("persistent_session_builtin") -> desc("persistent_table_mria_opts") -> "Tuning options for the mria table."; desc("stats") -> - "Enable/disable statistic data collection.\n" - "Statistic data such as message receive/send count/rate etc. " - "It provides insights of system performance and helps to diagnose issues. " - "You can find statistic data from the dashboard, or from the '/stats' API."; + "deprecated since 5.0.23"; desc("authorization") -> "Settings for client authorization."; desc("mqtt") -> diff --git a/apps/emqx/src/emqx_zone_schema.erl b/apps/emqx/src/emqx_zone_schema.erl index c2595725b..6ac8cefa3 100644 --- a/apps/emqx/src/emqx_zone_schema.erl +++ b/apps/emqx/src/emqx_zone_schema.erl @@ -15,8 +15,10 @@ %%-------------------------------------------------------------------- -module(emqx_zone_schema). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). --export([namespace/0, roots/0, fields/1, desc/1]). +-export([namespace/0, roots/0, fields/1, desc/1, zone/0, zone_without_hidden/0]). namespace() -> zone. @@ -33,6 +35,29 @@ roots() -> "overload_protection" ]. +zone() -> + Fields = roots(), + Hidden = hidden(), + lists:map( + fun(F) -> + case lists:member(F, Hidden) of + true -> + {F, ?HOCON(?R_REF(F), #{importance => ?IMPORTANCE_HIDDEN})}; + false -> + {F, ?HOCON(?R_REF(F), #{})} + end + end, + Fields + ). + +zone_without_hidden() -> + lists:map(fun(F) -> {F, ?HOCON(?R_REF(F), #{})} end, roots() -- hidden()). + +hidden() -> + [ + "stats" + ]. + %% zone schemas are clones from the same name from root level %% only not allowed to have default values. fields(Name) -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 55cc50597..13614335e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -352,8 +352,7 @@ with_default_value(Type, Value) -> Type#{example => emqx_map_lib:binary_string(Value)}. global_zone_roots() -> - lists:map(fun({K, _}) -> K end, global_zone_schema()). + lists:map(fun({K, _}) -> list_to_binary(K) end, global_zone_schema()). global_zone_schema() -> - Roots = hocon_schema:roots(emqx_zone_schema), - lists:map(fun({RootKey, {_Root, Schema}}) -> {RootKey, Schema} end, Roots). + emqx_zone_schema:zone_without_hidden(). From 7934a1cea13437ee48067186e853f94e556178ea Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 12 Apr 2023 14:39:59 +0800 Subject: [PATCH 139/279] feat: hide overload_protection,conn_congestion,flapping_detect --- apps/emqx/src/emqx_channel.erl | 2 +- apps/emqx/src/emqx_flapping.erl | 50 ++++++++++++------- apps/emqx/src/emqx_schema.erl | 48 +++++++++++------- apps/emqx/src/emqx_zone_schema.erl | 5 +- apps/emqx/test/emqx_flapping_SUITE.erl | 18 +++++++ .../test/emqx_mgmt_api_configs_SUITE.erl | 4 +- 6 files changed, 86 insertions(+), 41 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 29a59e482..1506e296f 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1630,7 +1630,7 @@ check_banned(_ConnPkt, #channel{clientinfo = ClientInfo}) -> %% Flapping count_flapping_event(_ConnPkt, Channel = #channel{clientinfo = ClientInfo = #{zone := Zone}}) -> - emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso + is_integer(emqx_config:get_zone_conf(Zone, [flapping_detect, window_time])) andalso emqx_flapping:detect(ClientInfo), {ok, Channel}. diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 64e4ed6c3..cc4af99d2 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -27,6 +27,10 @@ %% API -export([detect/1]). +-ifdef(TEST). +-export([get_policy/2]). +-endif. + %% gen_server callbacks -export([ init/1, @@ -39,15 +43,6 @@ %% Tab -define(FLAPPING_TAB, ?MODULE). -%% Default Policy --define(FLAPPING_THRESHOLD, 30). --define(FLAPPING_DURATION, 60000). --define(FLAPPING_BANNED_INTERVAL, 300000). --define(DEFAULT_DETECT_POLICY, #{ - max_count => ?FLAPPING_THRESHOLD, - window_time => ?FLAPPING_DURATION, - ban_time => ?FLAPPING_BANNED_INTERVAL -}). -record(flapping, { clientid :: emqx_types:clientid(), @@ -69,7 +64,7 @@ stop() -> gen_server:stop(?MODULE). %% @doc Detect flapping when a MQTT client disconnected. -spec detect(emqx_types:clientinfo()) -> boolean(). detect(#{clientid := ClientId, peerhost := PeerHost, zone := Zone}) -> - Policy = #{max_count := Threshold} = get_policy(Zone), + Policy = #{max_count := Threshold} = get_policy([max_count, window_time, ban_time], Zone), %% The initial flapping record sets the detect_cnt to 0. InitVal = #flapping{ clientid = ClientId, @@ -89,8 +84,22 @@ detect(#{clientid := ClientId, peerhost := PeerHost, zone := Zone}) -> end end. -get_policy(Zone) -> - emqx_config:get_zone_conf(Zone, [flapping_detect]). +get_policy(Keys, Zone) when is_list(Keys) -> + RootKey = flapping_detect, + Conf = emqx_config:get_zone_conf(Zone, [RootKey]), + lists:foldl( + fun(Key, Acc) -> + case maps:find(Key, Conf) of + {ok, V} -> Acc#{Key => V}; + error -> Acc#{Key => emqx_config:get([RootKey, Key])} + end + end, + #{}, + Keys + ); +get_policy(Key, Zone) -> + #{Key := Conf} = get_policy([Key], Zone), + Conf. now_diff(TS) -> erlang:system_time(millisecond) - TS. @@ -166,8 +175,7 @@ handle_cast(Msg, State) -> handle_info({timeout, _TRef, {garbage_collect, Zone}}, State) -> Timestamp = - erlang:system_time(millisecond) - - maps:get(window_time, get_policy(Zone)), + erlang:system_time(millisecond) - get_policy(window_time, Zone), MatchSpec = [{{'_', '_', '_', '$1', '_'}, [{'<', '$1', Timestamp}], [true]}], ets:select_delete(?FLAPPING_TAB, MatchSpec), _ = start_timer(Zone), @@ -183,15 +191,19 @@ code_change(_OldVsn, State, _Extra) -> {ok, State}. start_timer(Zone) -> - WindTime = maps:get(window_time, get_policy(Zone)), - emqx_misc:start_timer(WindTime, {garbage_collect, Zone}). + case get_policy(window_time, Zone) of + WindowTime when is_integer(WindowTime) -> + emqx_misc:start_timer(WindowTime, {garbage_collect, Zone}); + disabled -> + ok + end. start_timers() -> - lists:foreach( - fun({Zone, _ZoneConf}) -> + maps:foreach( + fun(Zone, _ZoneConf) -> start_timer(Zone) end, - maps:to_list(emqx:get_config([zones], #{})) + emqx:get_config([zones], #{}) ). fmt_host(PeerHost) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index f03ba07c5..c3d566554 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -145,17 +145,23 @@ roots(high) -> {"listeners", sc( ref("listeners"), - #{} - )}, - {"zones", - sc( - map("name", ref("zone")), - #{desc => ?DESC(zones)} + #{importance => ?IMPORTANCE_HIGH} )}, {"mqtt", sc( ref("mqtt"), - #{desc => ?DESC(mqtt)} + #{ + desc => ?DESC(mqtt), + importance => ?IMPORTANCE_MEDIUM + } + )}, + {"zones", + sc( + map("name", ref("zone")), + #{ + desc => ?DESC(zones), + importance => ?IMPORTANCE_LOW + } )}, {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication(global)}, %% NOTE: authorization schema here is only to keep emqx app prue @@ -199,7 +205,9 @@ roots(low) -> {"conn_congestion", sc( ref("conn_congestion"), - #{} + #{ + importance => ?IMPORTANCE_HIDDEN + } )}, {"stats", sc( @@ -221,7 +229,7 @@ roots(low) -> {"flapping_detect", sc( ref("flapping_detect"), - #{} + #{importance => ?IMPORTANCE_HIDDEN} )}, {"persistent_session_store", sc( @@ -620,25 +628,27 @@ fields("flapping_detect") -> boolean(), #{ default => false, + deprecated => {since, "5.0.22"}, desc => ?DESC(flapping_detect_enable) } )}, - {"max_count", - sc( - integer(), - #{ - default => 15, - desc => ?DESC(flapping_detect_max_count) - } - )}, {"window_time", sc( - duration(), + hoconsc:union([disabled, duration()]), #{ - default => <<"1m">>, + default => disabled, + importance => ?IMPORTANCE_HIGH, desc => ?DESC(flapping_detect_window_time) } )}, + {"max_count", + sc( + non_neg_integer(), + #{ + default => 15, + desc => ?DESC(flapping_detect_max_count) + } + )}, {"ban_time", sc( duration(), diff --git a/apps/emqx/src/emqx_zone_schema.erl b/apps/emqx/src/emqx_zone_schema.erl index 6ac8cefa3..5d6720986 100644 --- a/apps/emqx/src/emqx_zone_schema.erl +++ b/apps/emqx/src/emqx_zone_schema.erl @@ -55,7 +55,10 @@ zone_without_hidden() -> hidden() -> [ - "stats" + "stats", + "overload_protection", + "conn_congestion", + "flapping_detect" ]. %% zone schemas are clones from the same name from root level diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl index e27ff67e0..877f05995 100644 --- a/apps/emqx/test/emqx_flapping_SUITE.erl +++ b/apps/emqx/test/emqx_flapping_SUITE.erl @@ -101,3 +101,21 @@ t_expired_detecting(_) -> ets:tab2list(emqx_flapping) ) ). + +t_conf_without_window_time(_) -> + %% enable is deprecated, so we need to make sure it won't be used. + Global = emqx_config:get([flapping_detect]), + ?assertNot(maps:is_key(enable, Global)), + %% zones don't have default value, so we need to make sure fallback to global conf. + %% this new_zone will fallback to global conf. + emqx_config:put_zone_conf(new_zone, [flapping_detect], #{}), + ?assertEqual(Global, get_policy(new_zone)), + + emqx_config:put_zone_conf(new_zone_1, [flapping_detect], #{window_time => 100}), + ?assertEqual(100, emqx_flapping:get_policy(window_time, new_zone_1)), + ?assertEqual(maps:get(ban_time, Global), emqx_flapping:get_policy(ban_time, new_zone_1)), + ?assertEqual(maps:get(max_count, Global), emqx_flapping:get_policy(max_count, new_zone_1)), + ok. + +get_policy(Zone) -> + emqx_flapping:get_policy([window_time, ban_time, max_count], Zone). diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 2d24bce99..06aa6f78d 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -121,7 +121,9 @@ t_log(_Config) -> t_global_zone(_Config) -> {ok, Zones} = get_global_zone(), - ZonesKeys = lists:map(fun({K, _}) -> K end, hocon_schema:roots(emqx_zone_schema)), + ZonesKeys = lists:map( + fun({K, _}) -> list_to_binary(K) end, emqx_zone_schema:zone_without_hidden() + ), ?assertEqual(lists:usort(ZonesKeys), lists:usort(maps:keys(Zones))), ?assertEqual( emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed]), From 3789ca2622509d3bfb854648f33a7951508578c6 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 21:31:34 +0800 Subject: [PATCH 140/279] chore: revert stats desc --- apps/emqx/src/emqx_schema.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index c3d566554..bbc7b3a4a 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1983,7 +1983,10 @@ desc("persistent_session_builtin") -> desc("persistent_table_mria_opts") -> "Tuning options for the mria table."; desc("stats") -> - "deprecated since 5.0.23"; + "Enable/disable statistic data collection.\n" + "Statistic data such as message receive/send count/rate etc. " + "It provides insights of system performance and helps to diagnose issues. " + "You can find statistic data from the dashboard, or from the '/stats' API."; desc("authorization") -> "Settings for client authorization."; desc("mqtt") -> From 99d6c5e179657c8414298007d47b5736d04c568f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 21:57:09 +0800 Subject: [PATCH 141/279] feat: hide dashboard's default_username/default_password conf --- apps/emqx_dashboard/src/emqx_dashboard_schema.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 7df661fb2..0ba970842 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -42,6 +42,7 @@ fields("dashboard") -> #{ default => <<"10s">>, desc => ?DESC(sample_interval), + importance => ?IMPORTANCE_HIDDEN, validator => fun validate_sample_interval/1 } )}, @@ -61,6 +62,7 @@ fields("dashboard") -> #{ desc => ?DESC(bootstrap_users_file), required => false, + importance => ?IMPORTANCE_HIDDEN, default => <<>> %% deprecated => {since, "5.1.0"} } @@ -97,7 +99,7 @@ fields("https") -> bind(18084) | common_listener_fields() ++ exclude_fields( - ["fail_if_no_peer_cert"], + ["fail_if_no_peer_cert", "password"], emqx_schema:server_ssl_opts_schema(#{}, true) ) ]. @@ -210,6 +212,7 @@ default_username(default) -> <<"admin">>; default_username(required) -> true; default_username(desc) -> ?DESC(default_username); default_username('readOnly') -> true; +default_username(importance) -> ?IMPORTANCE_HIDDEN; default_username(_) -> undefined. default_password(type) -> binary(); @@ -219,6 +222,7 @@ default_password('readOnly') -> true; default_password(sensitive) -> true; default_password(converter) -> fun emqx_schema:password_converter/2; default_password(desc) -> ?DESC(default_password); +default_password(importance) -> ?IMPORTANCE_HIDDEN; default_password(_) -> undefined. cors(type) -> boolean(); @@ -231,6 +235,7 @@ i18n_lang(type) -> ?ENUM([en, zh]); i18n_lang(default) -> en; i18n_lang('readOnly') -> true; i18n_lang(desc) -> ?DESC(i18n_lang); +i18n_lang(importance) -> ?IMPORTANCE_HIDDEN; i18n_lang(_) -> undefined. validate_sample_interval(Second) -> From a9976287b53e3d759136445eddddff4c6a7e5486 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 16:26:32 +0200 Subject: [PATCH 142/279] fix: force proper calling decode to return proplist --- apps/emqx_utils/test/props/prop_emqx_utils_json.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_utils/test/props/prop_emqx_utils_json.erl b/apps/emqx_utils/test/props/prop_emqx_utils_json.erl index 86f60b39c..0be1508da 100644 --- a/apps/emqx_utils/test/props/prop_emqx_utils_json.erl +++ b/apps/emqx_utils/test/props/prop_emqx_utils_json.erl @@ -66,7 +66,7 @@ prop_object_proplist_to_proplist() -> begin {ok, J} = safe_encode(T), {ok, T} = safe_decode(J), - T = decode(encode(T)), + T = decode(encode(T), []), true end ). @@ -108,7 +108,7 @@ prop_object_map_to_proplist() -> T = to_list(T0), {ok, J} = safe_encode(T0), {ok, T} = safe_decode(J), - T = decode(encode(T0)), + T = decode(encode(T0), []), true end ). From bcce98990605eb33f93adca3600b8abd8025cbba Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Apr 2023 22:30:11 +0800 Subject: [PATCH 143/279] chore: update has_deprecated_file doc Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/src/emqx_config.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 75396d779..c73fdbc72 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -514,6 +514,8 @@ read_override_conf(#{} = Opts) -> end, load_hocon_file(File, map). +%% @doc Return `true' if this node is upgraded from older version which used cluster-override.conf for +%% cluster-wide config persistence. has_deprecated_file() -> DeprecatedFile = deprecated_conf_file(#{override_to => cluster}), filelib:is_regular(DeprecatedFile). From 53871e3a2ce8b2e947a89f611ebe502c9a35a5c9 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 16:30:27 +0200 Subject: [PATCH 144/279] fix: stale ref to emqx_json after rebase --- .../test/emqx_bridge_kafka_impl_producer_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index 5e2a2a7a8..a2111b1a8 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -320,7 +320,7 @@ kafka_bridge_rest_api_helper(Config) -> <<"sql">> => <<"SELECT * from \"kafka_bridge_topic/#\"">> } ), - #{<<"id">> := RuleId} = emqx_json:decode(Rule, [return_maps]), + #{<<"id">> := RuleId} = emqx_utils_json:decode(Rule, [return_maps]), %% counters should be empty before ?assertEqual(0, emqx_resource_metrics:matched_get(ResourceId)), ?assertEqual(0, emqx_resource_metrics:success_get(ResourceId)), From 7c5dead03a4a5b5074b16915469d90a96e424a0e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 14 Apr 2023 22:30:35 +0800 Subject: [PATCH 145/279] chore: update cluster_hocon_file doc Co-authored-by: Zaiming (Stone) Shi --- apps/emqx/src/emqx_config.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index c73fdbc72..b676b07e2 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -530,6 +530,7 @@ deprecated_conf_file(Opts) when is_map(Opts) -> deprecated_conf_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). +%% The newer version cluster-wide config persistence file. cluster_hocon_file(Opts) when is_map(Opts) -> Key = case maps:get(override_to, Opts, cluster) of From edd1bc579f6eaba4119126dd46a3f71f18f5e898 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 16:32:42 +0200 Subject: [PATCH 146/279] fix: stale ref to emqx_json after rebase --- lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl index c1b33c3c9..6e6d7bf03 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -501,7 +501,7 @@ create_bridge_http(Params) -> Path = emqx_mgmt_api_test_util:api_path(["bridges"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; Error -> Error end. From 14ed4a7adae564497719e1fc3bc66ec504beeb82 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 10:05:20 -0300 Subject: [PATCH 147/279] feat(buffer_worker): set default queue mode to `memory_only` Fixes https://emqx.atlassian.net/browse/EMQX-9367 For better user experience and performance for the average bridge, we should change the default queue mode to `memory_only`, as was the behavior of most bridges in e4.x. This leads to better performance when message rate is high enough and the remote resource is not keeping up with EMQX. Also, we set the default segment size to equal max queue bytes. --- apps/emqx_resource/include/emqx_resource.hrl | 3 - .../src/emqx_resource_buffer_worker.erl | 43 ++++--- .../src/schema/emqx_resource_schema.erl | 17 ++- .../test/emqx_resource_SUITE.erl | 112 ++++++++++++++++++ changes/ce/feat-10404.en.md | 2 + rel/i18n/emqx_resource_schema.hocon | 38 ++++++ 6 files changed, 196 insertions(+), 19 deletions(-) create mode 100644 changes/ce/feat-10404.en.md diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 904eeffa5..f8d671b40 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -85,9 +85,6 @@ -define(WORKER_POOL_SIZE, 16). --define(DEFAULT_QUEUE_SEG_SIZE, 10 * 1024 * 1024). --define(DEFAULT_QUEUE_SEG_SIZE_RAW, <<"10MB">>). - -define(DEFAULT_QUEUE_SIZE, 256 * 1024 * 1024). -define(DEFAULT_QUEUE_SIZE_RAW, <<"256MB">>). diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 0fa4c0bd8..c53627ca4 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -178,20 +178,7 @@ init({Id, Index, Opts}) -> process_flag(trap_exit, true), true = gproc_pool:connect_worker(Id, {Id, Index}), BatchSize = maps:get(batch_size, Opts, ?DEFAULT_BATCH_SIZE), - SegBytes0 = maps:get(queue_seg_bytes, Opts, ?DEFAULT_QUEUE_SEG_SIZE), - TotalBytes = maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE), - SegBytes = min(SegBytes0, TotalBytes), - QueueOpts = - #{ - dir => disk_queue_dir(Id, Index), - marshaller => fun ?MODULE:queue_item_marshaller/1, - max_total_bytes => TotalBytes, - %% we don't want to retain the queue after - %% resource restarts. - offload => {true, volatile}, - seg_bytes => SegBytes, - sizer => fun ?MODULE:estimate_size/1 - }, + QueueOpts = replayq_opts(Id, Index, Opts), Queue = replayq:open(QueueOpts), emqx_resource_metrics:queuing_set(Id, Index, queue_count(Queue)), emqx_resource_metrics:inflight_set(Id, Index, 0), @@ -214,7 +201,7 @@ init({Id, Index, Opts}) -> resume_interval => ResumeInterval, tref => undefined }, - ?tp(buffer_worker_init, #{id => Id, index => Index}), + ?tp(buffer_worker_init, #{id => Id, index => Index, queue_opts => QueueOpts}), {ok, running, Data}. running(enter, _, #{tref := _Tref} = Data) -> @@ -1679,6 +1666,32 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) -> end, BatchTime. +replayq_opts(Id, Index, Opts) -> + QueueMode = maps:get(queue_mode, Opts, memory_only), + TotalBytes = maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE), + case QueueMode of + memory_only -> + #{ + mem_only => true, + marshaller => fun ?MODULE:queue_item_marshaller/1, + max_total_bytes => TotalBytes, + sizer => fun ?MODULE:estimate_size/1 + }; + volatile_offload -> + SegBytes0 = maps:get(queue_seg_bytes, Opts, TotalBytes), + SegBytes = min(SegBytes0, TotalBytes), + #{ + dir => disk_queue_dir(Id, Index), + marshaller => fun ?MODULE:queue_item_marshaller/1, + max_total_bytes => TotalBytes, + %% we don't want to retain the queue after + %% resource restarts. + offload => {true, volatile}, + seg_bytes => SegBytes, + sizer => fun ?MODULE:estimate_size/1 + } + end. + %% The request timeout should be greater than the resume interval, as %% it defines how often the buffer worker tries to unblock. If request %% timeout is <= resume interval and the buffer worker is ever diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 647a40fed..9116927fa 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -40,6 +40,7 @@ fields("resource_opts") -> ]; fields("creation_opts") -> [ + {queue_mode, fun queue_mode/1}, {worker_pool_size, fun worker_pool_size/1}, {health_check_interval, fun health_check_interval/1}, {resume_interval, fun resume_interval/1}, @@ -53,7 +54,8 @@ fields("creation_opts") -> {batch_size, fun batch_size/1}, {batch_time, fun batch_time/1}, {enable_queue, fun enable_queue/1}, - {max_queue_bytes, fun max_queue_bytes/1} + {max_queue_bytes, fun max_queue_bytes/1}, + {queue_seg_bytes, fun queue_seg_bytes/1} ]. resource_opts_meta() -> @@ -149,4 +151,17 @@ max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW; max_queue_bytes(required) -> false; max_queue_bytes(_) -> undefined. +queue_mode(type) -> enum([memory_only, volatile_offload]); +queue_mode(desc) -> ?DESC("queue_mode"); +queue_mode(default) -> memory_only; +queue_mode(required) -> false; +queue_mode(importance) -> ?IMPORTANCE_HIDDEN; +queue_mode(_) -> undefined. + +queue_seg_bytes(type) -> emqx_schema:bytesize(); +queue_seg_bytes(desc) -> ?DESC("queue_seg_bytes"); +queue_seg_bytes(required) -> false; +queue_seg_bytes(importance) -> ?IMPORTANCE_HIDDEN; +queue_seg_bytes(_) -> undefined. + desc("creation_opts") -> ?DESC("creation_opts"). diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 5d8e85697..087a39eee 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1314,6 +1314,7 @@ t_delete_and_re_create_with_same_name(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => NumBufferWorkers, + queue_mode => volatile_offload, queue_seg_bytes => 100, resume_interval => 1_000 } @@ -2639,6 +2640,117 @@ t_call_mode_uncoupled_from_query_mode(_Config) -> end ). +%% The default mode is currently `memory_only'. +t_volatile_offload_mode(_Config) -> + MaxQueueBytes = 1_000, + DefaultOpts = #{ + max_queue_bytes => MaxQueueBytes, + worker_pool_size => 1 + }, + ?check_trace( + begin + emqx_connector_demo:set_callback_mode(async_if_possible), + %% Create without any specified segment bytes; should + %% default to equal max bytes. + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{queue_mode => volatile_offload} + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + %% Create with segment bytes < max bytes + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + queue_mode => volatile_offload, + queue_seg_bytes => MaxQueueBytes div 2 + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + %% Create with segment bytes = max bytes + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + queue_mode => volatile_offload, + queue_seg_bytes => MaxQueueBytes + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + %% Create with segment bytes > max bytes; should normalize + %% to max bytes. + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + DefaultOpts#{ + queue_mode => volatile_offload, + queue_seg_bytes => 2 * MaxQueueBytes + } + ) + ), + ?assertEqual(ok, emqx_resource:remove_local(?ID)), + + ok + end, + fun(Trace) -> + HalfMaxQueueBytes = MaxQueueBytes div 2, + ?assertMatch( + [ + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + %% uses the specified value since it's smaller + %% than max bytes. + seg_bytes := HalfMaxQueueBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + }, + #{ + dir := _, + max_total_bytes := MaxTotalBytes, + seg_bytes := MaxTotalBytes, + offload := {true, volatile} + } + ], + ?projection(queue_opts, ?of_kind(buffer_worker_init, Trace)) + ), + ok + end + ). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/changes/ce/feat-10404.en.md b/changes/ce/feat-10404.en.md new file mode 100644 index 000000000..ad216336e --- /dev/null +++ b/changes/ce/feat-10404.en.md @@ -0,0 +1,2 @@ +Change the default queue mode for buffer workers to `memory_only`. +Before this change, the default queue mode was `volatile_offload`. When under high message rate pressure and when the resource is not keeping up with such rate, the buffer performance degraded a lot due to the constant disk operations. diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index c73f8b1aa..7b693c256 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -179,5 +179,43 @@ When disabled the messages are buffered in RAM only.""" } } + queue_seg_bytes { + desc { + en: "Applicable when buffer mode is set to volatile_offload.\n" + "This value is to specify the size of each on-disk buffer file." + zh: "当缓存模式是 volatile_offload 时适用。" + "该配置用于指定缓存到磁盘上的文件的大小。" + } + label { + en: "Segment File Bytes" + zh: "缓存文件大小" + } + } + + queue_mode { + desc { + en: "Queue operation mode.\n" + "\n" + "memory_only: Buffer all messages in memory." + " The messages will be lost in case of EMQX node restart.\n" + "volatile_offload: Buffer message in memory first, when up to certain limit" + " (see queue_seg_bytes config for more information), then start offloading" + " messages to disk, Like memory_only mode, the messages will be lost in case of" + " EMQX node restart." + zh: "队列操作模式。\n" + "\n" + "memory_only: 所有的消息都缓存在内存里。" + " 如果 EMQX 服务重启，缓存的消息会丢失。\n" + "hybrid: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" + "（配置项 queue_seg_bytes 描述了该限制）后，" + " 后续的消息会缓存到磁盘上。与 memory_only 模式一样，" + " 如果 EMQX 服务重启，缓存的消息会丢失。" + } + label { + en: "Queue Mode" + zh: "排队模式" + } + } + } From dd381227972f0965d680bd4d8872f6f15765f70a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 11:22:04 -0300 Subject: [PATCH 148/279] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- rel/i18n/emqx_resource_schema.hocon | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index 7b693c256..885a6c4dd 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -195,25 +195,18 @@ When disabled the messages are buffered in RAM only.""" queue_mode { desc { en: "Queue operation mode.\n" - "\n" "memory_only: Buffer all messages in memory." - " The messages will be lost in case of EMQX node restart.\n" "volatile_offload: Buffer message in memory first, when up to certain limit" - " (see queue_seg_bytes config for more information), then start offloading" - " messages to disk, Like memory_only mode, the messages will be lost in case of" - " EMQX node restart." + " (see buffer_seg_bytes config for more information), then start offloading messages to disk" zh: "队列操作模式。\n" - "\n" "memory_only: 所有的消息都缓存在内存里。" - " 如果 EMQX 服务重启，缓存的消息会丢失。\n" - "hybrid: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" - "（配置项 queue_seg_bytes 描述了该限制）后，" - " 后续的消息会缓存到磁盘上。与 memory_only 模式一样，" - " 如果 EMQX 服务重启，缓存的消息会丢失。" + "volatile_offload: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" + "（配置项 buffer_seg_bytes 该限制）后，" + " 消息会缓存到磁盘上" } label { - en: "Queue Mode" - zh: "排队模式" + en: "Buffer Mode" + zh: "缓存模式" } } From e073bc90bc76d3bea1387a0f4196a3fe6b23ac9a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 11:29:52 -0300 Subject: [PATCH 149/279] refactor(buffer_worker): rename `s/queue/buffer/g` --- apps/emqx_bridge/src/emqx_bridge_api.erl | 4 +- .../schema/emqx_bridge_compatible_config.erl | 2 +- .../test/emqx_bridge_webhook_SUITE.erl | 2 +- apps/emqx_resource/include/emqx_resource.hrl | 6 +-- .../src/emqx_resource_buffer_worker.erl | 8 ++-- .../src/schema/emqx_resource_schema.erl | 39 ++++++++++--------- .../test/emqx_resource_SUITE.erl | 30 +++++++------- .../src/emqx_ee_bridge_cassa.erl | 2 +- .../src/emqx_ee_bridge_clickhouse.erl | 2 +- .../src/emqx_ee_bridge_dynamo.erl | 2 +- .../src/emqx_ee_bridge_mysql.erl | 2 +- .../src/emqx_ee_bridge_pgsql.erl | 2 +- .../src/emqx_ee_bridge_rocketmq.erl | 2 +- .../src/emqx_ee_bridge_sqlserver.erl | 2 +- .../src/emqx_ee_bridge_tdengine.erl | 2 +- rel/i18n/emqx_resource_schema.hocon | 8 ++-- scripts/test/influx/influx-bridge.conf | 2 +- 17 files changed, 59 insertions(+), 58 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index b29cefacd..137cd1241 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -220,7 +220,7 @@ info_example_basic(webhook) -> auto_restart_interval => 15000, query_mode => async, inflight_window => 100, - max_queue_bytes => 100 * 1024 * 1024 + max_buffer_bytes => 100 * 1024 * 1024 } }; info_example_basic(mqtt) -> @@ -245,7 +245,7 @@ mqtt_main_example() -> health_check_interval => <<"15s">>, auto_restart_interval => <<"60s">>, query_mode => sync, - max_queue_bytes => 100 * 1024 * 1024 + max_buffer_bytes => 100 * 1024 * 1024 }, ssl => #{ enable => false diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl index fe173fa89..595b75ecf 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl @@ -89,7 +89,7 @@ default_resource_opts() -> <<"inflight_window">> => 100, <<"auto_restart_interval">> => <<"60s">>, <<"health_check_interval">> => <<"15s">>, - <<"max_queue_bytes">> => <<"1GB">>, + <<"max_buffer_bytes">> => <<"1GB">>, <<"query_mode">> => <<"sync">>, %% there is only one underlying MQTT connection %% doesn't make a lot of sense to have a large pool diff --git a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl index e222190d2..f08c87b6e 100644 --- a/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl @@ -175,7 +175,7 @@ bridge_async_config(#{port := Port} = Config) -> " inflight_window = 100\n" " auto_restart_interval = \"60s\"\n" " health_check_interval = \"15s\"\n" - " max_queue_bytes = \"1GB\"\n" + " max_buffer_bytes = \"1GB\"\n" " query_mode = \"~s\"\n" " request_timeout = \"~s\"\n" " start_after_created = \"true\"\n" diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index f8d671b40..91572eac3 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -70,7 +70,7 @@ auto_restart_interval => pos_integer(), batch_size => pos_integer(), batch_time => pos_integer(), - max_queue_bytes => pos_integer(), + max_buffer_bytes => pos_integer(), query_mode => query_mode(), resume_interval => pos_integer(), inflight_window => pos_integer() @@ -85,8 +85,8 @@ -define(WORKER_POOL_SIZE, 16). --define(DEFAULT_QUEUE_SIZE, 256 * 1024 * 1024). --define(DEFAULT_QUEUE_SIZE_RAW, <<"256MB">>). +-define(DEFAULT_BUFFER_BYTES, 256 * 1024 * 1024). +-define(DEFAULT_BUFFER_BYTES_RAW, <<"256MB">>). -define(DEFAULT_REQUEST_TIMEOUT, timer:seconds(15)). diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index c53627ca4..a77335140 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -1667,9 +1667,9 @@ adjust_batch_time(Id, RequestTimeout, BatchTime0) -> BatchTime. replayq_opts(Id, Index, Opts) -> - QueueMode = maps:get(queue_mode, Opts, memory_only), - TotalBytes = maps:get(max_queue_bytes, Opts, ?DEFAULT_QUEUE_SIZE), - case QueueMode of + BufferMode = maps:get(buffer_mode, Opts, memory_only), + TotalBytes = maps:get(max_buffer_bytes, Opts, ?DEFAULT_BUFFER_BYTES), + case BufferMode of memory_only -> #{ mem_only => true, @@ -1678,7 +1678,7 @@ replayq_opts(Id, Index, Opts) -> sizer => fun ?MODULE:estimate_size/1 }; volatile_offload -> - SegBytes0 = maps:get(queue_seg_bytes, Opts, TotalBytes), + SegBytes0 = maps:get(buffer_seg_bytes, Opts, TotalBytes), SegBytes = min(SegBytes0, TotalBytes), #{ dir => disk_queue_dir(Id, Index), diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 9116927fa..3b4fb66e5 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -40,7 +40,7 @@ fields("resource_opts") -> ]; fields("creation_opts") -> [ - {queue_mode, fun queue_mode/1}, + {buffer_mode, fun buffer_mode/1}, {worker_pool_size, fun worker_pool_size/1}, {health_check_interval, fun health_check_interval/1}, {resume_interval, fun resume_interval/1}, @@ -54,8 +54,8 @@ fields("creation_opts") -> {batch_size, fun batch_size/1}, {batch_time, fun batch_time/1}, {enable_queue, fun enable_queue/1}, - {max_queue_bytes, fun max_queue_bytes/1}, - {queue_seg_bytes, fun queue_seg_bytes/1} + {max_buffer_bytes, fun max_buffer_bytes/1}, + {buffer_seg_bytes, fun buffer_seg_bytes/1} ]. resource_opts_meta() -> @@ -145,23 +145,24 @@ batch_time(default) -> ?DEFAULT_BATCH_TIME_RAW; batch_time(required) -> false; batch_time(_) -> undefined. -max_queue_bytes(type) -> emqx_schema:bytesize(); -max_queue_bytes(desc) -> ?DESC("max_queue_bytes"); -max_queue_bytes(default) -> ?DEFAULT_QUEUE_SIZE_RAW; -max_queue_bytes(required) -> false; -max_queue_bytes(_) -> undefined. +max_buffer_bytes(type) -> emqx_schema:bytesize(); +max_buffer_bytes(aliases) -> [max_queue_bytes]; +max_buffer_bytes(desc) -> ?DESC("max_buffer_bytes"); +max_buffer_bytes(default) -> ?DEFAULT_BUFFER_BYTES_RAW; +max_buffer_bytes(required) -> false; +max_buffer_bytes(_) -> undefined. -queue_mode(type) -> enum([memory_only, volatile_offload]); -queue_mode(desc) -> ?DESC("queue_mode"); -queue_mode(default) -> memory_only; -queue_mode(required) -> false; -queue_mode(importance) -> ?IMPORTANCE_HIDDEN; -queue_mode(_) -> undefined. +buffer_mode(type) -> enum([memory_only, volatile_offload]); +buffer_mode(desc) -> ?DESC("buffer_mode"); +buffer_mode(default) -> memory_only; +buffer_mode(required) -> false; +buffer_mode(importance) -> ?IMPORTANCE_HIDDEN; +buffer_mode(_) -> undefined. -queue_seg_bytes(type) -> emqx_schema:bytesize(); -queue_seg_bytes(desc) -> ?DESC("queue_seg_bytes"); -queue_seg_bytes(required) -> false; -queue_seg_bytes(importance) -> ?IMPORTANCE_HIDDEN; -queue_seg_bytes(_) -> undefined. +buffer_seg_bytes(type) -> emqx_schema:bytesize(); +buffer_seg_bytes(desc) -> ?DESC("buffer_seg_bytes"); +buffer_seg_bytes(required) -> false; +buffer_seg_bytes(importance) -> ?IMPORTANCE_HIDDEN; +buffer_seg_bytes(_) -> undefined. desc("creation_opts") -> ?DESC("creation_opts"). diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 087a39eee..385b4cb91 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -1314,8 +1314,8 @@ t_delete_and_re_create_with_same_name(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => NumBufferWorkers, - queue_mode => volatile_offload, - queue_seg_bytes => 100, + buffer_mode => volatile_offload, + buffer_seg_bytes => 100, resume_interval => 1_000 } ), @@ -1374,7 +1374,7 @@ t_delete_and_re_create_with_same_name(_Config) -> query_mode => async, batch_size => 1, worker_pool_size => 2, - queue_seg_bytes => 100, + buffer_seg_bytes => 100, resume_interval => 1_000 } ), @@ -1406,7 +1406,7 @@ t_always_overflow(_Config) -> query_mode => sync, batch_size => 1, worker_pool_size => 1, - max_queue_bytes => 1, + max_buffer_bytes => 1, resume_interval => 1_000 } ), @@ -2642,9 +2642,9 @@ t_call_mode_uncoupled_from_query_mode(_Config) -> %% The default mode is currently `memory_only'. t_volatile_offload_mode(_Config) -> - MaxQueueBytes = 1_000, + MaxBufferBytes = 1_000, DefaultOpts = #{ - max_queue_bytes => MaxQueueBytes, + max_buffer_bytes => MaxBufferBytes, worker_pool_size => 1 }, ?check_trace( @@ -2659,7 +2659,7 @@ t_volatile_offload_mode(_Config) -> ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, #{name => test_resource}, - DefaultOpts#{queue_mode => volatile_offload} + DefaultOpts#{buffer_mode => volatile_offload} ) ), ?assertEqual(ok, emqx_resource:remove_local(?ID)), @@ -2673,8 +2673,8 @@ t_volatile_offload_mode(_Config) -> ?TEST_RESOURCE, #{name => test_resource}, DefaultOpts#{ - queue_mode => volatile_offload, - queue_seg_bytes => MaxQueueBytes div 2 + buffer_mode => volatile_offload, + buffer_seg_bytes => MaxBufferBytes div 2 } ) ), @@ -2688,8 +2688,8 @@ t_volatile_offload_mode(_Config) -> ?TEST_RESOURCE, #{name => test_resource}, DefaultOpts#{ - queue_mode => volatile_offload, - queue_seg_bytes => MaxQueueBytes + buffer_mode => volatile_offload, + buffer_seg_bytes => MaxBufferBytes } ) ), @@ -2705,8 +2705,8 @@ t_volatile_offload_mode(_Config) -> ?TEST_RESOURCE, #{name => test_resource}, DefaultOpts#{ - queue_mode => volatile_offload, - queue_seg_bytes => 2 * MaxQueueBytes + buffer_mode => volatile_offload, + buffer_seg_bytes => 2 * MaxBufferBytes } ) ), @@ -2715,7 +2715,7 @@ t_volatile_offload_mode(_Config) -> ok end, fun(Trace) -> - HalfMaxQueueBytes = MaxQueueBytes div 2, + HalfMaxBufferBytes = MaxBufferBytes div 2, ?assertMatch( [ #{ @@ -2729,7 +2729,7 @@ t_volatile_offload_mode(_Config) -> max_total_bytes := MaxTotalBytes, %% uses the specified value since it's smaller %% than max bytes. - seg_bytes := HalfMaxQueueBytes, + seg_bytes := HalfMaxBufferBytes, offload := {true, volatile} }, #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl index 78db8352a..26c6de04d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl @@ -63,7 +63,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl index 0b611c142..56671c586 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl @@ -61,7 +61,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl index e6a3d1a58..ba1fd0c70 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl @@ -56,7 +56,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl index f3ed44247..7914c77e2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl @@ -57,7 +57,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl index 958bc3449..a5dcb19e6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl @@ -59,7 +59,7 @@ values(_Method, Type) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl index 78fd527d3..28b94a1a4 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_rocketmq.erl @@ -56,7 +56,7 @@ values(post) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }; values(put) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl index 49a5ed0ce..e216299c2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_sqlserver.erl @@ -60,7 +60,7 @@ values(post) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => async, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }; values(put) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl index 7a958d45f..54406541d 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl @@ -58,7 +58,7 @@ values(_Method) -> batch_size => ?DEFAULT_BATCH_SIZE, batch_time => ?DEFAULT_BATCH_TIME, query_mode => sync, - max_queue_bytes => ?DEFAULT_QUEUE_SIZE + max_buffer_bytes => ?DEFAULT_BUFFER_BYTES } }. diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index 885a6c4dd..031a5b412 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -168,7 +168,7 @@ When disabled the messages are buffered in RAM only.""" } } - max_queue_bytes { + max_buffer_bytes { desc { en: """Maximum number of bytes to buffer for each buffer worker.""" zh: """每个缓存 worker 允许使用的最大字节数。""" @@ -179,7 +179,7 @@ When disabled the messages are buffered in RAM only.""" } } - queue_seg_bytes { + buffer_seg_bytes { desc { en: "Applicable when buffer mode is set to volatile_offload.\n" "This value is to specify the size of each on-disk buffer file." @@ -192,9 +192,9 @@ When disabled the messages are buffered in RAM only.""" } } - queue_mode { + buffer_mode { desc { - en: "Queue operation mode.\n" + en: "Buffer operation mode.\n" "memory_only: Buffer all messages in memory." "volatile_offload: Buffer message in memory first, when up to certain limit" " (see buffer_seg_bytes config for more information), then start offloading messages to disk" diff --git a/scripts/test/influx/influx-bridge.conf b/scripts/test/influx/influx-bridge.conf index 0416e42b6..0574ac38a 100644 --- a/scripts/test/influx/influx-bridge.conf +++ b/scripts/test/influx/influx-bridge.conf @@ -11,7 +11,7 @@ bridges { batch_size = 100 batch_time = "10ms" health_check_interval = "15s" - max_queue_bytes = "1GB" + max_buffer_bytes = "1GB" query_mode = "sync" request_timeout = "15s" start_after_created = "true" From 24df1045dec62fdeb777fe6be8a4de785c974a4d Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 16:40:25 +0200 Subject: [PATCH 150/279] fix: test not updated after rebase --- apps/emqx_utils/test/emqx_utils_SUITE.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_utils/test/emqx_utils_SUITE.erl b/apps/emqx_utils/test/emqx_utils_SUITE.erl index 598f05a21..99516b0eb 100644 --- a/apps/emqx_utils/test/emqx_utils_SUITE.erl +++ b/apps/emqx_utils/test/emqx_utils_SUITE.erl @@ -147,7 +147,10 @@ t_check(_) -> [self() ! {msg, I} || I <- lists:seq(1, 5)], ?assertEqual(ok, emqx_utils:check_oom(Policy)), [self() ! {msg, I} || I <- lists:seq(1, 6)], - ?assertEqual({shutdown, message_queue_too_long}, emqx_utils:check_oom(Policy)). + ?assertEqual( + {shutdown, #{reason => message_queue_too_long, value => 11, max => 10}}, + emqx_utils:check_oom(Policy) + ). drain() -> drain([]). From 8facd130f695464370949945a8a9cbb51d4624a4 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 22:45:28 +0800 Subject: [PATCH 151/279] chore: update cluster_hocon_file/0 function --- apps/emqx/src/emqx_config.erl | 22 +++++----------- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 26 +++++++------------ .../test/emqx_mgmt_api_configs_SUITE.erl | 17 +++++++++--- .../test/emqx_rule_funcs_SUITE.erl | 2 +- changes/ce/feat-10156.en.md | 7 +++++ 5 files changed, 37 insertions(+), 37 deletions(-) create mode 100644 changes/ce/feat-10156.en.md diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index b676b07e2..ab028cf7f 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -418,7 +418,7 @@ do_parse_hocon(false, Conf, IncDirs) -> true -> hocon:binary(Conf, Opts); false -> - ClusterFile = cluster_hocon_file(#{override_to => cluster}), + ClusterFile = cluster_hocon_file(), hocon:files([ClusterFile | Conf], Opts) end. @@ -493,7 +493,7 @@ fill_defaults(SchemaMod, RawConf, Opts0) -> delete_override_conf_files() -> F1 = deprecated_conf_file(#{override_to => local}), F2 = deprecated_conf_file(#{override_to => cluster}), - F3 = cluster_hocon_file(#{override_to => cluster}), + F3 = cluster_hocon_file(), ok = ensure_file_deleted(F1), ok = ensure_file_deleted(F2), ok = ensure_file_deleted(F3). @@ -510,7 +510,7 @@ read_override_conf(#{} = Opts) -> File = case has_deprecated_file() of true -> deprecated_conf_file(Opts); - false -> cluster_hocon_file(Opts) + false -> cluster_hocon_file() end, load_hocon_file(File, map). @@ -531,16 +531,8 @@ deprecated_conf_file(Which) when is_atom(Which) -> application:get_env(emqx, Which, undefined). %% The newer version cluster-wide config persistence file. -cluster_hocon_file(Opts) when is_map(Opts) -> - Key = - case maps:get(override_to, Opts, cluster) of - %% no local config file support - local -> undefined; - cluster -> cluster_hocon_file - end, - application:get_env(emqx, Key, undefined); -cluster_hocon_file(Which) when is_atom(Which) -> - application:get_env(emqx, Which, undefined). +cluster_hocon_file() -> + application:get_env(emqx, cluster_hocon_file, undefined). -spec save_schema_mod_and_names(module()) -> ok. save_schema_mod_and_names(SchemaMod) -> @@ -619,8 +611,8 @@ save_to_override_conf(true, RawConf, Opts) -> {error, Reason} end end; -save_to_override_conf(false, RawConf, Opts) -> - case cluster_hocon_file(Opts) of +save_to_override_conf(false, RawConf, _Opts) -> + case cluster_hocon_file() of undefined -> ok; FileName -> diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index 825f01441..583405158 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -59,14 +59,15 @@ t_copy_new_data_dir(_Config) -> %% 1. Start all nodes [First | Rest] = Nodes = start_cluster(Cluster), try + File = "/configs/cluster.hocon", assert_config_load_done(Nodes), - rpc:call(First, ?MODULE, create_data_dir, []), + rpc:call(First, ?MODULE, create_data_dir, [File]), {[ok, ok, ok], []} = rpc:multicall(Nodes, application, stop, [emqx_conf]), {[ok, ok, ok], []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []), ok = rpc:call(First, application, start, [emqx_conf]), {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]), - assert_data_copy_done(Nodes, "/configs/cluster.hocon"), + assert_data_copy_done(Nodes, File), stop_cluster(Nodes), ok after @@ -82,14 +83,15 @@ t_copy_deprecated_data_dir(_Config) -> %% 1. Start all nodes [First | Rest] = Nodes = start_cluster(Cluster), try + File = "/configs/cluster-override.conf", assert_config_load_done(Nodes), - rpc:call(First, ?MODULE, create_deprecated_data_dir, []), + rpc:call(First, ?MODULE, create_data_dir, [File]), {[ok, ok, ok], []} = rpc:multicall(Nodes, application, stop, [emqx_conf]), {[ok, ok, ok], []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []), ok = rpc:call(First, application, start, [emqx_conf]), {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]), - assert_data_copy_done(Nodes, "/configs/cluster-override.conf"), + assert_data_copy_done(Nodes, File), stop_cluster(Nodes), ok after @@ -100,7 +102,7 @@ t_copy_deprecated_data_dir(_Config) -> %% Helper functions %%------------------------------------------------------------------------------ -create_data_dir() -> +create_data_dir(File) -> Node = atom_to_list(node()), ok = filelib:ensure_dir(Node ++ "/certs/"), ok = filelib:ensure_dir(Node ++ "/authz/"), @@ -108,17 +110,7 @@ create_data_dir() -> ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)), ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)), Telemetry = <<"telemetry.enable = false">>, - ok = file:write_file(Node ++ "/configs/cluster.hocon", Telemetry). - -create_deprecated_data_dir() -> - Node = atom_to_list(node()), - ok = filelib:ensure_dir(Node ++ "/certs/"), - ok = filelib:ensure_dir(Node ++ "/authz/"), - ok = filelib:ensure_dir(Node ++ "/configs/"), - ok = file:write_file(Node ++ "/certs/fake-cert", list_to_binary(Node)), - ok = file:write_file(Node ++ "/authz/fake-authz", list_to_binary(Node)), - Telemetry = <<"telemetry.enable = false">>, - ok = file:write_file(Node ++ "/configs/cluster-override.conf", Telemetry). + ok = file:write_file(Node ++ File, Telemetry). set_data_dir_env() -> Node = atom_to_list(node()), @@ -134,7 +126,7 @@ set_data_dir_env() -> application:set_env(emqx, config_files, [NewConfigFile]), application:set_env(emqx, data_dir, Node), %% We set env both cluster.hocon and cluster-override.conf, but only one will be used - application:set_env(emqx, cluster_conf_file, Node ++ "/configs/cluster.hocon"), + application:set_env(emqx, cluster_hocon_file, Node ++ "/configs/cluster.hocon"), application:set_env(emqx, cluster_override_conf_file, Node ++ "/configs/cluster-override.conf"), ok. diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index f5ee4e5be..fc1620fec 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -62,8 +62,8 @@ t_update(_Config) -> ?assertEqual(BusyPort, not BusyPort1), assert_busy_port(BusyPort1), %% Make sure the override config is updated, and remove the default value. - ?assertEqual( - #{<<"vm">> => #{<<"busy_port">> => BusyPort1}}, + ?assertMatch( + #{<<"vm">> := #{<<"busy_port">> := BusyPort1}}, maps:get(<<"sysmon">>, emqx_config:read_override_conf(#{override_to => cluster})) ), @@ -136,7 +136,7 @@ t_global_zone(_Config) -> {ok, #{}} = update_global_zone(NewZones), ?assertEqual(1, emqx_config:get_zone_conf(no_default, [mqtt, max_qos_allowed])), %% Make sure the override config is updated, and remove the default value. - ?assertEqual(#{<<"max_qos_allowed">> => 1}, read_conf(<<"mqtt">>)), + ?assertMatch(#{<<"max_qos_allowed">> := 1}, read_conf(<<"mqtt">>)), BadZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 3), ?assertMatch({error, {"HTTP/1.1", 400, _}}, update_global_zone(BadZones)), @@ -155,7 +155,16 @@ t_global_zone(_Config) -> DefaultZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2), {ok, #{}} = update_global_zone(DefaultZones), - ?assertEqual(undefined, read_conf(<<"mqtt">>)), + #{<<"mqtt">> := Mqtt} = emqx_config:fill_defaults(emqx_schema, #{<<"mqtt">> => #{}}, #{}), + Default = maps:map( + fun + (_, V) when is_boolean(V) -> V; + (_, V) when is_atom(V) -> atom_to_binary(V); + (_, V) -> V + end, + Mqtt + ), + ?assertEqual(Default, read_conf(<<"mqtt">>)), ok. get_global_zone() -> diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 94adb3506..ee798868e 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -686,7 +686,7 @@ t_jq(_) -> %% Got timeout as expected got_timeout end, - ConfigRootKey = emqx_rule_engine_schema:namespace(), + _ConfigRootKey = emqx_rule_engine_schema:namespace(), ?assertThrow( {jq_exception, {timeout, _}}, apply_func(jq, [TOProgram, <<"-2">>]) diff --git a/changes/ce/feat-10156.en.md b/changes/ce/feat-10156.en.md new file mode 100644 index 000000000..bc864fcc3 --- /dev/null +++ b/changes/ce/feat-10156.en.md @@ -0,0 +1,7 @@ +Change the priority of the configuration: +1. If it is a new installation of EMQX, the priority of s configuration is `ENV > emqx.conf > HTTP API`. +2. If it is an upgrade of EMQX, the priority of s configuration is the same as before `HTTP API > ENV > emqx.conf`. + +Deprecated data/configs/local-override.conf. + +Stabilizing the HTTP API for hot updates. From 16c49b2cc185f703b15590d443382b642d74e6af Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 17:21:38 +0200 Subject: [PATCH 152/279] fix: undo wrong thinking about returned types from decode --- apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl index 672ddc73c..ab673b225 100644 --- a/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_cache_SUITE.erl @@ -60,19 +60,19 @@ set_special_configs(emqx_authz) -> set_special_configs(_App) -> ok. -t_clean_cahce(_) -> +t_clean_cache(_) -> {ok, C} = emqtt:start_link([{clientid, <<"emqx0">>}, {username, <<"emqx0">>}]), {ok, _} = emqtt:connect(C), {ok, _, _} = emqtt:subscribe(C, <<"a/b/c">>, 0), ok = emqtt:publish(C, <<"a/b/c">>, <<"{\"x\":1,\"y\":1}">>, 0), {ok, 200, Result3} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(2, maps:size(emqx_utils_json:decode(Result3))), + ?assertEqual(2, length(emqx_utils_json:decode(Result3))), request(delete, uri(["authorization", "cache"])), {ok, 200, Result4} = request(get, uri(["clients", "emqx0", "authorization", "cache"])), - ?assertEqual(0, maps:size(emqx_utils_json:decode(Result4))), + ?assertEqual(0, length(emqx_utils_json:decode(Result4))), ok. From 8948be49e7cbff3e512f346dcd35d63e637cf0c4 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 23:29:41 +0800 Subject: [PATCH 153/279] feat: hide node advance config --- apps/emqx_conf/src/emqx_conf_schema.erl | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index f689997c2..140e9c17d 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -457,7 +457,7 @@ fields("node") -> mapping => "vm_args.+e", desc => ?DESC(max_ets_tables), default => 262144, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, 'readOnly' => true } )}, @@ -503,7 +503,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP", desc => ?DESC(node_crash_dump_file), default => crash_dump_file_default(), - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, 'readOnly' => true } )}, @@ -514,7 +514,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP_SECONDS", default => <<"30s">>, desc => ?DESC(node_crash_dump_seconds), - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, 'readOnly' => true } )}, @@ -525,7 +525,7 @@ fields("node") -> mapping => "vm_args.-env ERL_CRASH_DUMP_BYTES", default => <<"100MB">>, desc => ?DESC(node_crash_dump_bytes), - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, 'readOnly' => true } )}, @@ -536,7 +536,7 @@ fields("node") -> mapping => "vm_args.-kernel net_ticktime", default => <<"2m">>, 'readOnly' => true, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(node_dist_net_ticktime) } )}, @@ -547,7 +547,7 @@ fields("node") -> mapping => "emqx_machine.backtrace_depth", default => 23, 'readOnly' => true, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(node_backtrace_depth) } )}, @@ -558,7 +558,7 @@ fields("node") -> mapping => "emqx_machine.applications", default => [], 'readOnly' => true, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(node_applications) } )}, @@ -568,7 +568,7 @@ fields("node") -> #{ desc => ?DESC(node_etc_dir), 'readOnly' => true, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, deprecated => {since, "5.0.8"} } )}, @@ -577,7 +577,7 @@ fields("node") -> ?R_REF("cluster_call"), #{ 'readOnly' => true, - importance => ?IMPORTANCE_LOW + importance => ?IMPORTANCE_HIDDEN } )}, {"db_backend", @@ -591,7 +591,7 @@ fields("node") -> desc => ?DESC(db_backend) } )}, - {"db_role", + {"role", sc( hoconsc:enum([core, replicant]), #{ @@ -599,6 +599,7 @@ fields("node") -> default => core, 'readOnly' => true, importance => ?IMPORTANCE_HIGH, + aliases => [db_role], desc => ?DESC(db_role) } )}, @@ -620,7 +621,7 @@ fields("node") -> mapping => "mria.tlog_push_mode", default => async, 'readOnly' => true, - importance => ?IMPORTANCE_LOW, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(db_tlog_push_mode) } )}, From 97a936ad4f8e2f7df317c6a7c3284d31f2c0af6a Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 14 Apr 2023 23:46:05 +0800 Subject: [PATCH 154/279] test: fix failed test --- apps/emqx/test/emqx_channel_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 6dd389350..29f8b1503 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -1137,7 +1137,7 @@ t_ws_cookie_init(_) -> %%-------------------------------------------------------------------- t_flapping_detect(_) -> - emqx_config:put_zone_conf(default, [flapping_detect, enable], true), + emqx_config:put_zone_conf(default, [flapping_detect, window_time], 60000), Parent = self(), ok = meck:expect( emqx_cm, From 3373a63137a7c0443831985919fef131c2d7d6b4 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 13:21:30 -0300 Subject: [PATCH 155/279] docs: improve descriptions Co-authored-by: Zaiming (Stone) Shi --- rel/i18n/emqx_resource_schema.hocon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index 031a5b412..f4a9982bc 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -201,8 +201,8 @@ When disabled the messages are buffered in RAM only.""" zh: "队列操作模式。\n" "memory_only: 所有的消息都缓存在内存里。" "volatile_offload: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" - "（配置项 buffer_seg_bytes 该限制）后，" - " 消息会缓存到磁盘上" + "（配置项 buffer_seg_bytes 指定该限制）后，" + " 消息会开始缓存到磁盘上。" } label { en: "Buffer Mode" From f0c13e01345f367553fddb1874677fbf0e906993 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 18:43:53 +0200 Subject: [PATCH 156/279] fix: stale ref to emqx_map_lib --- lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl index 6e6d7bf03..68bf7a057 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_sqlserver_SUITE.erl @@ -489,7 +489,7 @@ create_bridge(Config, Overrides) -> BridgeType = ?config(sqlserver_bridge_type, Config), Name = ?config(sqlserver_name, Config), SSConfig0 = ?config(sqlserver_config, Config), - SSConfig = emqx_map_lib:deep_merge(SSConfig0, Overrides), + SSConfig = emqx_utils_maps:deep_merge(SSConfig0, Overrides), emqx_bridge:create(BridgeType, Name, SSConfig). delete_bridge(Config) -> From a20797160e11336a765f308fe3b2024fc548373f Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 18:45:25 +0200 Subject: [PATCH 157/279] style: remove unnecessary ifdef(TEST) --- apps/emqx_utils/test/emqx_utils_binary_tests.erl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apps/emqx_utils/test/emqx_utils_binary_tests.erl b/apps/emqx_utils/test/emqx_utils_binary_tests.erl index 3709fb014..79851dca5 100644 --- a/apps/emqx_utils/test/emqx_utils_binary_tests.erl +++ b/apps/emqx_utils/test/emqx_utils_binary_tests.erl @@ -26,8 +26,6 @@ -include_lib("eunit/include/eunit.hrl"). --ifdef(TEST). - trim1_test_() -> [ ?_assertEqual(trim(<<>>), <<>>), @@ -213,5 +211,3 @@ optimize_patterns_test_() -> optimize_patterns([<<"test">>, <<"t">>, <<"t">>]) ) ]. - --endif. From 9ccfa643cecfb98e826a86a6175f260b0c7efbbc Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 18:51:51 +0200 Subject: [PATCH 158/279] style: fix wording Co-authored-by: Thales Macedo Garitezi --- apps/emqx_utils/README.md | 6 +++--- apps/emqx_utils/src/emqx_utils.app.src | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_utils/README.md b/apps/emqx_utils/README.md index 046b66321..f8c386f3d 100644 --- a/apps/emqx_utils/README.md +++ b/apps/emqx_utils/README.md @@ -1,10 +1,10 @@ -# emqx_utils - Erlang utility library for EMQ X +# Erlang utility library for EMQX ## Overview -`emqx_utils` is a collection of utility functions for EMQ X, organized into +`emqx_utils` is a collection of utility functions for EMQX, organized into several modules. It provides various functionalities to make it easier to work -with EMQ X, such as binary manipulations, maps, JSON en- and decoding, ets table +with EMQX, such as binary manipulations, maps, JSON en- and decoding, ets table handling, data conversions, and more. ## Features diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index b0dedbd72..eb6371411 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -1,6 +1,6 @@ %% -*- mode: erlang -*- {application, emqx_utils, [ - {description, "An OTP application"}, + {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! {vsn, "5.0.0"}, {modules, [ From 89fa07c9be64465c50d6762b9849513d758b2093 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 14 Apr 2023 18:53:01 +0200 Subject: [PATCH 159/279] fix: emqx_utils is just a library, so 'load' is enough Co-authored-by: Thales Macedo Garitezi --- mix.exs | 2 +- rebar.config.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 299f23ce6..454c398a2 100644 --- a/mix.exs +++ b/mix.exs @@ -303,7 +303,7 @@ defmodule EMQXUmbrella.MixProject do tools: :load, covertool: :load, system_monitor: :load, - emqx_utils: :permanent, + emqx_utils: :load, emqx_http_lib: :permanent, emqx_resource: :permanent, emqx_connector: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index b6b6a7c18..7c00622c2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -391,7 +391,7 @@ relx_apps(ReleaseType, Edition) -> {covertool, load}, % started by emqx_machine {system_monitor, load}, - emqx_utils, + {emqx_utils, load}, emqx_http_lib, emqx_resource, emqx_connector, From 8461551f51cf5ee56ac315172f06bf8b75306bdb Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Sat, 15 Apr 2023 06:56:33 +0800 Subject: [PATCH 160/279] test: rename module from emqx_map_lib to emqx_utils_maps --- apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 5fe9f881e..db735bb27 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -159,7 +159,7 @@ t_global_zone(_Config) -> ?assertEqual(2, emqx_utils_maps:deep_get([<<"max_qos_allowed">>], Mqtt3)), ok = emqx_config:put_raw([<<"mqtt">>], Mqtt0), - DefaultZones = emqx_map_lib:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2), + DefaultZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 2), {ok, #{}} = update_global_zone(DefaultZones), #{<<"mqtt">> := Mqtt} = emqx_config:fill_defaults(emqx_schema, #{<<"mqtt">> => #{}}, #{}), Default = maps:map( @@ -324,7 +324,7 @@ reset_config(Name, Key) -> read_conf(RootKeys) when is_list(RootKeys) -> case emqx_config:read_override_conf(#{override_to => cluster}) of undefined -> undefined; - Conf -> emqx_map_lib:deep_get(RootKeys, Conf, undefined) + Conf -> emqx_utils_maps:deep_get(RootKeys, Conf, undefined) end; read_conf(RootKey) -> read_conf([RootKey]). From e1faae451a27801fb4ae23cc916f2ba9774aab20 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sat, 15 Apr 2023 18:17:28 +0800 Subject: [PATCH 161/279] chore: Update changes/ce/feat-10156.en.md Co-authored-by: JianBo He --- changes/ce/feat-10156.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/feat-10156.en.md b/changes/ce/feat-10156.en.md index bc864fcc3..c1291a7ce 100644 --- a/changes/ce/feat-10156.en.md +++ b/changes/ce/feat-10156.en.md @@ -1,5 +1,5 @@ Change the priority of the configuration: -1. If it is a new installation of EMQX, the priority of s configuration is `ENV > emqx.conf > HTTP API`. +1. If it is a new installation of EMQX, the priority of configuration is `ENV > emqx.conf > HTTP API`. 2. If it is an upgrade of EMQX, the priority of s configuration is the same as before `HTTP API > ENV > emqx.conf`. Deprecated data/configs/local-override.conf. From 3a511c6229b56ec34f5f987961ee811b4ba82473 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 15 Apr 2023 19:41:07 +0800 Subject: [PATCH 162/279] fix(gw): load emqx applications before hocon configs checking --- apps/emqx_gateway/src/emqx_gateway_app.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_utils.erl | 6 +++--- bin/nodetool | 16 +++++++++++++++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 01a1aaddd..f0406bcaa 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -45,7 +45,7 @@ load_default_gateway_applications() -> fun(Def) -> load_gateway_application(Def) end, - emqx_gateway_utils:find_gateway_definations() + emqx_gateway_utils:find_gateway_definitions() ). load_gateway_application( diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index f0e65627f..8c80fc1fa 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -75,7 +75,7 @@ fields(gateway) -> } )} end, - emqx_gateway_utils:find_gateway_definations() + emqx_gateway_utils:find_gateway_definitions() ); fields(clientinfo_override) -> [ diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 07185ef42..7a0188387 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -47,7 +47,7 @@ listener_chain/3, make_deprecated_paths/1, make_compatible_schema/2, - find_gateway_definations/0 + find_gateway_definitions/0 ]). -export([stringfy/1]). @@ -564,8 +564,8 @@ make_compatible_schema2(Path, SchemaFun) -> Schema ). --spec find_gateway_definations() -> list(gateway_def()). -find_gateway_definations() -> +-spec find_gateway_definitions() -> list(gateway_def()). +find_gateway_definitions() -> lists:flatten( lists:map( fun(App) -> diff --git a/bin/nodetool b/bin/nodetool index 9a5d5e069..9f32f9b3b 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -362,6 +362,20 @@ add_libs_dir() -> add_lib_dir(RootDir, Name, Vsn) -> LibDir = filename:join([RootDir, lib, atom_to_list(Name) ++ "-" ++ Vsn, ebin]), case code:add_patha(LibDir) of - true -> ok; + true -> + %% load all applications into application controller, before performing + %% the configuration check of HOCON + %% + %% It helps to implement the feature of dynamically searching schema. + %% See `emqx_gateway_schema:fields(gateway)` + is_emqx_application(Name) andalso application:load(Name), + ok; {error, _} -> error(LibDir) end. + +is_emqx_application(Name) when is_atom(Name) -> + is_emqx_application(atom_to_list(Name)); +is_emqx_application("emqx_" ++ _Rest) -> + true; +is_emqx_application(_) -> + false. From 73c15d9668e552d4b1bf3caad0b04eca8b2b20b9 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 15 Apr 2023 21:18:37 +0800 Subject: [PATCH 163/279] chore: update changes --- bin/nodetool | 11 +++++++++-- changes/ce/fix-10410.en.md | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 changes/ce/fix-10410.en.md diff --git a/bin/nodetool b/bin/nodetool index 9f32f9b3b..8170e68e2 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -272,7 +272,7 @@ chkconfig(File) -> end. check_license(Config) -> - ok = application:load(emqx_license), + ok = ensure_application_load(emqx_license), %% This checks formal license validity to ensure %% that the node can successfully start with the given license. @@ -368,7 +368,7 @@ add_lib_dir(RootDir, Name, Vsn) -> %% %% It helps to implement the feature of dynamically searching schema. %% See `emqx_gateway_schema:fields(gateway)` - is_emqx_application(Name) andalso application:load(Name), + is_emqx_application(Name) andalso ensure_application_load(Name), ok; {error, _} -> error(LibDir) end. @@ -379,3 +379,10 @@ is_emqx_application("emqx_" ++ _Rest) -> true; is_emqx_application(_) -> false. + +ensure_application_load(Name) -> + case application:load(Name) of + ok -> ok; + {error, {already_loaded, _}} -> ok; + {error, Reason} -> error({failed_to_load_application, Name, Reason}) + end. diff --git a/changes/ce/fix-10410.en.md b/changes/ce/fix-10410.en.md new file mode 100644 index 000000000..48b55ea31 --- /dev/null +++ b/changes/ce/fix-10410.en.md @@ -0,0 +1,2 @@ +Fix EMQX starting failed once any gateways configured in emqx.conf. +This issue was first introduced in v5.0.22 via [#10278](https://github.com/emqx/emqx/pull/10278). From 454d244535aa7bee1ff2630c43e20874e5c64558 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Sun, 16 Apr 2023 09:58:22 +0800 Subject: [PATCH 164/279] chore: update changes/ce/feat-10156.en.md Co-authored-by: JianBo He --- changes/ce/feat-10156.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/feat-10156.en.md b/changes/ce/feat-10156.en.md index c1291a7ce..589578f26 100644 --- a/changes/ce/feat-10156.en.md +++ b/changes/ce/feat-10156.en.md @@ -1,6 +1,6 @@ Change the priority of the configuration: 1. If it is a new installation of EMQX, the priority of configuration is `ENV > emqx.conf > HTTP API`. -2. If it is an upgrade of EMQX, the priority of s configuration is the same as before `HTTP API > ENV > emqx.conf`. +2. If EMQX is upgraded from an old version (i.e., the cluster-override.conf file still exists in EMQX's data directory), then the configuration priority remains the same as before. That is, `HTTP API > ENV > emqx.conf`. Deprecated data/configs/local-override.conf. From ad65fefac2f98baf55a2c80bf7f7266cc81b1aa0 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Sun, 16 Apr 2023 10:26:12 +0800 Subject: [PATCH 165/279] fix: rename emqx_misc to emqx_utils --- apps/emqx/src/emqx_flapping.erl | 2 +- apps/emqx/src/emqx_schema.erl | 2 +- changes/ce/feat-10358.en.md | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changes/ce/feat-10358.en.md diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index 9a4d56bc6..70b1a3232 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -193,7 +193,7 @@ code_change(_OldVsn, State, _Extra) -> start_timer(Zone) -> case get_policy(window_time, Zone) of WindowTime when is_integer(WindowTime) -> - emqx_misc:start_timer(WindowTime, {garbage_collect, Zone}); + emqx_utils:start_timer(WindowTime, {garbage_collect, Zone}); disabled -> ok end. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index da856badc..8335d69b8 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -628,7 +628,7 @@ fields("flapping_detect") -> boolean(), #{ default => false, - deprecated => {since, "5.0.22"}, + deprecated => {since, "5.0.23"}, desc => ?DESC(flapping_detect_enable) } )}, diff --git a/changes/ce/feat-10358.en.md b/changes/ce/feat-10358.en.md new file mode 100644 index 000000000..e6d05c84b --- /dev/null +++ b/changes/ce/feat-10358.en.md @@ -0,0 +1,2 @@ +Hide `flapping_detect/conn_congestion/stats` configuration. +Deprecate `flapping_detect.enable`. From 98e8287260f3e000eab7715082e8be17a2b06c91 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Sun, 16 Apr 2023 10:11:36 +0800 Subject: [PATCH 166/279] fix: hidden password in dashboard https's shema --- apps/emqx_dashboard/src/emqx_dashboard_schema.erl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 0ba970842..d3e4233d3 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -97,13 +97,15 @@ fields("https") -> [ enable(false), bind(18084) - | common_listener_fields() ++ - exclude_fields( - ["fail_if_no_peer_cert", "password"], - emqx_schema:server_ssl_opts_schema(#{}, true) - ) + | common_listener_fields() ++ server_ssl_opts() ]. +server_ssl_opts() -> + Opts0 = emqx_schema:server_ssl_opts_schema(#{}, true), + Opts1 = exclude_fields(["fail_if_no_peer_cert"], Opts0), + {value, {_, Meta}, Opts2} = lists:keytake("password", 1, Opts1), + [{"password", Meta#{importance => ?IMPORTANCE_HIDDEN}} | Opts2]. + exclude_fields([], Fields) -> Fields; exclude_fields([FieldName | Rest], Fields) -> From 55376144ce183bb39669cd4f1bdd87bf8ab5e658 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 17 Apr 2023 10:06:36 +0800 Subject: [PATCH 167/279] fix(limiter): simplify the configuration of the limiter --- apps/emqx/src/emqx_connection.erl | 6 +- .../src/emqx_limiter/src/emqx_htb_limiter.erl | 3 +- .../src/emqx_limiter_bucket_ref.erl | 5 + .../emqx_limiter/src/emqx_limiter_schema.erl | 152 ++++++++++++------ .../emqx_limiter/src/emqx_limiter_server.erl | 46 +++--- apps/emqx/src/emqx_ws_connection.erl | 4 +- apps/emqx/test/emqx_ratelimiter_SUITE.erl | 75 ++++----- rel/i18n/emqx_limiter_schema.hocon | 34 +--- 8 files changed, 183 insertions(+), 142 deletions(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 8d47f033c..27b6f3e84 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -182,10 +182,8 @@ -define(ALARM_SOCK_STATS_KEYS, [send_pend, recv_cnt, recv_oct, send_cnt, send_oct]). -define(ALARM_SOCK_OPTS_KEYS, [high_watermark, high_msgq_watermark, sndbuf, recbuf, buffer]). -%% use macro to do compile time limiter's type check --define(LIMITER_BYTES_IN, bytes_in). --define(LIMITER_MESSAGE_IN, message_in). --define(EMPTY_QUEUE, {[], []}). +-define(LIMITER_BYTES_IN, bytes). +-define(LIMITER_MESSAGE_IN, messages). -dialyzer({no_match, [info/2]}). -dialyzer( diff --git a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl index bbebd9460..53f26deb5 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl @@ -139,7 +139,8 @@ make_token_bucket_limiter(Cfg, Bucket) -> Cfg#{ tokens => emqx_limiter_server:get_initial_val(Cfg), lasttime => ?NOW, - bucket => Bucket + bucket => Bucket, + capacity => emqx_limiter_schema:calc_capacity(Cfg) }. %%@doc create a limiter server's reference diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl index fe30e41e9..139564df7 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_bucket_ref.erl @@ -23,6 +23,7 @@ %% API -export([ new/3, + infinity_bucket/0, check/3, try_restore/2, available/1 @@ -58,6 +59,10 @@ new(Counter, Index, Rate) -> rate => Rate }. +-spec infinity_bucket() -> bucket_ref(). +infinity_bucket() -> + infinity. + %% @doc check tokens -spec check(pos_integer(), bucket_ref(), Disivisble :: boolean()) -> HasToken :: diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index f45fc55b6..f59ddc35b 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -31,20 +31,20 @@ get_bucket_cfg_path/2, desc/1, types/0, - infinity_value/0 + calc_capacity/1 ]). -define(KILOBYTE, 1024). -define(BUCKET_KEYS, [ - {bytes_in, bucket_infinity}, - {message_in, bucket_infinity}, + {bytes, bucket_infinity}, + {messages, bucket_infinity}, {connection, bucket_limit}, {message_routing, bucket_infinity} ]). -type limiter_type() :: - bytes_in - | message_in + bytes + | messages | connection | message_routing %% internal limiter for unclassified resources @@ -90,14 +90,17 @@ namespace() -> limiter. -roots() -> [limiter]. +roots() -> + [{limiter, hoconsc:mk(hoconsc:ref(?MODULE, limiter), #{importance => ?IMPORTANCE_HIDDEN})}]. fields(limiter) -> [ {Type, ?HOCON(?R_REF(node_opts), #{ desc => ?DESC(Type), - default => #{} + default => #{}, + importance => ?IMPORTANCE_HIDDEN, + aliases => alias_of_type(Type) })} || Type <- types() ] ++ @@ -107,6 +110,7 @@ fields(limiter) -> ?R_REF(client_fields), #{ desc => ?DESC(client), + importance => ?IMPORTANCE_HIDDEN, default => maps:from_list([ {erlang:atom_to_binary(Type), #{}} || Type <- types() @@ -124,30 +128,50 @@ fields(node_opts) -> })} ]; fields(client_fields) -> - [ - {Type, - ?HOCON(?R_REF(client_opts), #{ - desc => ?DESC(Type), - default => #{} - })} - || Type <- types() - ]; + client_fields(types(), #{default => #{}}); fields(bucket_infinity) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})}, - {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => <<"infinity">>})}, - {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})} + {burst, + ?HOCON(capacity(), #{ + desc => ?DESC(capacity), + default => <<"0">>, + importance => ?IMPORTANCE_HIDDEN, + aliases => [capacity] + })}, + {initial, + ?HOCON(initial(), #{ + default => <<"0">>, + desc => ?DESC(initial), + importance => ?IMPORTANCE_HIDDEN + })} ]; fields(bucket_limit) -> [ {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"1000/s">>})}, - {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => <<"1000">>})}, - {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})} + {burst, + ?HOCON(capacity(), #{ + desc => ?DESC(burst), + default => <<"0">>, + importance => ?IMPORTANCE_HIDDEN, + aliases => [capacity] + })}, + {initial, + ?HOCON(initial(), #{ + default => <<"0">>, + desc => ?DESC(initial), + importance => ?IMPORTANCE_HIDDEN + })} ]; fields(client_opts) -> [ {rate, ?HOCON(rate(), #{default => <<"infinity">>, desc => ?DESC(rate)})}, - {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})}, + {initial, + ?HOCON(initial(), #{ + default => <<"0">>, + desc => ?DESC(initial), + importance => ?IMPORTANCE_HIDDEN + })}, %% low_watermark add for emqx_channel and emqx_session %% both modules consume first and then check %% so we need to use this value to prevent excessive consumption @@ -157,20 +181,24 @@ fields(client_opts) -> initial(), #{ desc => ?DESC(low_watermark), - default => <<"0">> + default => <<"0">>, + importance => ?IMPORTANCE_HIDDEN } )}, - {capacity, + {burst, ?HOCON(capacity(), #{ - desc => ?DESC(client_bucket_capacity), - default => <<"infinity">> + desc => ?DESC(burst), + default => <<"0">>, + importance => ?IMPORTANCE_HIDDEN, + aliases => [capacity] })}, {divisible, ?HOCON( boolean(), #{ desc => ?DESC(divisible), - default => false + default => false, + importance => ?IMPORTANCE_HIDDEN } )}, {max_retry_time, @@ -178,7 +206,8 @@ fields(client_opts) -> emqx_schema:duration(), #{ desc => ?DESC(max_retry_time), - default => <<"10s">> + default => <<"10s">>, + importance => ?IMPORTANCE_HIDDEN } )}, {failure_strategy, @@ -186,16 +215,18 @@ fields(client_opts) -> failure_strategy(), #{ desc => ?DESC(failure_strategy), - default => force + default => force, + importance => ?IMPORTANCE_HIDDEN } )} ]; fields(listener_fields) -> - bucket_fields(?BUCKET_KEYS, listener_client_fields); + composite_bucket_fields(?BUCKET_KEYS, listener_client_fields); fields(listener_client_fields) -> - client_fields(?BUCKET_KEYS); + {Types, _} = lists:unzip(?BUCKET_KEYS), + client_fields(Types, #{required => false}); fields(Type) -> - bucket_field(Type). + simple_bucket_field(Type). desc(limiter) -> "Settings for the rate limiter."; @@ -230,19 +261,14 @@ get_bucket_cfg_path(Type, BucketName) -> [limiter, Type, bucket, BucketName]. types() -> - [bytes_in, message_in, connection, message_routing, internal]. + [bytes, messages, connection, message_routing, internal]. -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -%% `infinity` to `infinity_value` rules: -%% 1. all infinity capacity will change to infinity_value -%% 2. if the rate of global and bucket both are `infinity`, -%% use `infinity_value` as bucket rate. see `emqx_limiter_server:get_counter_rate/2` -infinity_value() -> - %% 1 TB - 1099511627776. +calc_capacity(#{rate := infinity}) -> + infinity; +calc_capacity(#{burst := infinity}) -> + infinity; +calc_capacity(#{rate := Rate, burst := Burst}) -> + erlang:floor(1000 * Rate / default_period()) + Burst. %%-------------------------------------------------------------------- %% Internal functions @@ -335,7 +361,7 @@ to_quota(Str, Regex) -> {match, [Quota, ""]} -> {ok, erlang:list_to_integer(Quota)}; {match, ""} -> - {ok, infinity_value()}; + {ok, infinity}; _ -> {error, Str} end @@ -350,7 +376,8 @@ apply_unit("mb", Val) -> Val * ?KILOBYTE * ?KILOBYTE; apply_unit("gb", Val) -> Val * ?KILOBYTE * ?KILOBYTE * ?KILOBYTE; apply_unit(Unit, _) -> throw("invalid unit:" ++ Unit). -bucket_field(Type) when is_atom(Type) -> +%% A bucket with only one type +simple_bucket_field(Type) when is_atom(Type) -> fields(bucket_infinity) ++ [ {client, @@ -358,16 +385,22 @@ bucket_field(Type) when is_atom(Type) -> ?R_REF(?MODULE, client_opts), #{ desc => ?DESC(client), - required => false + required => false, + importance => importance_of_type(Type), + aliases => alias_of_type(Type) } )} ]. -bucket_fields(Types, ClientRef) -> + +%% A bucket with multi types +composite_bucket_fields(Types, ClientRef) -> [ {Type, ?HOCON(?R_REF(?MODULE, Opts), #{ desc => ?DESC(?MODULE, Type), - required => false + required => false, + importance => importance_of_type(Type), + aliases => alias_of_type(Type) })} || {Type, Opts} <- Types ] ++ @@ -382,12 +415,29 @@ bucket_fields(Types, ClientRef) -> )} ]. -client_fields(Types) -> +client_fields(Types, Meta) -> [ {Type, - ?HOCON(?R_REF(client_opts), #{ + ?HOCON(?R_REF(client_opts), Meta#{ desc => ?DESC(Type), - required => false + importance => importance_of_type(Type), + aliases => alias_of_type(Type) })} - || {Type, _} <- Types + || Type <- Types ]. + +importance_of_type(interval) -> + ?IMPORTANCE_HIDDEN; +importance_of_type(message_routing) -> + ?IMPORTANCE_HIDDEN; +importance_of_type(connection) -> + ?IMPORTANCE_HIDDEN; +importance_of_type(_) -> + ?DEFAULT_IMPORTANCE. + +alias_of_type(messages) -> + [message_in]; +alias_of_type(bytess) -> + [bytes_in]; +alias_of_type(_) -> + []. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl index f1daeaaeb..58db66f82 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl @@ -118,17 +118,24 @@ connect(_Id, _Type, undefined) -> {ok, emqx_htb_limiter:make_infinity_limiter()}; connect(Id, Type, Cfg) -> case find_limiter_cfg(Type, Cfg) of - {undefined, _} -> + {_ClientCfg, undefined, _NodeCfg} -> {ok, emqx_htb_limiter:make_infinity_limiter()}; + {#{rate := infinity}, #{rate := infinity}, #{rate := infinity}} -> + {ok, emqx_htb_limiter:make_infinity_limiter()}; + {ClientCfg, #{rate := infinity}, #{rate := infinity}} -> + {ok, + emqx_htb_limiter:make_token_bucket_limiter( + ClientCfg, emqx_limiter_bucket_ref:infinity_bucket() + )}; { - #{ - rate := BucketRate, - capacity := BucketSize - }, - #{rate := CliRate, capacity := CliSize} = ClientCfg + #{rate := CliRate} = ClientCfg, + #{rate := BucketRate} = BucketCfg, + _ } -> case emqx_limiter_manager:find_bucket(Id, Type) of {ok, Bucket} -> + BucketSize = emqx_limiter_schema:calc_capacity(BucketCfg), + CliSize = emqx_limiter_schema:calc_capacity(ClientCfg), {ok, if CliRate < BucketRate orelse CliSize < BucketSize -> @@ -493,12 +500,14 @@ make_root(#{rate := Rate, burst := Burst}) -> produced => 0.0 }. -do_add_bucket(Id, #{rate := Rate, capacity := Capacity} = Cfg, #{buckets := Buckets} = State) -> +do_add_bucket(_Id, #{rate := infinity}, #{root := #{rate := infinity}} = State) -> + State; +do_add_bucket(Id, #{rate := Rate} = Cfg, #{buckets := Buckets} = State) -> case maps:get(Id, Buckets, undefined) of undefined -> make_bucket(Id, Cfg, State); Bucket -> - Bucket2 = Bucket#{rate := Rate, capacity := Capacity}, + Bucket2 = Bucket#{rate := Rate, capacity := emqx_limiter_schema:calc_capacity(Cfg)}, State#{buckets := Buckets#{Id := Bucket2}} end. @@ -509,7 +518,7 @@ make_bucket(Id, Cfg, #{index := ?COUNTER_SIZE} = State) -> }); make_bucket( Id, - #{rate := Rate, capacity := Capacity} = Cfg, + #{rate := Rate} = Cfg, #{type := Type, counter := Counter, index := Index, buckets := Buckets} = State ) -> NewIndex = Index + 1, @@ -519,7 +528,7 @@ make_bucket( rate => Rate, obtained => Initial, correction => 0, - capacity => Capacity, + capacity => emqx_limiter_schema:calc_capacity(Cfg), counter => Counter, index => NewIndex }, @@ -541,19 +550,14 @@ do_del_bucket(Id, #{type := Type, buckets := Buckets} = State) -> get_initial_val( #{ initial := Initial, - rate := Rate, - capacity := Capacity + rate := Rate } ) -> - %% initial will nevner be infinity(see the emqx_limiter_schema) - InfVal = emqx_limiter_schema:infinity_value(), if Initial > 0 -> Initial; Rate =/= infinity -> - erlang:min(Rate, Capacity); - Capacity =/= infinity andalso Capacity =/= InfVal -> - Capacity; + Rate; true -> 0 end. @@ -568,11 +572,12 @@ call(Type, Msg) -> end. find_limiter_cfg(Type, #{rate := _} = Cfg) -> - {Cfg, find_client_cfg(Type, maps:get(client, Cfg, undefined))}; + {find_client_cfg(Type, maps:get(client, Cfg, undefined)), Cfg, find_node_cfg(Type)}; find_limiter_cfg(Type, Cfg) -> { + find_client_cfg(Type, emqx_utils_maps:deep_get([client, Type], Cfg, undefined)), maps:get(Type, Cfg, undefined), - find_client_cfg(Type, emqx_utils_maps:deep_get([client, Type], Cfg, undefined)) + find_node_cfg(Type) }. find_client_cfg(Type, BucketCfg) -> @@ -585,3 +590,6 @@ merge_client_cfg(NodeCfg, undefined) -> NodeCfg; merge_client_cfg(NodeCfg, BucketCfg) -> maps:merge(NodeCfg, BucketCfg). + +find_node_cfg(Type) -> + emqx:get_config([limiter, Type], #{rate => infinity, burst => 0}). diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 20962809f..faf62f98d 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -121,8 +121,8 @@ -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]). -define(ENABLED(X), (X =/= undefined)). --define(LIMITER_BYTES_IN, bytes_in). --define(LIMITER_MESSAGE_IN, message_in). +-define(LIMITER_BYTES_IN, bytes). +-define(LIMITER_MESSAGE_IN, messages). -dialyzer({no_match, [info/2]}). -dialyzer({nowarn_function, [websocket_init/1]}). diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index f3b97d517..7288dcf7c 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -72,7 +72,7 @@ t_consume(_) -> Cfg = fun(Cfg) -> Cfg#{ rate := 100, - capacity := 100, + burst := 0, initial := 100, max_retry_time := 1000, failure_strategy := force @@ -89,7 +89,7 @@ t_retry(_) -> Cfg = fun(Cfg) -> Cfg#{ rate := 50, - capacity := 200, + burst := 150, initial := 0, max_retry_time := 1000, failure_strategy := force @@ -109,7 +109,7 @@ t_restore(_) -> Cfg = fun(Cfg) -> Cfg#{ rate := 1, - capacity := 200, + burst := 199, initial := 50, max_retry_time := 100, failure_strategy := force @@ -129,7 +129,7 @@ t_max_retry_time(_) -> Cfg = fun(Cfg) -> Cfg#{ rate := 1, - capacity := 1, + burst := 0, max_retry_time := 500, failure_strategy := drop } @@ -139,8 +139,12 @@ t_max_retry_time(_) -> Begin = ?NOW, Result = emqx_htb_limiter:consume(101, Client), ?assertMatch({drop, _}, Result), - Time = ?NOW - Begin, - ?assert(Time >= 500 andalso Time < 550) + End = ?NOW, + Time = End - Begin, + ?assert( + Time >= 500 andalso Time < 550, + lists:flatten(io_lib:format("Begin:~p, End:~p, Time:~p~n", [Begin, End, Time])) + ) end, with_per_client(Cfg, Case). @@ -150,7 +154,7 @@ t_divisible(_) -> divisible := true, rate := ?RATE("1000/1s"), initial := 600, - capacity := 600 + burst := 0 } end, Case = fun(BucketCfg) -> @@ -176,7 +180,7 @@ t_low_watermark(_) -> low_watermark := 400, rate := ?RATE("1000/1s"), initial := 1000, - capacity := 1000 + burst := 0 } end, Case = fun(BucketCfg) -> @@ -201,8 +205,7 @@ t_infinity_client(_) -> Fun = fun(Cfg) -> Cfg end, Case = fun(Cfg) -> Client = connect(Cfg), - InfVal = emqx_limiter_schema:infinity_value(), - ?assertMatch(#{bucket := #{rate := InfVal}}, Client), + ?assertMatch(infinity, Client), Result = emqx_htb_limiter:check(100000, Client), ?assertEqual({ok, Client}, Result) end, @@ -212,12 +215,12 @@ t_try_restore_agg(_) -> Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := 1, - capacity := 200, + burst := 199, initial := 50 }, Cli2 = Cli#{ rate := infinity, - capacity := infinity, + burst := infinity, divisible := true, max_retry_time := 100, failure_strategy := force @@ -239,11 +242,11 @@ t_short_board(_) -> Bucket2 = Bucket#{ rate := ?RATE("100/1s"), initial := 0, - capacity := 100 + burst := 0 }, Cli2 = Cli#{ rate := ?RATE("600/1s"), - capacity := 600, + burst := 0, initial := 600 }, Bucket2#{client := Cli2} @@ -261,46 +264,45 @@ t_rate(_) -> Bucket2 = Bucket#{ rate := ?RATE("100/100ms"), initial := 0, - capacity := infinity + burst := infinity }, Cli2 = Cli#{ rate := infinity, - capacity := infinity, + burst := infinity, initial := 0 }, Bucket2#{client := Cli2} end, Case = fun(Cfg) -> + Time = 1000, Client = connect(Cfg), - Ts1 = erlang:system_time(millisecond), C1 = emqx_htb_limiter:available(Client), - timer:sleep(1000), - Ts2 = erlang:system_time(millisecond), + timer:sleep(1100), C2 = emqx_htb_limiter:available(Client), - ShouldInc = floor((Ts2 - Ts1) / 100) * 100, + ShouldInc = floor(Time / 100) * 100, Inc = C2 - C1, ?assert(in_range(Inc, ShouldInc - 100, ShouldInc + 100), "test bucket rate") end, with_bucket(Fun, Case). t_capacity(_) -> - Capacity = 600, + Capacity = 1200, Fun = fun(#{client := Cli} = Bucket) -> Bucket2 = Bucket#{ rate := ?RATE("100/100ms"), initial := 0, - capacity := 600 + burst := 200 }, Cli2 = Cli#{ rate := infinity, - capacity := infinity, + burst := infinity, initial := 0 }, Bucket2#{client := Cli2} end, Case = fun(Cfg) -> Client = connect(Cfg), - timer:sleep(1000), + timer:sleep(1500), C1 = emqx_htb_limiter:available(Client), ?assertEqual(Capacity, C1, "test bucket capacity") end, @@ -318,11 +320,11 @@ t_collaborative_alloc(_) -> Bucket2 = Bucket#{ rate := ?RATE("400/1s"), initial := 0, - capacity := 600 + burst := 200 }, Cli2 = Cli#{ rate := ?RATE("50"), - capacity := 100, + burst := 50, initial := 100 }, Bucket2#{client := Cli2} @@ -363,11 +365,11 @@ t_burst(_) -> Bucket2 = Bucket#{ rate := ?RATE("200/1s"), initial := 0, - capacity := 200 + burst := 0 }, Cli2 = Cli#{ rate := ?RATE("50/1s"), - capacity := 200, + burst := 150, divisible := true }, Bucket2#{client := Cli2} @@ -401,11 +403,11 @@ t_limit_global_with_unlimit_other(_) -> Bucket2 = Bucket#{ rate := infinity, initial := 0, - capacity := infinity + burst := infinity }, Cli2 = Cli#{ rate := infinity, - capacity := infinity, + burst := infinity, initial := 0 }, Bucket2#{client := Cli2} @@ -414,7 +416,7 @@ t_limit_global_with_unlimit_other(_) -> Case = fun() -> C1 = counters:new(1, []), start_client({b1, Bucket}, ?NOW + 2000, C1, 20), - timer:sleep(2100), + timer:sleep(2200), check_average_rate(C1, 2, 600) end, @@ -432,7 +434,7 @@ t_check_container(_) -> Cfg#{ rate := ?RATE("1000/1s"), initial := 1000, - capacity := 1000 + burst := 0 } end, Case = fun(#{client := Client} = BucketCfg) -> @@ -565,7 +567,7 @@ t_schema_unit(_) -> ?assertMatch({error, _}, M:to_rate("100MB/1")), ?assertMatch({error, _}, M:to_rate("100/10x")), - ?assertEqual({ok, emqx_limiter_schema:infinity_value()}, M:to_capacity("infinity")), + ?assertEqual({ok, infinity}, M:to_capacity("infinity")), ?assertEqual({ok, 100}, M:to_capacity("100")), ?assertEqual({ok, 100 * 1024}, M:to_capacity("100KB")), ?assertEqual({ok, 100 * 1024 * 1024}, M:to_capacity("100MB")), @@ -748,17 +750,16 @@ connect(Name, Cfg) -> Limiter. make_limiter_cfg() -> - Infinity = emqx_limiter_schema:infinity_value(), Client = #{ - rate => Infinity, + rate => infinity, initial => 0, - capacity => Infinity, + burst => infinity, low_watermark => 0, divisible => false, max_retry_time => timer:seconds(5), failure_strategy => force }, - #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + #{client => Client, rate => infinity, initial => 0, burst => infinity}. add_bucket(Cfg) -> add_bucket(?MODULE, Cfg). diff --git a/rel/i18n/emqx_limiter_schema.hocon b/rel/i18n/emqx_limiter_schema.hocon index 3657df694..2874999a5 100644 --- a/rel/i18n/emqx_limiter_schema.hocon +++ b/rel/i18n/emqx_limiter_schema.hocon @@ -33,28 +33,6 @@ emqx_limiter_schema { } } - client_bucket_capacity { - desc { - en: """The capacity of per user.""" - zh: """每个使用者的令牌容量上限""" - } - label: { - en: """Capacity""" - zh: """容量""" - } - } - - capacity { - desc { - en: """The capacity of this token bucket.""" - zh: """该令牌桶的容量""" - } - label: { - en: """Capacity""" - zh: """容量""" - } - } - low_watermark { desc { en: """If the remaining tokens are lower than this value, @@ -152,30 +130,30 @@ Once the limit is reached, new connections will be refused""" } } - message_in { + messages { desc { - en: """The message in limiter. + en: """The messages limiter. This is used to limit the inbound message numbers for this EMQX node Once the limit is reached, the restricted client will be slow down even be hung for a while.""" zh: """流入速率控制器。 这个用来控制当前节点上的消息流入速率，当达到最大速率后，会话将会被限速甚至被强制挂起一小段时间""" } label: { - en: """Message In""" + en: """Messages""" zh: """消息流入速率""" } } - bytes_in { + bytes { desc { - en: """The bytes_in limiter. + en: """The bytes limiter. This is used to limit the inbound bytes rate for this EMQX node. Once the limit is reached, the restricted client will be slow down even be hung for a while.""" zh: """流入字节率控制器。 这个是用来控制当前节点上的数据流入的字节率，每条消息将会消耗和其二进制大小等量的令牌，当达到最大速率后，会话将会被限速甚至被强制挂起一小段时间""" } label: { - en: """Bytes In""" + en: """Bytes""" zh: """流入字节率""" } } From 2a54d93c7e05851b994396d23214a33a5a600002 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 12 Apr 2023 17:42:23 +0800 Subject: [PATCH 168/279] chore: update changes --- changes/ce/perf-10376.en.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/ce/perf-10376.en.md diff --git a/changes/ce/perf-10376.en.md b/changes/ce/perf-10376.en.md new file mode 100644 index 000000000..d585ad5b2 --- /dev/null +++ b/changes/ce/perf-10376.en.md @@ -0,0 +1,6 @@ +Simplify the configuration of the limiter feature and optimize some codes +- Rename `message_in` to `messages` +- Rename `bytes_in` to `bytes` +- Use `burst` instead of `capacity` +- Hide non-importance fields +- Optimize limiter instances in different rate settings From 02f8d073f8dc125675bde9cc67bd5b8c8d013a17 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 13 Apr 2023 11:45:51 +0800 Subject: [PATCH 169/279] test(limiter): fix test errors and make spellcheck happy --- .../emqx_limiter/src/emqx_limiter_schema.erl | 2 +- apps/emqx/test/emqx_channel_SUITE.erl | 9 +++--- apps/emqx/test/emqx_connection_SUITE.erl | 22 +++++++------- apps/emqx/test/emqx_ws_connection_SUITE.erl | 29 +++++++++---------- .../test/emqx_retainer_SUITE.erl | 9 +++--- rel/i18n/emqx_limiter_schema.hocon | 4 +-- 6 files changed, 34 insertions(+), 41 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index f59ddc35b..730559f80 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -437,7 +437,7 @@ importance_of_type(_) -> alias_of_type(messages) -> [message_in]; -alias_of_type(bytess) -> +alias_of_type(bytes) -> [bytes_in]; alias_of_type(_) -> []. diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 29f8b1503..eccb5c865 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -162,8 +162,7 @@ limiter_conf() -> Make = fun() -> #{ burst => 0, - rate => infinity, - capacity => infinity + rate => infinity } end, @@ -172,7 +171,7 @@ limiter_conf() -> Acc#{Name => Make()} end, #{}, - [bytes_in, message_in, message_routing, connection, internal] + [bytes, messages, message_routing, connection, internal] ). stats_conf() -> @@ -1258,7 +1257,7 @@ limiter_cfg() -> Client = #{ rate => 5, initial => 0, - capacity => 5, + burst => 0, low_watermark => 1, divisible => false, max_retry_time => timer:seconds(5), @@ -1270,7 +1269,7 @@ limiter_cfg() -> }. bucket_cfg() -> - #{rate => 10, initial => 0, capacity => 10}. + #{rate => 10, initial => 0, burst => 0}. add_bucket() -> emqx_limiter_server:add_bucket(?MODULE, message_routing, bucket_cfg()). diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 21ed45119..f24c1c895 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -427,7 +427,7 @@ t_ensure_rate_limit(_) -> fun(_, Client) -> {pause, 3000, undefined, Client} end ), {ok, State2} = emqx_connection:check_limiter( - [{1000, bytes_in}], + [{1000, bytes}], [], WhenOk, [], @@ -703,31 +703,29 @@ handle_call(Pid, Call, St) -> emqx_connection:handle_call(Pid, Call, St). -define(LIMITER_ID, 'tcp:default'). init_limiter() -> - emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], limiter_cfg()). + emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes, messages], limiter_cfg()). limiter_cfg() -> - Infinity = emqx_limiter_schema:infinity_value(), Cfg = bucket_cfg(), Client = #{ - rate => Infinity, + rate => infinity, initial => 0, - capacity => Infinity, + burst => 0, low_watermark => 1, divisible => false, max_retry_time => timer:seconds(5), failure_strategy => force }, - #{bytes_in => Cfg, message_in => Cfg, client => #{bytes_in => Client, message_in => Client}}. + #{bytes => Cfg, messages => Cfg, client => #{bytes => Client, messages => Client}}. bucket_cfg() -> - Infinity = emqx_limiter_schema:infinity_value(), - #{rate => Infinity, initial => 0, capacity => Infinity}. + #{rate => infinity, initial => 0, burst => 0}. add_bucket() -> Cfg = bucket_cfg(), - emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), - emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). + emqx_limiter_server:add_bucket(?LIMITER_ID, bytes, Cfg), + emqx_limiter_server:add_bucket(?LIMITER_ID, messages, Cfg). del_bucket() -> - emqx_limiter_server:del_bucket(?LIMITER_ID, bytes_in), - emqx_limiter_server:del_bucket(?LIMITER_ID, message_in). + emqx_limiter_server:del_bucket(?LIMITER_ID, bytes), + emqx_limiter_server:del_bucket(?LIMITER_ID, messages). diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index de8b1c9af..1ae23361e 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -509,16 +509,16 @@ t_handle_timeout_emit_stats(_) -> t_ensure_rate_limit(_) -> {ok, Rate} = emqx_limiter_schema:to_rate("50MB"), Limiter = init_limiter(#{ - bytes_in => bucket_cfg(), - message_in => bucket_cfg(), - client => #{bytes_in => client_cfg(Rate)} + bytes => bucket_cfg(), + messages => bucket_cfg(), + client => #{bytes => client_cfg(Rate)} }), St = st(#{limiter => Limiter}), %% must bigger than value in emqx_ratelimit_SUITE {ok, Need} = emqx_limiter_schema:to_capacity("1GB"), St1 = ?ws_conn:check_limiter( - [{Need, bytes_in}], + [{Need, bytes}], [], fun(_, _, S) -> S end, [], @@ -699,23 +699,21 @@ init_limiter() -> init_limiter(limiter_cfg()). init_limiter(LimiterCfg) -> - emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes_in, message_in], LimiterCfg). + emqx_limiter_container:get_limiter_by_types(?LIMITER_ID, [bytes, messages], LimiterCfg). limiter_cfg() -> Cfg = bucket_cfg(), Client = client_cfg(), - #{bytes_in => Cfg, message_in => Cfg, client => #{bytes_in => Client, message_in => Client}}. + #{bytes => Cfg, messages => Cfg, client => #{bytes => Client, messages => Client}}. client_cfg() -> - Infinity = emqx_limiter_schema:infinity_value(), - client_cfg(Infinity). + client_cfg(infinity). client_cfg(Rate) -> - Infinity = emqx_limiter_schema:infinity_value(), #{ rate => Rate, initial => 0, - capacity => Infinity, + burst => 0, low_watermark => 1, divisible => false, max_retry_time => timer:seconds(5), @@ -723,14 +721,13 @@ client_cfg(Rate) -> }. bucket_cfg() -> - Infinity = emqx_limiter_schema:infinity_value(), - #{rate => Infinity, initial => 0, capacity => Infinity}. + #{rate => infinity, initial => 0, burst => 0}. add_bucket() -> Cfg = bucket_cfg(), - emqx_limiter_server:add_bucket(?LIMITER_ID, bytes_in, Cfg), - emqx_limiter_server:add_bucket(?LIMITER_ID, message_in, Cfg). + emqx_limiter_server:add_bucket(?LIMITER_ID, bytes, Cfg), + emqx_limiter_server:add_bucket(?LIMITER_ID, messages, Cfg). del_bucket() -> - emqx_limiter_server:del_bucket(?LIMITER_ID, bytes_in), - emqx_limiter_server:del_bucket(?LIMITER_ID, message_in). + emqx_limiter_server:del_bucket(?LIMITER_ID, bytes), + emqx_limiter_server:del_bucket(?LIMITER_ID, messages). diff --git a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl index 09d1f77da..c90ec6b2b 100644 --- a/apps/emqx_retainer/test/emqx_retainer_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_SUITE.erl @@ -758,23 +758,22 @@ with_conf(ConfMod, Case) -> end. make_limiter_cfg(Rate) -> - Infinity = emqx_limiter_schema:infinity_value(), Client = #{ rate => Rate, initial => 0, - capacity => Infinity, + burst => 0, low_watermark => 1, divisible => false, max_retry_time => timer:seconds(5), failure_strategy => force }, - #{client => Client, rate => Infinity, initial => 0, capacity => Infinity}. + #{client => Client, rate => Rate, initial => 0, burst => 0}. make_limiter_json(Rate) -> Client = #{ <<"rate">> => Rate, <<"initial">> => 0, - <<"capacity">> => <<"infinity">>, + <<"burst">> => <<"0">>, <<"low_watermark">> => 0, <<"divisible">> => <<"false">>, <<"max_retry_time">> => <<"5s">>, @@ -784,5 +783,5 @@ make_limiter_json(Rate) -> <<"client">> => Client, <<"rate">> => <<"infinity">>, <<"initial">> => 0, - <<"capacity">> => <<"infinity">> + <<"burst">> => <<"0">> }. diff --git a/rel/i18n/emqx_limiter_schema.hocon b/rel/i18n/emqx_limiter_schema.hocon index 2874999a5..37eb4ee1e 100644 --- a/rel/i18n/emqx_limiter_schema.hocon +++ b/rel/i18n/emqx_limiter_schema.hocon @@ -132,7 +132,7 @@ Once the limit is reached, new connections will be refused""" messages { desc { - en: """The messages limiter. + en: """The `messages` limiter. This is used to limit the inbound message numbers for this EMQX node Once the limit is reached, the restricted client will be slow down even be hung for a while.""" zh: """流入速率控制器。 @@ -146,7 +146,7 @@ Once the limit is reached, the restricted client will be slow down even be hung bytes { desc { - en: """The bytes limiter. + en: """The `bytes` limiter. This is used to limit the inbound bytes rate for this EMQX node. Once the limit is reached, the restricted client will be slow down even be hung for a while.""" zh: """流入字节率控制器。 From 9fc8a498f8e1e4fa1a97ad71901c37bf21caeecb Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 17 Apr 2023 09:01:41 +0200 Subject: [PATCH 170/279] chore: bump apps versions --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_authn/src/emqx_authn.app.src | 2 +- apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_connector/src/emqx_connector.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_management/src/emqx_management.app.src | 2 +- apps/emqx_modules/src/emqx_modules.app.src | 2 +- apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src | 2 +- apps/emqx_prometheus/src/emqx_prometheus.app.src | 2 +- apps/emqx_resource/src/emqx_resource.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +- .../emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src | 2 +- lib-ee/emqx_license/src/emqx_license.app.src | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index c4894c933..b2dfca9e1 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.22"}, + {vsn, "5.0.23"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index caa59e455..063771e24 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.16"}, + {vsn, "0.1.17"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 2f8b26894..a17ce5dea 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.16"}, + {vsn, "0.1.17"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index cd2668ef8..04f2b58a7 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.15"}, + {vsn, "0.1.16"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index c904688ca..234690374 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.16"}, + {vsn, "0.1.17"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index e3de5aeff..38ca230b2 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.19"}, + {vsn, "0.1.20"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index 15530d679..b810f9c5f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.17"}, + {vsn, "5.0.18"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 850d38cdd..0419666b4 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.14"}, + {vsn, "0.1.15"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index dac82403a..f423213af 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.18"}, + {vsn, "5.0.19"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index fdc13f354..1e38e25d1 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.12"}, + {vsn, "5.0.13"}, {modules, []}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_modules_app, []}}, diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index dcb330df4..24b5a3240 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugin_libs, [ {description, "EMQX Plugin utility libs"}, - {vsn, "4.3.8"}, + {vsn, "4.3.9"}, {modules, []}, {applications, [kernel, stdlib]}, {env, []} diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 1e7e59f7a..ae879da8f 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index dfb0047c7..00c315714 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.12"}, + {vsn, "0.1.13"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index fa33a1ea5..133138252 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.13"}, + {vsn, "5.0.14"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index f1793d2e0..d6c59c716 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.9"}, + {vsn, "0.1.10"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 2e2ff2a62..7ebc320e5 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.9"}, + {vsn, "0.1.10"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src index c40fb808a..4d0f4e9c7 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_schema_registry, [ {description, "EMQX Schema Registry"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, [emqx_ee_schema_registry_sup]}, {mod, {emqx_ee_schema_registry_app, []}}, {applications, [ diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src index 0a97ee83b..fcdcbc05b 100644 --- a/lib-ee/emqx_license/src/emqx_license.app.src +++ b/lib-ee/emqx_license/src/emqx_license.app.src @@ -1,6 +1,6 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {registered, [emqx_license_sup]}, {applications, [kernel, stdlib, emqx_ctl]}, From e9e0ae7f0a55eb2ea31a10be2ff12a777381adec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Mon, 17 Apr 2023 17:23:39 +0800 Subject: [PATCH 171/279] chore: When matching authz's and/or rules, check the simple ones first to improve efficiency --- apps/emqx/etc/ssl_dist.conf | 2 +- apps/emqx_authz/src/emqx_authz_rule.erl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/etc/ssl_dist.conf b/apps/emqx/etc/ssl_dist.conf index af1c7506d..b4c16e2cc 100644 --- a/apps/emqx/etc/ssl_dist.conf +++ b/apps/emqx/etc/ssl_dist.conf @@ -1,4 +1,4 @@ -%% This additional config file is used when the config 'cluster.proto_dis' in emqx.conf is set to 'inet_tls'. +%% This additional config file is used when the config 'cluster.proto_dist' in emqx.conf is set to 'inet_tls'. %% Which means the EMQX nodes will connect to each other over TLS. %% For more information about inter-broker security, see: https://docs.emqx.com/en/enterprise/v5.0/deploy/cluster/security.html diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index 306ca9433..bdd0904f7 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -185,7 +185,7 @@ match_who(#{peerhost := IpAddress}, {ipaddrs, CIDRs}) -> match_who(ClientInfo, {'and', Principals}) when is_list(Principals) -> lists:foldl( fun(Principal, Permission) -> - match_who(ClientInfo, Principal) andalso Permission + Permission andalso match_who(ClientInfo, Principal) end, true, Principals @@ -193,7 +193,7 @@ match_who(ClientInfo, {'and', Principals}) when is_list(Principals) -> match_who(ClientInfo, {'or', Principals}) when is_list(Principals) -> lists:foldl( fun(Principal, Permission) -> - match_who(ClientInfo, Principal) orelse Permission + Permission orelse match_who(ClientInfo, Principal) end, false, Principals From 549283ae75941a5ec9d361fa32d0a10f9fe8dd99 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 17 Apr 2023 17:35:50 +0800 Subject: [PATCH 172/279] chore: update changes/ce/fix-10410.en.md Co-authored-by: Zaiming (Stone) Shi --- changes/ce/fix-10410.en.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/ce/fix-10410.en.md b/changes/ce/fix-10410.en.md index 48b55ea31..aa219c96e 100644 --- a/changes/ce/fix-10410.en.md +++ b/changes/ce/fix-10410.en.md @@ -1,2 +1,2 @@ -Fix EMQX starting failed once any gateways configured in emqx.conf. -This issue was first introduced in v5.0.22 via [#10278](https://github.com/emqx/emqx/pull/10278). +Fix config check failed when gateways are configured in emqx.conf. +This issue was first introduced in v5.0.22 via [#10278](https://github.com/emqx/emqx/pull/10278), the boot-time config check was missing. From 37f42a486c990f6151efa51dea2e7c2aa2a8a283 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 17 Apr 2023 12:03:13 +0200 Subject: [PATCH 173/279] test: improve proper test so it generates more cases --- apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 2bce5a1b4..4dc9280da 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -959,7 +959,7 @@ prop_format_date_fun() -> Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>], ?FORALL( S, - erlang:system_time(second), + range(0, 4000000000), S == apply_func( date_to_unix_ts, @@ -975,7 +975,7 @@ prop_format_date_fun() -> Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>], ?FORALL( S, - erlang:system_time(millisecond), + range(0, 4000000000), S == apply_func( date_to_unix_ts, @@ -991,7 +991,7 @@ prop_format_date_fun() -> Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], ?FORALL( S, - erlang:system_time(second), + range(0, 4000000000), S == apply_func( date_to_unix_ts, @@ -1009,7 +1009,7 @@ prop_format_date_fun() -> ArgsOffset = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], ?FORALL( S, - erlang:system_time(second), + range(0, 4000000000), S == apply_func( date_to_unix_ts, From 334058eeec5532aedba75a1f540f19672f32f736 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 5 Apr 2023 12:23:19 +0200 Subject: [PATCH 174/279] build: add a script to split en and zh descriptions --- scripts/split-i18n-files.escript | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 scripts/split-i18n-files.escript diff --git a/scripts/split-i18n-files.escript b/scripts/split-i18n-files.escript new file mode 100755 index 000000000..5910db667 --- /dev/null +++ b/scripts/split-i18n-files.escript @@ -0,0 +1,78 @@ +#!/usr/bin/env escript + +%% This script is for one-time use. +%% will be deleted after the migration is done. + +-mode(compile). + +main([]) -> + %% we need to parse hocon + %% so we'll just add all compiled libs to path + code:add_pathsz(find_ebin_paths("_build/default/lib/*")), + Files = filelib:wildcard("rel/i18n/*.hocon"), + ok = lists:foreach(fun split_file/1, Files), + ok. + +find_ebin_paths(DirPattern) -> + LibDirs = filelib:wildcard(DirPattern), + lists:filtermap(fun add_ebin/1, LibDirs). + +add_ebin(Dir) -> + EbinDir = filename:join(Dir, "ebin"), + case filelib:is_dir(EbinDir) of + true -> {true, EbinDir}; + false -> false + end. + +split_file(Path) -> + {ok, DescMap} = hocon:load(Path), + [{Module, Descs}] = maps:to_list(DescMap), + ok = split(Path, Module, <<"en">>, Descs), + ok = split(Path, Module, <<"zh">>, Descs), + ok. + +split(Path, Module, Lang, Fields) when is_map(Fields) -> + split(Path, Module, Lang, maps:to_list(Fields)); +split(Path, Module, Lang, Fields) when is_list(Fields) -> + Split = lists:map(fun({Name, Desc})-> do_split(Path, Name, Lang, Desc) end, Fields), + IoData = [Module, " {\n\n", Split, "}\n"], + %% assert it's a valid HOCON object + {ok, _} = hocon:binary(IoData), + %io:format(user, "~s", [IoData]). + WritePath = case Lang of + <<"en">> -> + Path; + <<"zh">> -> + rename(Path, "zh") + end, + ok = filelib:ensure_dir(WritePath), + ok = file:write_file(WritePath, IoData), + ok. + +rename(FilePath, Lang) -> + Dir = filename:dirname(FilePath), + BaseName = filename:basename(FilePath), + filename:join([Dir, Lang, BaseName]). + +do_split(Path, Name, Lang, #{<<"desc">> := Desc} = D) -> + try + Label = maps:get(<<"label">>, D, #{}), + DescL = maps:get(Lang, Desc), + LabelL = maps:get(Lang, Label, undefined), + [fmt([Name, ".desc:\n"], DescL), + fmt([Name, ".label:\n"], LabelL) + ] + catch + C : E : S-> + erlang:raise(C, {Path, Name, E}, S) + end. + + +tq() -> + "\"\"\"". + +fmt(_Key, undefined) -> + []; +fmt(Key, Content) -> + [Key, tq(), Content, tq(), "\n\n"]. + From b9a43ead393403b5b66aa75864bce5d17e08a3d1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 17 Apr 2023 13:35:33 +0200 Subject: [PATCH 175/279] refactor: Split i18n files --- rel/i18n/emqx_authn_api.hocon | 247 +- rel/i18n/emqx_authn_http.hocon | 108 +- rel/i18n/emqx_authn_jwt.hocon | 297 +- rel/i18n/emqx_authn_mnesia.hocon | 23 +- rel/i18n/emqx_authn_mongodb.hocon | 106 +- rel/i18n/emqx_authn_mysql.hocon | 40 +- rel/i18n/emqx_authn_pgsql.hocon | 25 +- rel/i18n/emqx_authn_redis.hocon | 41 +- rel/i18n/emqx_authn_schema.hocon | 332 +- rel/i18n/emqx_authn_user_import_api.hocon | 16 +- rel/i18n/emqx_authz_api_cache.hocon | 10 +- rel/i18n/emqx_authz_api_mnesia.hocon | 218 +- rel/i18n/emqx_authz_api_schema.hocon | 227 +- rel/i18n/emqx_authz_api_settings.hocon | 18 +- rel/i18n/emqx_authz_api_sources.hocon | 136 +- rel/i18n/emqx_authz_schema.hocon | 778 ++-- rel/i18n/emqx_auto_subscribe_api.hocon | 25 +- rel/i18n/emqx_auto_subscribe_schema.hocon | 107 +- rel/i18n/emqx_bridge_api.hocon | 242 +- rel/i18n/emqx_bridge_kafka.hocon | 1020 ++--- rel/i18n/emqx_bridge_mqtt_schema.hocon | 47 +- rel/i18n/emqx_bridge_schema.hocon | 420 +- rel/i18n/emqx_bridge_webhook_schema.hocon | 224 +- rel/i18n/emqx_coap_api.hocon | 65 +- rel/i18n/emqx_coap_schema.hocon | 82 +- rel/i18n/emqx_conf_schema.hocon | 2027 +++------ rel/i18n/emqx_connector_api.hocon | 110 +- rel/i18n/emqx_connector_http.hocon | 183 +- rel/i18n/emqx_connector_ldap.hocon | 44 +- rel/i18n/emqx_connector_mongo.hocon | 375 +- rel/i18n/emqx_connector_mqtt.hocon | 46 +- rel/i18n/emqx_connector_mqtt_schema.hocon | 429 +- rel/i18n/emqx_connector_mysql.hocon | 17 +- rel/i18n/emqx_connector_pgsql.hocon | 17 +- rel/i18n/emqx_connector_redis.hocon | 110 +- rel/i18n/emqx_connector_schema_lib.hocon | 105 +- rel/i18n/emqx_dashboard_api.hocon | 168 +- rel/i18n/emqx_dashboard_schema.hocon | 346 +- rel/i18n/emqx_delayed_api.hocon | 184 +- rel/i18n/emqx_ee_bridge_cassa.hocon | 97 +- rel/i18n/emqx_ee_bridge_clickhouse.hocon | 110 +- rel/i18n/emqx_ee_bridge_dynamo.hocon | 93 +- rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon | 191 +- rel/i18n/emqx_ee_bridge_hstreamdb.hocon | 127 +- rel/i18n/emqx_ee_bridge_influxdb.hocon | 105 +- rel/i18n/emqx_ee_bridge_mongodb.hocon | 137 +- rel/i18n/emqx_ee_bridge_mysql.hocon | 93 +- rel/i18n/emqx_ee_bridge_pgsql.hocon | 93 +- rel/i18n/emqx_ee_bridge_redis.hocon | 103 +- rel/i18n/emqx_ee_bridge_rocketmq.hocon | 93 +- rel/i18n/emqx_ee_bridge_sqlserver.hocon | 112 +- rel/i18n/emqx_ee_bridge_tdengine.hocon | 93 +- rel/i18n/emqx_ee_connector_cassa.hocon | 33 +- rel/i18n/emqx_ee_connector_clickhouse.hocon | 30 +- rel/i18n/emqx_ee_connector_dynamo.hocon | 15 +- rel/i18n/emqx_ee_connector_hstreamdb.hocon | 111 +- rel/i18n/emqx_ee_connector_influxdb.hocon | 177 +- rel/i18n/emqx_ee_connector_rocketmq.hocon | 79 +- rel/i18n/emqx_ee_connector_sqlserver.hocon | 17 +- rel/i18n/emqx_ee_connector_tdengine.hocon | 17 +- .../emqx_ee_schema_registry_http_api.hocon | 94 +- rel/i18n/emqx_ee_schema_registry_schema.hocon | 107 +- rel/i18n/emqx_exhook_api.hocon | 205 +- rel/i18n/emqx_exhook_schema.hocon | 118 +- rel/i18n/emqx_exproto_schema.hocon | 60 +- rel/i18n/emqx_gateway_api.hocon | 197 +- rel/i18n/emqx_gateway_api_authn.hocon | 114 +- rel/i18n/emqx_gateway_api_clients.hocon | 679 +-- rel/i18n/emqx_gateway_api_listeners.hocon | 173 +- rel/i18n/emqx_gateway_schema.hocon | 326 +- rel/i18n/emqx_license_http_api.hocon | 32 +- rel/i18n/emqx_license_schema.hocon | 78 +- rel/i18n/emqx_limiter_schema.hocon | 238 +- rel/i18n/emqx_lwm2m_api.hocon | 65 +- rel/i18n/emqx_lwm2m_schema.hocon | 145 +- rel/i18n/emqx_mgmt_api_alarms.hocon | 95 +- rel/i18n/emqx_mgmt_api_banned.hocon | 136 +- rel/i18n/emqx_mgmt_api_key_schema.hocon | 36 +- rel/i18n/emqx_mgmt_api_publish.hocon | 185 +- rel/i18n/emqx_mgmt_api_status.hocon | 61 +- rel/i18n/emqx_modules_schema.hocon | 93 +- rel/i18n/emqx_mqttsn_schema.hocon | 73 +- rel/i18n/emqx_plugins_schema.hocon | 126 +- rel/i18n/emqx_prometheus_schema.hocon | 118 +- rel/i18n/emqx_psk_schema.hocon | 61 +- rel/i18n/emqx_resource_schema.hocon | 282 +- rel/i18n/emqx_retainer_api.hocon | 160 +- rel/i18n/emqx_retainer_schema.hocon | 138 +- rel/i18n/emqx_rewrite_api.hocon | 25 +- rel/i18n/emqx_rule_api_schema.hocon | 945 ++-- rel/i18n/emqx_rule_engine_api.hocon | 237 +- rel/i18n/emqx_rule_engine_schema.hocon | 514 +-- rel/i18n/emqx_schema.hocon | 4048 ++++++----------- rel/i18n/emqx_slow_subs_api.hocon | 72 +- rel/i18n/emqx_slow_subs_schema.hocon | 40 +- rel/i18n/emqx_statsd_api.hocon | 17 +- rel/i18n/emqx_statsd_schema.hocon | 73 +- rel/i18n/emqx_stomp_schema.hocon | 38 +- rel/i18n/emqx_telemetry_api.hocon | 137 +- rel/i18n/emqx_topic_metrics_api.hocon | 273 +- rel/i18n/zh/emqx_authn_api.hocon | 96 + rel/i18n/zh/emqx_authn_http.hocon | 45 + rel/i18n/zh/emqx_authn_jwt.hocon | 118 + rel/i18n/zh/emqx_authn_mnesia.hocon | 12 + rel/i18n/zh/emqx_authn_mongodb.hocon | 45 + rel/i18n/zh/emqx_authn_mysql.hocon | 18 + rel/i18n/zh/emqx_authn_pgsql.hocon | 12 + rel/i18n/zh/emqx_authn_redis.hocon | 18 + rel/i18n/zh/emqx_authn_schema.hocon | 135 + rel/i18n/zh/emqx_authn_user_import_api.hocon | 9 + rel/i18n/zh/emqx_authz_api_cache.hocon | 6 + rel/i18n/zh/emqx_authz_api_mnesia.hocon | 87 + rel/i18n/zh/emqx_authz_api_schema.hocon | 90 + rel/i18n/zh/emqx_authz_api_settings.hocon | 9 + rel/i18n/zh/emqx_authz_api_sources.hocon | 48 + rel/i18n/zh/emqx_authz_schema.hocon | 285 ++ rel/i18n/zh/emqx_auto_subscribe_api.hocon | 12 + rel/i18n/zh/emqx_auto_subscribe_schema.hocon | 48 + rel/i18n/zh/emqx_bridge_api.hocon | 100 + rel/i18n/zh/emqx_bridge_kafka.hocon | 354 ++ rel/i18n/zh/emqx_bridge_mqtt_schema.hocon | 21 + rel/i18n/zh/emqx_bridge_schema.hocon | 158 + rel/i18n/zh/emqx_bridge_webhook_schema.hocon | 87 + rel/i18n/zh/emqx_coap_api.hocon | 27 + rel/i18n/zh/emqx_coap_schema.hocon | 37 + rel/i18n/zh/emqx_conf_schema.hocon | 774 ++++ rel/i18n/zh/emqx_connector_api.hocon | 46 + rel/i18n/zh/emqx_connector_http.hocon | 77 + rel/i18n/zh/emqx_connector_ldap.hocon | 21 + rel/i18n/zh/emqx_connector_mongo.hocon | 152 + rel/i18n/zh/emqx_connector_mqtt.hocon | 21 + rel/i18n/zh/emqx_connector_mqtt_schema.hocon | 170 + rel/i18n/zh/emqx_connector_mysql.hocon | 11 + rel/i18n/zh/emqx_connector_pgsql.hocon | 11 + rel/i18n/zh/emqx_connector_redis.hocon | 50 + rel/i18n/zh/emqx_connector_schema_lib.hocon | 45 + rel/i18n/zh/emqx_dashboard_api.hocon | 66 + rel/i18n/zh/emqx_dashboard_schema.hocon | 130 + rel/i18n/zh/emqx_delayed_api.hocon | 72 + rel/i18n/zh/emqx_ee_bridge_cassa.hocon | 40 + rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon | 46 + rel/i18n/zh/emqx_ee_bridge_dynamo.hocon | 40 + rel/i18n/zh/emqx_ee_bridge_gcp_pubsub.hocon | 77 + rel/i18n/zh/emqx_ee_bridge_hstreamdb.hocon | 52 + rel/i18n/zh/emqx_ee_bridge_influxdb.hocon | 47 + rel/i18n/zh/emqx_ee_bridge_mongodb.hocon | 57 + rel/i18n/zh/emqx_ee_bridge_mysql.hocon | 40 + rel/i18n/zh/emqx_ee_bridge_pgsql.hocon | 40 + rel/i18n/zh/emqx_ee_bridge_redis.hocon | 41 + rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon | 40 + rel/i18n/zh/emqx_ee_bridge_sqlserver.hocon | 46 + rel/i18n/zh/emqx_ee_bridge_tdengine.hocon | 40 + rel/i18n/zh/emqx_ee_connector_cassa.hocon | 17 + .../zh/emqx_ee_connector_clickhouse.hocon | 15 + rel/i18n/zh/emqx_ee_connector_dynamo.hocon | 9 + rel/i18n/zh/emqx_ee_connector_hstreamdb.hocon | 45 + rel/i18n/zh/emqx_ee_connector_influxdb.hocon | 71 + rel/i18n/zh/emqx_ee_connector_rocketmq.hocon | 35 + rel/i18n/zh/emqx_ee_connector_sqlserver.hocon | 11 + rel/i18n/zh/emqx_ee_connector_tdengine.hocon | 11 + .../zh/emqx_ee_schema_registry_http_api.hocon | 39 + .../zh/emqx_ee_schema_registry_schema.hocon | 45 + rel/i18n/zh/emqx_exhook_api.hocon | 81 + rel/i18n/zh/emqx_exhook_schema.hocon | 43 + rel/i18n/zh/emqx_exproto_schema.hocon | 26 + rel/i18n/zh/emqx_gateway_api.hocon | 73 + rel/i18n/zh/emqx_gateway_api_authn.hocon | 45 + rel/i18n/zh/emqx_gateway_api_clients.hocon | 207 + rel/i18n/zh/emqx_gateway_api_listeners.hocon | 65 + rel/i18n/zh/emqx_gateway_schema.hocon | 112 + rel/i18n/zh/emqx_license_http_api.hocon | 15 + rel/i18n/zh/emqx_license_schema.hocon | 29 + rel/i18n/zh/emqx_limiter_schema.hocon | 89 + rel/i18n/zh/emqx_lwm2m_api.hocon | 27 + rel/i18n/zh/emqx_lwm2m_schema.hocon | 56 + rel/i18n/zh/emqx_mgmt_api_alarms.hocon | 37 + rel/i18n/zh/emqx_mgmt_api_banned.hocon | 54 + rel/i18n/zh/emqx_mgmt_api_key_schema.hocon | 19 + rel/i18n/zh/emqx_mgmt_api_publish.hocon | 81 + rel/i18n/zh/emqx_mgmt_api_status.hocon | 22 + rel/i18n/zh/emqx_modules_schema.hocon | 45 + rel/i18n/zh/emqx_mqttsn_schema.hocon | 30 + rel/i18n/zh/emqx_plugins_schema.hocon | 46 + rel/i18n/zh/emqx_prometheus_schema.hocon | 47 + rel/i18n/zh/emqx_psk_schema.hocon | 28 + rel/i18n/zh/emqx_resource_schema.hocon | 112 + rel/i18n/zh/emqx_retainer_api.hocon | 63 + rel/i18n/zh/emqx_retainer_schema.hocon | 46 + rel/i18n/zh/emqx_rewrite_api.hocon | 12 + rel/i18n/zh/emqx_rule_api_schema.hocon | 381 ++ rel/i18n/zh/emqx_rule_engine_api.hocon | 93 + rel/i18n/zh/emqx_rule_engine_schema.hocon | 184 + rel/i18n/zh/emqx_schema.hocon | 1473 ++++++ rel/i18n/zh/emqx_slow_subs_api.hocon | 30 + rel/i18n/zh/emqx_slow_subs_schema.hocon | 18 + rel/i18n/zh/emqx_statsd_api.hocon | 9 + rel/i18n/zh/emqx_statsd_schema.hocon | 30 + rel/i18n/zh/emqx_stomp_schema.hocon | 15 + rel/i18n/zh/emqx_telemetry_api.hocon | 54 + rel/i18n/zh/emqx_topic_metrics_api.hocon | 105 + 200 files changed, 15283 insertions(+), 14601 deletions(-) create mode 100644 rel/i18n/zh/emqx_authn_api.hocon create mode 100644 rel/i18n/zh/emqx_authn_http.hocon create mode 100644 rel/i18n/zh/emqx_authn_jwt.hocon create mode 100644 rel/i18n/zh/emqx_authn_mnesia.hocon create mode 100644 rel/i18n/zh/emqx_authn_mongodb.hocon create mode 100644 rel/i18n/zh/emqx_authn_mysql.hocon create mode 100644 rel/i18n/zh/emqx_authn_pgsql.hocon create mode 100644 rel/i18n/zh/emqx_authn_redis.hocon create mode 100644 rel/i18n/zh/emqx_authn_schema.hocon create mode 100644 rel/i18n/zh/emqx_authn_user_import_api.hocon create mode 100644 rel/i18n/zh/emqx_authz_api_cache.hocon create mode 100644 rel/i18n/zh/emqx_authz_api_mnesia.hocon create mode 100644 rel/i18n/zh/emqx_authz_api_schema.hocon create mode 100644 rel/i18n/zh/emqx_authz_api_settings.hocon create mode 100644 rel/i18n/zh/emqx_authz_api_sources.hocon create mode 100644 rel/i18n/zh/emqx_authz_schema.hocon create mode 100644 rel/i18n/zh/emqx_auto_subscribe_api.hocon create mode 100644 rel/i18n/zh/emqx_auto_subscribe_schema.hocon create mode 100644 rel/i18n/zh/emqx_bridge_api.hocon create mode 100644 rel/i18n/zh/emqx_bridge_kafka.hocon create mode 100644 rel/i18n/zh/emqx_bridge_mqtt_schema.hocon create mode 100644 rel/i18n/zh/emqx_bridge_schema.hocon create mode 100644 rel/i18n/zh/emqx_bridge_webhook_schema.hocon create mode 100644 rel/i18n/zh/emqx_coap_api.hocon create mode 100644 rel/i18n/zh/emqx_coap_schema.hocon create mode 100644 rel/i18n/zh/emqx_conf_schema.hocon create mode 100644 rel/i18n/zh/emqx_connector_api.hocon create mode 100644 rel/i18n/zh/emqx_connector_http.hocon create mode 100644 rel/i18n/zh/emqx_connector_ldap.hocon create mode 100644 rel/i18n/zh/emqx_connector_mongo.hocon create mode 100644 rel/i18n/zh/emqx_connector_mqtt.hocon create mode 100644 rel/i18n/zh/emqx_connector_mqtt_schema.hocon create mode 100644 rel/i18n/zh/emqx_connector_mysql.hocon create mode 100644 rel/i18n/zh/emqx_connector_pgsql.hocon create mode 100644 rel/i18n/zh/emqx_connector_redis.hocon create mode 100644 rel/i18n/zh/emqx_connector_schema_lib.hocon create mode 100644 rel/i18n/zh/emqx_dashboard_api.hocon create mode 100644 rel/i18n/zh/emqx_dashboard_schema.hocon create mode 100644 rel/i18n/zh/emqx_delayed_api.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_cassa.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_dynamo.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_gcp_pubsub.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_hstreamdb.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_influxdb.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_mongodb.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_mysql.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_pgsql.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_redis.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_sqlserver.hocon create mode 100644 rel/i18n/zh/emqx_ee_bridge_tdengine.hocon create mode 100644 rel/i18n/zh/emqx_ee_connector_cassa.hocon create mode 100644 rel/i18n/zh/emqx_ee_connector_clickhouse.hocon create mode 100644 rel/i18n/zh/emqx_ee_connector_dynamo.hocon create mode 100644 rel/i18n/zh/emqx_ee_connector_hstreamdb.hocon create mode 100644 rel/i18n/zh/emqx_ee_connector_influxdb.hocon create mode 100644 rel/i18n/zh/emqx_ee_connector_rocketmq.hocon create mode 100644 rel/i18n/zh/emqx_ee_connector_sqlserver.hocon create mode 100644 rel/i18n/zh/emqx_ee_connector_tdengine.hocon create mode 100644 rel/i18n/zh/emqx_ee_schema_registry_http_api.hocon create mode 100644 rel/i18n/zh/emqx_ee_schema_registry_schema.hocon create mode 100644 rel/i18n/zh/emqx_exhook_api.hocon create mode 100644 rel/i18n/zh/emqx_exhook_schema.hocon create mode 100644 rel/i18n/zh/emqx_exproto_schema.hocon create mode 100644 rel/i18n/zh/emqx_gateway_api.hocon create mode 100644 rel/i18n/zh/emqx_gateway_api_authn.hocon create mode 100644 rel/i18n/zh/emqx_gateway_api_clients.hocon create mode 100644 rel/i18n/zh/emqx_gateway_api_listeners.hocon create mode 100644 rel/i18n/zh/emqx_gateway_schema.hocon create mode 100644 rel/i18n/zh/emqx_license_http_api.hocon create mode 100644 rel/i18n/zh/emqx_license_schema.hocon create mode 100644 rel/i18n/zh/emqx_limiter_schema.hocon create mode 100644 rel/i18n/zh/emqx_lwm2m_api.hocon create mode 100644 rel/i18n/zh/emqx_lwm2m_schema.hocon create mode 100644 rel/i18n/zh/emqx_mgmt_api_alarms.hocon create mode 100644 rel/i18n/zh/emqx_mgmt_api_banned.hocon create mode 100644 rel/i18n/zh/emqx_mgmt_api_key_schema.hocon create mode 100644 rel/i18n/zh/emqx_mgmt_api_publish.hocon create mode 100644 rel/i18n/zh/emqx_mgmt_api_status.hocon create mode 100644 rel/i18n/zh/emqx_modules_schema.hocon create mode 100644 rel/i18n/zh/emqx_mqttsn_schema.hocon create mode 100644 rel/i18n/zh/emqx_plugins_schema.hocon create mode 100644 rel/i18n/zh/emqx_prometheus_schema.hocon create mode 100644 rel/i18n/zh/emqx_psk_schema.hocon create mode 100644 rel/i18n/zh/emqx_resource_schema.hocon create mode 100644 rel/i18n/zh/emqx_retainer_api.hocon create mode 100644 rel/i18n/zh/emqx_retainer_schema.hocon create mode 100644 rel/i18n/zh/emqx_rewrite_api.hocon create mode 100644 rel/i18n/zh/emqx_rule_api_schema.hocon create mode 100644 rel/i18n/zh/emqx_rule_engine_api.hocon create mode 100644 rel/i18n/zh/emqx_rule_engine_schema.hocon create mode 100644 rel/i18n/zh/emqx_schema.hocon create mode 100644 rel/i18n/zh/emqx_slow_subs_api.hocon create mode 100644 rel/i18n/zh/emqx_slow_subs_schema.hocon create mode 100644 rel/i18n/zh/emqx_statsd_api.hocon create mode 100644 rel/i18n/zh/emqx_statsd_schema.hocon create mode 100644 rel/i18n/zh/emqx_stomp_schema.hocon create mode 100644 rel/i18n/zh/emqx_telemetry_api.hocon create mode 100644 rel/i18n/zh/emqx_topic_metrics_api.hocon diff --git a/rel/i18n/emqx_authn_api.hocon b/rel/i18n/emqx_authn_api.hocon index a068edee2..07f9c6c3e 100644 --- a/rel/i18n/emqx_authn_api.hocon +++ b/rel/i18n/emqx_authn_api.hocon @@ -1,217 +1,96 @@ emqx_authn_api { - authentication_get { - desc { - en: """List authenticators for global authentication.""" - zh: """列出全局认证链上的认证器。""" - } - } +authentication_get.desc: +"""List authenticators for global authentication.""" - authentication_post { - desc { - en: """Create authenticator for global authentication.""" - zh: """为全局认证链创建认证器。""" - } - } +authentication_id_delete.desc: +"""Delete authenticator from global authentication chain.""" - authentication_id_get { - desc { - en: """Get authenticator from global authentication chain.""" - zh: """获取全局认证链上的指定认证器。""" - } - } +authentication_id_get.desc: +"""Get authenticator from global authentication chain.""" - authentication_id_put { - desc { - en: """Update authenticator from global authentication chain.""" - zh: """更新全局认证链上的指定认证器。""" - } - } +authentication_id_position_put.desc: +"""Move authenticator in global authentication chain.""" - authentication_id_delete { - desc { - en: """Delete authenticator from global authentication chain.""" - zh: """删除全局认证链上的指定认证器。""" - } - } +authentication_id_put.desc: +"""Update authenticator from global authentication chain.""" - authentication_id_status_get { - desc { - en: """Get authenticator status from global authentication chain.""" - zh: """获取全局认证链上指定认证器的状态。""" - } - } +authentication_id_status_get.desc: +"""Get authenticator status from global authentication chain.""" - listeners_listener_id_authentication_get { - desc { - en: """List authenticators for listener authentication.""" - zh: """列出监听器认证链上的认证器。""" - } - } +authentication_id_users_get.desc: +"""List users in authenticator in global authentication chain.""" - listeners_listener_id_authentication_post { - desc { - en: """Create authenticator for listener authentication.""" - zh: """在监听器认证链上创建认证器。""" - } - } +authentication_id_users_post.desc: +"""Create users for authenticator in global authentication chain.""" - listeners_listener_id_authentication_id_get { - desc { - en: """Get authenticator from listener authentication chain.""" - zh: """获取监听器认证链上的指定认证器。""" - } - } +authentication_id_users_user_id_delete.desc: +"""Delete user in authenticator in global authentication chain.""" - listeners_listener_id_authentication_id_put { - desc { - en: """Update authenticator from listener authentication chain.""" - zh: """更新监听器认证链上的指定认证器。""" - } - } +authentication_id_users_user_id_get.desc: +"""Get user from authenticator in global authentication chain.""" - listeners_listener_id_authentication_id_delete { - desc { - en: """Delete authenticator from listener authentication chain.""" - zh: """删除监听器认证链上的指定认证器。""" - } - } +authentication_id_users_user_id_put.desc: +"""Update user in authenticator in global authentication chain.""" - listeners_listener_id_authentication_id_status_get { - desc { - en: """Get authenticator status from listener authentication chain.""" - zh: """获取监听器认证链上指定认证器的状态。""" - } - } +authentication_post.desc: +"""Create authenticator for global authentication.""" - authentication_id_position_put { - desc { - en: """Move authenticator in global authentication chain.""" - zh: """更改全局认证链上指定认证器的顺序。""" - } - } +is_superuser.desc: +"""Is superuser""" - listeners_listener_id_authentication_id_position_put { - desc { - en: """Move authenticator in listener authentication chain.""" - zh: """更改监听器认证链上指定认证器的顺序。""" - } - } +like_user_id.desc: +"""Fuzzy search user_id (username or clientid).""" - authentication_id_users_post { - desc { - en: """Create users for authenticator in global authentication chain.""" - zh: """为全局认证链上的指定认证器创建用户数据。""" - } - } +like_user_id.label: +"""like_user_id""" - authentication_id_users_get { - desc { - en: """List users in authenticator in global authentication chain.""" - zh: """获取全局认证链上指定认证器中的用户数据。""" - } - } +listeners_listener_id_authentication_get.desc: +"""List authenticators for listener authentication.""" - listeners_listener_id_authentication_id_users_post { - desc { - en: """Create users for authenticator in listener authentication chain.""" - zh: """为监听器认证链上的指定认证器创建用户数据。""" - } - } +listeners_listener_id_authentication_id_delete.desc: +"""Delete authenticator from listener authentication chain.""" - listeners_listener_id_authentication_id_users_get { - desc { - en: """List users in authenticator in listener authentication chain.""" - zh: """列出监听器认证链上指定认证器中的用户数据。""" - } - } +listeners_listener_id_authentication_id_get.desc: +"""Get authenticator from listener authentication chain.""" - authentication_id_users_user_id_get { - desc { - en: """Get user from authenticator in global authentication chain.""" - zh: """获取全局认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_position_put.desc: +"""Move authenticator in listener authentication chain.""" - authentication_id_users_user_id_put { - desc { - en: """Update user in authenticator in global authentication chain.""" - zh: """更新全局认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_put.desc: +"""Update authenticator from listener authentication chain.""" - authentication_id_users_user_id_delete { - desc { - en: """Delete user in authenticator in global authentication chain.""" - zh: """删除全局认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_status_get.desc: +"""Get authenticator status from listener authentication chain.""" - listeners_listener_id_authentication_id_users_user_id_get { - desc { - en: """Get user from authenticator in listener authentication chain.""" - zh: """获取监听器认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_users_get.desc: +"""List users in authenticator in listener authentication chain.""" - listeners_listener_id_authentication_id_users_user_id_put { - desc { - en: """Update user in authenticator in listener authentication chain.""" - zh: """更新监听器认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_users_post.desc: +"""Create users for authenticator in listener authentication chain.""" - listeners_listener_id_authentication_id_users_user_id_delete { - desc { - en: """Delete user in authenticator in listener authentication chain.""" - zh: """删除监听器认证链上指定认证器中的指定用户数据。""" - } - } +listeners_listener_id_authentication_id_users_user_id_delete.desc: +"""Delete user in authenticator in listener authentication chain.""" - param_auth_id { - desc { - en: """Authenticator ID.""" - zh: """认证器 ID。""" - } - } +listeners_listener_id_authentication_id_users_user_id_get.desc: +"""Get user from authenticator in listener authentication chain.""" - param_listener_id { - desc { - en: """Listener ID.""" - zh: """监听器 ID。""" - } - } +listeners_listener_id_authentication_id_users_user_id_put.desc: +"""Update user in authenticator in listener authentication chain.""" - param_user_id { - desc { - en: """User ID.""" - zh: """用户 ID。""" - } - } +listeners_listener_id_authentication_post.desc: +"""Create authenticator for listener authentication.""" - param_position { - desc { - en: """Position of authenticator in chain. Possible values are 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'.""" - zh: """认证者在链中的位置。可能的值是 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'""" - } - } +param_auth_id.desc: +"""Authenticator ID.""" - like_user_id { - desc { - en: """Fuzzy search user_id (username or clientid).""" - zh: """使用用户 ID （username 或 clientid）模糊查询。""" - } - label { - en: """like_user_id""" - zh: """like_user_id""" - } - } +param_listener_id.desc: +"""Listener ID.""" - is_superuser { - desc { - en: """Is superuser""" - zh: """是否是超级用户""" - } - } +param_position.desc: +"""Position of authenticator in chain. Possible values are 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'.""" + +param_user_id.desc: +"""User ID.""" } diff --git a/rel/i18n/emqx_authn_http.hocon b/rel/i18n/emqx_authn_http.hocon index 129db5054..582bacea8 100644 --- a/rel/i18n/emqx_authn_http.hocon +++ b/rel/i18n/emqx_authn_http.hocon @@ -1,81 +1,45 @@ emqx_authn_http { - get { - desc { - en: """Configuration of authenticator using HTTP Server as authentication service (Using GET request).""" - zh: """使用 HTTP Server 作为认证服务的认证器的配置项 (使用 GET 请求)。""" - } - } - post { - desc { - en: """Configuration of authenticator using HTTP Server as authentication service (Using POST request).""" - zh: """使用 HTTP Server 作为认证服务的认证器的配置项 (使用 POST 请求)。""" - } - } +body.desc: +"""HTTP request body.""" - method { - desc { - en: """HTTP request method.""" - zh: """HTTP 请求方法。""" - } - label { - en: """Request Method""" - zh: """请求方法""" - } - } +body.label: +"""Request Body""" - url { - desc { - en: """URL of the HTTP server.""" - zh: """认证 HTTP 服务器地址。""" - } - label { - en: """URL""" - zh: """URL""" - } - } +get.desc: +"""Configuration of authenticator using HTTP Server as authentication service (Using GET request).""" - headers { - desc { - en: """List of HTTP Headers.""" - zh: """HTTP Headers 列表""" - } - label { - en: """Headers""" - zh: """请求头""" - } - } +headers.desc: +"""List of HTTP Headers.""" - headers_no_content_type { - desc { - en: """List of HTTP headers (without content-type).""" - zh: """HTTP Headers 列表 (无 content-type) 。""" - } - label { - en: """headers_no_content_type""" - zh: """请求头(无 content-type)""" - } - } +headers.label: +"""Headers""" - body { - desc { - en: """HTTP request body.""" - zh: """HTTP request body。""" - } - label { - en: """Request Body""" - zh: """Request Body""" - } - } +headers_no_content_type.desc: +"""List of HTTP headers (without content-type).""" + +headers_no_content_type.label: +"""headers_no_content_type""" + +method.desc: +"""HTTP request method.""" + +method.label: +"""Request Method""" + +post.desc: +"""Configuration of authenticator using HTTP Server as authentication service (Using POST request).""" + +request_timeout.desc: +"""HTTP request timeout.""" + +request_timeout.label: +"""Request Timeout""" + +url.desc: +"""URL of the HTTP server.""" + +url.label: +"""URL""" - request_timeout { - desc { - en: """HTTP request timeout.""" - zh: """HTTP 请求超时时长。""" - } - label { - en: """Request Timeout""" - zh: """请求超时时间""" - } - } } diff --git a/rel/i18n/emqx_authn_jwt.hocon b/rel/i18n/emqx_authn_jwt.hocon index 6a4a1e2d4..e28213e37 100644 --- a/rel/i18n/emqx_authn_jwt.hocon +++ b/rel/i18n/emqx_authn_jwt.hocon @@ -1,219 +1,118 @@ emqx_authn_jwt { - use_jwks { - desc { - en: """Whether to use JWKS.""" - zh: """是否使用 JWKS。""" - } - label { - en: """Whether to Use JWKS""" - zh: """是否使用 JWKS""" - } - } - algorithm { - desc { - en: """JWT signing algorithm, Supports HMAC (configured as hmac-based) and RSA, ECDSA (configured as public-key).""" - zh: """JWT 签名算法，支持 HMAC (配置为 hmac-based）和 RSA、ECDSA (配置为 public-key)。""" - } - label { - en: """JWT Signing Algorithm""" - zh: """JWT 签名算法""" - } - } +acl_claim_name.desc: +"""JWT claim name to use for getting ACL rules.""" - public_key { - desc { - en: """The public key used to verify the JWT.""" - zh: """用于验证 JWT 的公钥。""" - } - label { - en: """Public Key""" - zh: """公钥""" - } - } +acl_claim_name.label: +"""ACL claim name""" - secret_base64_encoded { - desc { - en: """Whether secret is base64 encoded.""" - zh: """密钥是否为 Base64 编码。""" - } - label { - en: """Whether Secret is Base64 Encoded""" - zh: """密钥是否为 Base64 编码""" - } - } +algorithm.desc: +"""JWT signing algorithm, Supports HMAC (configured as hmac-based) and RSA, ECDSA (configured as public-key).""" - secret { - desc { - en: """The key to verify the JWT using HMAC algorithm.""" - zh: """使用 HMAC 算法时用于验证 JWT 的密钥""" - } - label { - en: """Secret""" - zh: """Secret""" - } - } +algorithm.label: +"""JWT Signing Algorithm""" - endpoint { - desc { - en: """JWKS endpoint, it's a read-only endpoint that returns the server's public key set in the JWKS format.""" - zh: """JWKS 端点， 它是一个以 JWKS 格式返回服务端的公钥集的只读端点。""" - } - label { - en: """JWKS Endpoint""" - zh: """JWKS Endpoint""" - } - } +cacertfile.desc: +"""Path to a file containing PEM-encoded CA certificates.""" - refresh_interval { - desc { - en: """JWKS refresh interval.""" - zh: """JWKS 刷新间隔。""" - } - label { - en: """JWKS Refresh Interval""" - zh: """JWKS 刷新间隔""" - } - } +cacertfile.label: +"""CA Certificate File""" - cacertfile { - desc { - en: """Path to a file containing PEM-encoded CA certificates.""" - zh: """包含 PEM 编码的 CA 证书的文件的路径。""" - } - label { - en: """CA Certificate File""" - zh: """CA 证书文件""" - } - } +certfile.desc: +"""Path to a file containing the user certificate.""" - certfile { - desc { - en: """Path to a file containing the user certificate.""" - zh: """包含用户证书的文件的路径。""" - } - label { - en: """Certificate File""" - zh: """证书文件""" - } - } +certfile.label: +"""Certificate File""" - keyfile { - desc { - en: """Path to a file containing the user's private PEM-encoded key.""" - zh: """包含 PEM 编码的用户私钥的文件的路径。""" - } - label { - en: """Key File""" - zh: """私钥文件""" - } - } +enable.desc: +"""Enable/disable SSL.""" - verify { - desc { - en: """Enable or disable SSL peer verification.""" - zh: """指定握手过程中是否校验对端证书。""" - } - label { - en: """Verify""" - zh: """Verify""" - } - } +enable.label: +"""Enable/disable SSL""" - server_name_indication { - desc { - en: """Server Name Indication (SNI).""" - zh: """服务器名称指示（SNI）。""" - } - label { - en: """Server Name Indication""" - zh: """服务器名称指示""" - } - } +endpoint.desc: +"""JWKS endpoint, it's a read-only endpoint that returns the server's public key set in the JWKS format.""" - verify_claims { - desc { - en: """A list of custom claims to validate, which is a list of name/value pairs. +endpoint.label: +"""JWKS Endpoint""" + +from.desc: +"""Field to take JWT from.""" + +from.label: +"""From Field""" + +hmac-based.desc: +"""Configuration when the JWT for authentication is issued using the HMAC algorithm.""" + +jwks.desc: +"""Configuration when JWTs used for authentication need to be fetched from the JWKS endpoint.""" + +keyfile.desc: +"""Path to a file containing the user's private PEM-encoded key.""" + +keyfile.label: +"""Key File""" + +public-key.desc: +"""Configuration when the JWT for authentication is issued using RSA or ECDSA algorithm.""" + +public_key.desc: +"""The public key used to verify the JWT.""" + +public_key.label: +"""Public Key""" + +refresh_interval.desc: +"""JWKS refresh interval.""" + +refresh_interval.label: +"""JWKS Refresh Interval""" + +secret.desc: +"""The key to verify the JWT using HMAC algorithm.""" + +secret.label: +"""Secret""" + +secret_base64_encoded.desc: +"""Whether secret is base64 encoded.""" + +secret_base64_encoded.label: +"""Whether Secret is Base64 Encoded""" + +server_name_indication.desc: +"""Server Name Indication (SNI).""" + +server_name_indication.label: +"""Server Name Indication""" + +ssl.desc: +"""SSL options.""" + +ssl.label: +"""SSL Options""" + +use_jwks.desc: +"""Whether to use JWKS.""" + +use_jwks.label: +"""Whether to Use JWKS""" + +verify.desc: +"""Enable or disable SSL peer verification.""" + +verify.label: +"""Verify""" + +verify_claims.desc: +"""A list of custom claims to validate, which is a list of name/value pairs. Values can use the following placeholders: - ${username}: Will be replaced at runtime with Username used by the client when connecting - ${clientid}: Will be replaced at runtime with Client ID used by the client when connecting Authentication will verify that the value of claims in the JWT (taken from the Password field) matches what is required in verify_claims.""" - zh: """需要验证的自定义声明列表，它是一个名称/值对列表。 -值可以使用以下占位符： -- ${username}: 将在运行时被替换为客户端连接时使用的用户名 -- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符 -认证时将验证 JWT（取自 Password 字段）中 claims 的值是否与 verify_claims 中要求的相匹配。""" - } - label { - en: """Verify Claims""" - zh: """Verify Claims""" - } - } - - ssl { - desc { - en: """SSL options.""" - zh: """SSL 选项。""" - } - label { - en: """SSL Options""" - zh: """SSL 选项""" - } - } - - enable { - desc { - en: """Enable/disable SSL.""" - zh: """启用/禁用 SSL。""" - } - label { - en: """Enable/disable SSL""" - zh: """启用/禁用 SSL""" - } - } - - hmac-based { - desc { - en: """Configuration when the JWT for authentication is issued using the HMAC algorithm.""" - zh: """用于认证的 JWT 使用 HMAC 算法签发时的配置。""" - } - } - - public-key { - desc { - en: """Configuration when the JWT for authentication is issued using RSA or ECDSA algorithm.""" - zh: """用于认证的 JWT 使用 RSA 或 ECDSA 算法签发时的配置。""" - } - } - - jwks { - desc { - en: """Configuration when JWTs used for authentication need to be fetched from the JWKS endpoint.""" - zh: """用于认证的 JWTs 需要从 JWKS 端点获取时的配置。""" - } - } - - acl_claim_name { - desc { - en: """JWT claim name to use for getting ACL rules.""" - zh: """JWT claim name to use for getting ACL rules.""" - } - label { - en: """ACL claim name""" - zh: """ACL claim name""" - } - } - - from { - desc { - en: """Field to take JWT from.""" - zh: """要从中获取 JWT 的字段。""" - } - label { - en: """From Field""" - zh: """源字段""" - } - } +verify_claims.label: +"""Verify Claims""" } diff --git a/rel/i18n/emqx_authn_mnesia.hocon b/rel/i18n/emqx_authn_mnesia.hocon index 0d07217d9..4cfab9493 100644 --- a/rel/i18n/emqx_authn_mnesia.hocon +++ b/rel/i18n/emqx_authn_mnesia.hocon @@ -1,21 +1,12 @@ emqx_authn_mnesia { - authentication { - desc { - en: """Configuration of authenticator using built-in database as data source.""" - zh: """使用内置数据库作为认证数据源的认证器的配置项。""" - } - } - user_id_type { - desc { - en: """Specify whether to use `clientid` or `username` for authentication.""" - zh: """指定使用客户端ID `clientid` 还是用户名 `username` 进行认证。""" - } +authentication.desc: +"""Configuration of authenticator using built-in database as data source.""" - label: { - en: """Authentication ID Type""" - zh: """认证 ID 类型""" - } - } +user_id_type.desc: +"""Specify whether to use `clientid` or `username` for authentication.""" + +user_id_type.label: +"""Authentication ID Type""" } diff --git a/rel/i18n/emqx_authn_mongodb.hocon b/rel/i18n/emqx_authn_mongodb.hocon index 80d6473ed..97311f751 100644 --- a/rel/i18n/emqx_authn_mongodb.hocon +++ b/rel/i18n/emqx_authn_mongodb.hocon @@ -1,83 +1,45 @@ emqx_authn_mongodb { - standalone { - desc { - en: """Configuration of authenticator using MongoDB (Standalone) as authentication data source.""" - zh: """使用 MongoDB (Standalone) 作为认证数据源的认证器的配置项。""" - } - } - replica-set { - desc { - en: """Configuration of authenticator using MongoDB (Replica Set) as authentication data source.""" - zh: """使用 MongoDB (Replica Set) 作为认证数据源的认证器的配置项。""" - } - } +collection.desc: +"""Collection used to store authentication data.""" - sharded-cluster { - desc { - en: """Configuration of authenticator using MongoDB (Sharded Cluster) as authentication data source.""" - zh: """使用 MongoDB (Sharded Cluster) 作为认证数据源的认证器的配置项。""" - } - } +collection.label: +"""Collection""" - collection { - desc { - en: """Collection used to store authentication data.""" - zh: """存储认证数据的集合。""" - } - label: { - en: """Collection""" - zh: """集合""" - } - } - - filter { - desc { - en: """Conditional expression that defines the filter condition in the query. +filter.desc: +"""Conditional expression that defines the filter condition in the query. Filter supports the following placeholders: - ${username}: Will be replaced at runtime with Username used by the client when connecting - ${clientid}: Will be replaced at runtime with Client ID used by the client when connecting""" - zh: """在查询中定义过滤条件的条件表达式。 -过滤器支持如下占位符： -- ${username}: 将在运行时被替换为客户端连接时使用的用户名 -- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符""" - } - label: { - en: """Filter""" - zh: """过滤器""" - } - } - password_hash_field { - desc { - en: """Document field that contains password hash.""" - zh: """文档中用于存放密码散列的字段。""" - } - label: { - en: """Password Hash Field""" - zh: """密码散列字段""" - } - } +filter.label: +"""Filter""" - salt_field { - desc { - en: """Document field that contains the password salt.""" - zh: """文档中用于存放盐值的字段。""" - } - label: { - en: """Salt Field""" - zh: """盐值字段""" - } - } +is_superuser_field.desc: +"""Document field that defines if the user has superuser privileges.""" + +is_superuser_field.label: +"""Is Superuser Field""" + +password_hash_field.desc: +"""Document field that contains password hash.""" + +password_hash_field.label: +"""Password Hash Field""" + +replica-set.desc: +"""Configuration of authenticator using MongoDB (Replica Set) as authentication data source.""" + +salt_field.desc: +"""Document field that contains the password salt.""" + +salt_field.label: +"""Salt Field""" + +sharded-cluster.desc: +"""Configuration of authenticator using MongoDB (Sharded Cluster) as authentication data source.""" + +standalone.desc: +"""Configuration of authenticator using MongoDB (Standalone) as authentication data source.""" - is_superuser_field { - desc { - en: """Document field that defines if the user has superuser privileges.""" - zh: """文档中用于定义用户是否具有超级用户权限的字段。""" - } - label: { - en: """Is Superuser Field""" - zh: """超级用户字段""" - } - } } diff --git a/rel/i18n/emqx_authn_mysql.hocon b/rel/i18n/emqx_authn_mysql.hocon index 4ddfd5701..45634a4ad 100644 --- a/rel/i18n/emqx_authn_mysql.hocon +++ b/rel/i18n/emqx_authn_mysql.hocon @@ -1,30 +1,18 @@ emqx_authn_mysql { - authentication { - desc { - en: """Configuration of authenticator using MySQL as authentication data source.""" - zh: """使用 MySQL 作为认证数据源的认证器的配置项。""" - } - } - query { - desc { - en: """SQL used to query data for authentication, such as password hash.""" - zh: """用于查询密码散列等用于认证的数据的 SQL 语句。""" - } - label: { - en: """Query""" - zh: """查询语句""" - } - } +authentication.desc: +"""Configuration of authenticator using MySQL as authentication data source.""" + +query.desc: +"""SQL used to query data for authentication, such as password hash.""" + +query.label: +"""Query""" + +query_timeout.desc: +"""Timeout for the SQL query.""" + +query_timeout.label: +"""Query Timeout""" - query_timeout { - desc { - en: """Timeout for the SQL query.""" - zh: """SQL 查询的超时时间。""" - } - label: { - en: """Query Timeout""" - zh: """查询超时""" - } - } } diff --git a/rel/i18n/emqx_authn_pgsql.hocon b/rel/i18n/emqx_authn_pgsql.hocon index 298e38774..a9d727785 100644 --- a/rel/i18n/emqx_authn_pgsql.hocon +++ b/rel/i18n/emqx_authn_pgsql.hocon @@ -1,19 +1,12 @@ emqx_authn_pgsql { - authentication { - desc { - en: """Configuration of authenticator using PostgreSQL as authentication data source.""" - zh: """使用 PostgreSQL 作为认证数据源的认证器的配置项。""" - } - } - query { - desc { - en: """SQL used to query data for authentication, such as password hash.""" - zh: """用于查询密码散列等用于认证的数据的 SQL 语句。""" - } - label: { - en: """Query""" - zh: """查询语句""" - } - } +authentication.desc: +"""Configuration of authenticator using PostgreSQL as authentication data source.""" + +query.desc: +"""SQL used to query data for authentication, such as password hash.""" + +query.label: +"""Query""" + } diff --git a/rel/i18n/emqx_authn_redis.hocon b/rel/i18n/emqx_authn_redis.hocon index a9cd4a414..f39d54061 100644 --- a/rel/i18n/emqx_authn_redis.hocon +++ b/rel/i18n/emqx_authn_redis.hocon @@ -1,33 +1,18 @@ emqx_authn_redis { - standalone { - desc { - en: """Configuration of authenticator using Redis (Standalone) as authentication data source.""" - zh: """使用 Redis (Standalone) 作为认证数据源的认证器的配置项。""" - } - } - cluster { - desc { - en: """Configuration of authenticator using Redis (Cluster) as authentication data source.""" - zh: """使用 Redis (Cluster) 作为认证数据源的认证器的配置项。""" - } - } +cluster.desc: +"""Configuration of authenticator using Redis (Cluster) as authentication data source.""" - sentinel { - desc { - en: """Configuration of authenticator using Redis (Sentinel) as authentication data source.""" - zh: """使用 Redis (Sentinel) 作为认证数据源的认证器的配置项。""" - } - } +cmd.desc: +"""The Redis Command used to query data for authentication such as password hash, currently only supports HGET and HMGET.""" + +cmd.label: +"""Command""" + +sentinel.desc: +"""Configuration of authenticator using Redis (Sentinel) as authentication data source.""" + +standalone.desc: +"""Configuration of authenticator using Redis (Standalone) as authentication data source.""" - cmd { - desc { - en: """The Redis Command used to query data for authentication such as password hash, currently only supports HGET and HMGET.""" - zh: """用于查询密码散列等用于认证的数据的 Redis Command，目前仅支持 HGET 与 HMGET。""" - } - label: { - en: """Command""" - zh: """Command""" - } - } } diff --git a/rel/i18n/emqx_authn_schema.hocon b/rel/i18n/emqx_authn_schema.hocon index 917554af0..98263ca49 100644 --- a/rel/i18n/emqx_authn_schema.hocon +++ b/rel/i18n/emqx_authn_schema.hocon @@ -1,243 +1,135 @@ emqx_authn_schema { - enable { - desc { - en: """Set to true or false to disable this auth provider.""" - zh: """设为 true 或 false 以启用或禁用此认证数据源。""" - } - label: { - en: """Enable""" - zh: """启用""" - } - } - mechanism { - desc { - en: """Authentication mechanism.""" - zh: """认证机制。""" - } - label: { - en: """Authentication Mechanism""" - zh: """认证机制""" - } - } +backend.desc: +"""Backend type.""" - backend { - desc { - en: """Backend type.""" - zh: """后端类型。""" - } - label: { - en: """Backend Type""" - zh: """后端类型""" - } - } +backend.label: +"""Backend Type""" - metrics { - desc { - en: """The metrics of the resource.""" - zh: """资源统计指标。""" - } - label: { - en: """Metrics""" - zh: """指标""" - } - } +enable.desc: +"""Set to true or false to disable this auth provider.""" - node_metrics { - desc { - en: """The metrics of the resource for each node.""" - zh: """每个节点上资源的统计指标。""" - } - label: { - en: """Resource Metrics in Node""" - zh: """节点资源指标""" - } - } +enable.label: +"""Enable""" - status { - desc { - en: """The status of the resource.""" - zh: """资源状态。""" - } - label: { - en: """Status""" - zh: """状态""" - } - } +failed.desc: +"""Count of query failed.""" - node_status { - desc { - en: """The status of the resource for each node.""" - zh: """每个节点上资源的状态。""" - } - label: { - en: """Resource Status in Node""" - zh: """节点资源状态""" - } - } +failed.label: +"""Failed""" - node_error { - desc { - en: """The error of node.""" - zh: """节点上产生的错误。""" - } - label: { - en: """Error in Node""" - zh: """节点产生的错误""" - } - } +matched.desc: +"""Count of this resource is queried.""" - matched { - desc { - en: """Count of this resource is queried.""" - zh: """请求命中次数。""" - } - label: { - en: """Matched""" - zh: """已命中""" - } - } +matched.label: +"""Matched""" - success { - desc { - en: """Count of query success.""" - zh: """请求成功次数。""" - } - label: { - en: """Success""" - zh: """成功""" - } - } +mechanism.desc: +"""Authentication mechanism.""" - failed { - desc { - en: """Count of query failed.""" - zh: """请求失败次数。""" - } - label: { - en: """Failed""" - zh: """失败""" - } - } +mechanism.label: +"""Authentication Mechanism""" - rate { - desc { - en: """The rate of matched, times/second.""" - zh: """命中速率，单位：次/秒。""" - } - label: { - en: """Rate""" - zh: """速率""" - } - } +metrics.desc: +"""The metrics of the resource.""" - rate_max { - desc { - en: """The max rate of matched, times/second.""" - zh: """最大命中速率，单位：次/秒。""" - } - label: { - en: """Max Rate""" - zh: """最大速率""" - } - } +metrics.label: +"""Metrics""" - rate_last5m { - desc { - en: """The average rate of matched in the last 5 minutes, times/second.""" - zh: """5分钟内平均命中速率，单位：次/秒。""" - } - label: { - en: """Rate in Last 5min""" - zh: """5分钟内速率""" - } - } +metrics_failed.desc: +"""The required authentication information is found in the current instance, and the instance returns authentication failure.""" - node { - desc { - en: """Node name.""" - zh: """节点名称。""" - } - label: { - en: """Node Name.""" - zh: """节点名称。""" - } - } +metrics_failed.label: +"""Authentication Failed Times""" - metrics_nomatch { - desc { - en: """The number of times the instance was ignored when the required authentication information was not found in the current instance.""" - zh: """在当前实例中没有找到需要的认证信息，实例被忽略的次数。""" - } - label: { - en: """Nomatch Times""" - zh: """实例被忽略的次数""" - } - } +metrics_nomatch.desc: +"""The number of times the instance was ignored when the required authentication information was not found in the current instance.""" - metrics_total { - desc { - en: """The total number of times the current instance was triggered.""" - zh: """当前实例被触发的总次数。""" - } - label: { - en: """Total Triggered Times""" - zh: """当前实例被触发的总次数""" - } - } +metrics_nomatch.label: +"""Nomatch Times""" - metrics_success { - desc { - en: """The required authentication information is found in the current instance, and the instance returns authentication success.""" - zh: """在当前实例中找到需要的认证信息，并且实例返回认证成功的次数。""" - } - label: { - en: """Authentication Success Times""" - zh: """实例认证成功的次数""" - } - } +metrics_rate.desc: +"""The total rate at which instances are triggered, times/second.""" - metrics_failed { - desc { - en: """The required authentication information is found in the current instance, and the instance returns authentication failure.""" - zh: """在当前实例中找到需要的认证信息，并且实例返回认证失败的次数。""" - } - label: { - en: """Authentication Failed Times""" - zh: """实例认证失败的次数""" - } - } +metrics_rate.label: +"""Total Triggered Rate""" - metrics_rate { - desc { - en: """The total rate at which instances are triggered, times/second.""" - zh: """实例被触发的速率。触发速率等于匹配速率 + 忽略速率，单位：次/秒。""" - } - label: { - en: """Total Triggered Rate""" - zh: """实例被触发的速率""" - } - } +metrics_rate_last5m.desc: +"""The average trigger rate of the instance within 5 minutes, times/second.""" - metrics_rate_max { - desc { - en: """The highest trigger rate the instance has ever reached, times/second.""" - zh: """实例曾经达到的最高触发速率，单位：次/秒。""" - } - label: { - en: """Highest Triggered Rate""" - zh: """实例曾经达到的最高触发速率""" - } - } +metrics_rate_last5m.label: +"""Average Triggered Rate in Last 5min""" + +metrics_rate_max.desc: +"""The highest trigger rate the instance has ever reached, times/second.""" + +metrics_rate_max.label: +"""Highest Triggered Rate""" + +metrics_success.desc: +"""The required authentication information is found in the current instance, and the instance returns authentication success.""" + +metrics_success.label: +"""Authentication Success Times""" + +metrics_total.desc: +"""The total number of times the current instance was triggered.""" + +metrics_total.label: +"""Total Triggered Times""" + +node.desc: +"""Node name.""" + +node.label: +"""Node Name.""" + +node_error.desc: +"""The error of node.""" + +node_error.label: +"""Error in Node""" + +node_metrics.desc: +"""The metrics of the resource for each node.""" + +node_metrics.label: +"""Resource Metrics in Node""" + +node_status.desc: +"""The status of the resource for each node.""" + +node_status.label: +"""Resource Status in Node""" + +rate.desc: +"""The rate of matched, times/second.""" + +rate.label: +"""Rate""" + +rate_last5m.desc: +"""The average rate of matched in the last 5 minutes, times/second.""" + +rate_last5m.label: +"""Rate in Last 5min""" + +rate_max.desc: +"""The max rate of matched, times/second.""" + +rate_max.label: +"""Max Rate""" + +status.desc: +"""The status of the resource.""" + +status.label: +"""Status""" + +success.desc: +"""Count of query success.""" + +success.label: +"""Success""" - metrics_rate_last5m { - desc { - en: """The average trigger rate of the instance within 5 minutes, times/second.""" - zh: """实例5分钟内平均触发速率，单位：次/秒。""" - } - label: { - en: """Average Triggered Rate in Last 5min""" - zh: """实例5分钟内平均触发速率""" - } - } } diff --git a/rel/i18n/emqx_authn_user_import_api.hocon b/rel/i18n/emqx_authn_user_import_api.hocon index 294897ec1..f8fb1757c 100644 --- a/rel/i18n/emqx_authn_user_import_api.hocon +++ b/rel/i18n/emqx_authn_user_import_api.hocon @@ -1,17 +1,9 @@ emqx_authn_user_import_api { - authentication_id_import_users_post { - desc { - en: """Import users into authenticator in global authentication chain.""" - zh: """为全局认证链上的指定认证器导入用户数据。""" - } - } +authentication_id_import_users_post.desc: +"""Import users into authenticator in global authentication chain.""" - listeners_listener_id_authentication_id_import_users_post { - desc { - en: """Import users into authenticator in listener authentication chain.""" - zh: """为监听器认证链上的指定认证器导入用户数据。""" - } - } +listeners_listener_id_authentication_id_import_users_post.desc: +"""Import users into authenticator in listener authentication chain.""" } diff --git a/rel/i18n/emqx_authz_api_cache.hocon b/rel/i18n/emqx_authz_api_cache.hocon index 9c620a22d..0789c1e71 100644 --- a/rel/i18n/emqx_authz_api_cache.hocon +++ b/rel/i18n/emqx_authz_api_cache.hocon @@ -1,8 +1,6 @@ emqx_authz_api_cache { - authorization_cache_delete { - desc { - en: """Clean all authorization cache in the cluster.""" - zh: """清除集群中所有授权数据缓存。""" - } - } + +authorization_cache_delete.desc: +"""Clean all authorization cache in the cluster.""" + } diff --git a/rel/i18n/emqx_authz_api_mnesia.hocon b/rel/i18n/emqx_authz_api_mnesia.hocon index 6d318d02b..4cfed2970 100644 --- a/rel/i18n/emqx_authz_api_mnesia.hocon +++ b/rel/i18n/emqx_authz_api_mnesia.hocon @@ -1,177 +1,87 @@ emqx_authz_api_mnesia { - users_username_get { - desc { - en: """Show the list of rules for users""" - zh: """获取内置数据库中所有用户名类型的规则记录""" - } - } - users_username_post { - desc { - en: """Add new rule for 'username'""" - zh: """添加内置数据库中用户名类型的规则记录""" - } - } +action.desc: +"""Authorized action (pub/sub/all)""" - users_clientid_get { - desc { - en: """Show the list of rules for clients""" - zh: """获取内置数据库中所有客户端标识符类型的规则记录""" - } - } +action.label: +"""action""" - users_clientid_post { - desc { - en: """Add new rule for 'clientid'""" - zh: """添加内置数据库中客户端标识符类型的规则记录""" - } - } +clientid.desc: +"""ClientID""" +clientid.label: +"""clientid""" - user_username_get { - desc { - en: """Get rule for 'username'""" - zh: """获取内置数据库中指定用户名类型的规则记录""" - } - } +fuzzy_clientid.desc: +"""Fuzzy search `clientid` as substring""" - user_username_put { - desc { - en: """Set rule for 'username'""" - zh: """更新内置数据库中指定用户名类型的规则记录""" - } - } +fuzzy_clientid.label: +"""fuzzy_clientid""" - user_username_delete { - desc { - en: """Delete rule for 'username'""" - zh: """删除内置数据库中指定用户名类型的规则记录""" - } - } +fuzzy_username.desc: +"""Fuzzy search `username` as substring""" - user_clientid_get { - desc { - en: """Get rule for 'clientid'""" - zh: """获取内置数据库中指定客户端标识符类型的规则记录""" - } - } +fuzzy_username.label: +"""fuzzy_username""" - user_clientid_put { - desc { - en: """Set rule for 'clientid'""" - zh: """更新内置数据库中指定客户端标识符类型的规则记录""" - } - } +permission.desc: +"""Permission""" - user_clientid_delete { - desc { - en: """Delete rule for 'clientid'""" - zh: """删除内置数据库中指定客户端标识符类型的规则记录""" - } - } +permission.label: +"""permission""" - rules_all_get { - desc { - en: """Show the list of rules for 'all'""" - zh: """列出为所有客户端启用的规则列表""" - } - } +rules_all_delete.desc: +"""Delete rules for 'all'""" - rules_all_post { - desc { - en: """Create/Update the list of rules for 'all'.""" - zh: """创建/更新 为所有客户端启用的规则列表。""" - } - } +rules_all_get.desc: +"""Show the list of rules for 'all'""" - rules_all_delete { - desc { - en: """Delete rules for 'all'""" - zh: """删除 `all` 规则""" - } - } +rules_all_post.desc: +"""Create/Update the list of rules for 'all'.""" - rules_delete { - desc { - en: """Delete all rules for all 'users', 'clients' and 'all'""" - zh: """清除内置数据库中的所有类型（'users' 、'clients' 、'all'）的所有规则""" - } - } +rules_delete.desc: +"""Delete all rules for all 'users', 'clients' and 'all'""" - fuzzy_username { - desc { - en: """Fuzzy search `username` as substring""" - zh: """使用字串匹配模糊搜索用户名""" - } - label { - en: """fuzzy_username""" - zh: """用户名子串""" - } - } +topic.desc: +"""Rule on specific topic""" - fuzzy_clientid { - desc { - en: """Fuzzy search `clientid` as substring""" - zh: """使用字串匹配模糊搜索客户端标识符""" - } - label { - en: """fuzzy_clientid""" - zh: """客户端标识符子串""" - } - } +topic.label: +"""topic""" - topic { - desc { - en: """Rule on specific topic""" - zh: """在指定主题上的规则""" - } - label { - en: """topic""" - zh: """主题""" - } - } +user_clientid_delete.desc: +"""Delete rule for 'clientid'""" - permission { - desc { - en: """Permission""" - zh: """权限""" - } - label { - en: """permission""" - zh: """权限""" - } - } +user_clientid_get.desc: +"""Get rule for 'clientid'""" - action { - desc { - en: """Authorized action (pub/sub/all)""" - zh: """被授权的行为 (发布/订阅/所有)""" - } - label { - en: """action""" - zh: """行为""" - } - } +user_clientid_put.desc: +"""Set rule for 'clientid'""" - clientid { - desc { - en: """ClientID""" - zh: """客户端标识符""" - } - label { - en: """clientid""" - zh: """客户端标识符""" - } - } +user_username_delete.desc: +"""Delete rule for 'username'""" + +user_username_get.desc: +"""Get rule for 'username'""" + +user_username_put.desc: +"""Set rule for 'username'""" + +username.desc: +"""Username""" + +username.label: +"""username""" + +users_clientid_get.desc: +"""Show the list of rules for clients""" + +users_clientid_post.desc: +"""Add new rule for 'clientid'""" + +users_username_get.desc: +"""Show the list of rules for users""" + +users_username_post.desc: +"""Add new rule for 'username'""" - username { - desc { - en: """Username""" - zh: """用户名""" - } - label { - en: """username""" - zh: """用户名""" - } - } } diff --git a/rel/i18n/emqx_authz_api_schema.hocon b/rel/i18n/emqx_authz_api_schema.hocon index d649dd5a0..2edfc8fcb 100644 --- a/rel/i18n/emqx_authz_api_schema.hocon +++ b/rel/i18n/emqx_authz_api_schema.hocon @@ -1,185 +1,90 @@ emqx_authz_api_schema { - enable { - desc { - en: """Set to true or false to disable this ACL provider.""" - zh: """设为 true 或 false 以启用或禁用此访问控制数据源。""" - } - label { - en: """enable""" - zh: """enable""" - } - } - type { - desc { - en: """Backend type.""" - zh: """数据后端类型。""" - } - label { - en: """type""" - zh: """type""" - } - } +body.desc: +"""HTTP request body.""" -#==== authz_file +body.label: +"""body""" - rules { - desc { - en: """Authorization static file rules.""" - zh: """静态授权文件规则。""" - } - label { - en: """rules""" - zh: """规则""" - } - } +cmd.desc: +"""Database query used to retrieve authorization data.""" -#==== authz_http +cmd.label: +"""cmd""" - method { - desc { - en: """HTTP method.""" - zh: """HTTP 请求方法。""" - } - label { - en: """method""" - zh: """method""" - } - } +collection.desc: +"""`MongoDB` collection containing the authorization data.""" - url { - desc { - en: """URL of the auth server.""" - zh: """认证服务器 URL。""" - } - label { - en: """url""" - zh: """url""" - } - } +collection.label: +"""collection""" - headers { - desc { - en: """List of HTTP Headers.""" - zh: """HTTP Headers 列表""" - } - label { - en: """Headers""" - zh: """请求头""" - } - } +enable.desc: +"""Set to true or false to disable this ACL provider.""" - headers_no_content_type { - desc { - en: """List of HTTP headers (without content-type).""" - zh: """HTTP Headers 列表（无 content-type）。""" - } - label { - en: """headers_no_content_type""" - zh: """请求头(无 content-type)""" - } - } +enable.label: +"""enable""" - body { - desc { - en: """HTTP request body.""" - zh: """HTTP 请求体。""" - } - label { - en: """body""" - zh: """请求体""" - } - } - - request_timeout { - desc { - en: """Request timeout.""" - zh: """请求超时时间。""" - } - label { - en: """request_timeout""" - zh: """请求超时""" - } - } - -#==== authz_mnesia - -# only common fields(`enable` and `type`) - -#==== authz_mongo - - collection { - desc { - en: """`MongoDB` collection containing the authorization data.""" - zh: """`MongoDB` 授权数据集。""" - } - label { - en: """collection""" - zh: """数据集""" - } - } - - filter { - desc { - en: """Conditional expression that defines the filter condition in the query. +filter.desc: +"""Conditional expression that defines the filter condition in the query. Filter supports the following placeholders: - ${username}: Will be replaced at runtime with Username used by the client when connecting; - ${clientid}: Will be replaced at runtime with Client ID used by the client when connecting.""" - zh: """在查询中定义过滤条件的条件表达式。 -过滤器支持如下占位符： -- ${username}: 将在运行时被替换为客户端连接时使用的用户名 -- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符""" - } - label { - en: """Filter""" - zh: """过滤器""" - } - } -#==== authz_mysql +filter.label: +"""Filter""" -# `query`, is common field +headers.desc: +"""List of HTTP Headers.""" -#==== authz_pgsql +headers.label: +"""Headers""" -# `query`, is common field +headers_no_content_type.desc: +"""List of HTTP headers (without content-type).""" -#==== authz_redis +headers_no_content_type.label: +"""headers_no_content_type""" - cmd { - desc { - en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查询命令。""" - } - label { - en: """cmd""" - zh: """查询命令""" - } - } +method.desc: +"""HTTP method.""" -#==== common field for DBs (except mongodb and redis) +method.label: +"""method""" - query { - desc { - en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查询语句。""" - } - label { - en: """query""" - zh: """查询语句""" - } - } +position.desc: +"""Where to place the source.""" -#==== fields +position.label: +"""position""" + +query.desc: +"""Database query used to retrieve authorization data.""" + +query.label: +"""query""" + +request_timeout.desc: +"""Request timeout.""" + +request_timeout.label: +"""request_timeout""" + +rules.desc: +"""Authorization static file rules.""" + +rules.label: +"""rules""" + +type.desc: +"""Backend type.""" + +type.label: +"""type""" + +url.desc: +"""URL of the auth server.""" + +url.label: +"""url""" - position { - desc { - en: """Where to place the source.""" - zh: """认证数据源位置。""" - } - label { - en: """position""" - zh: """位置""" - } - } } diff --git a/rel/i18n/emqx_authz_api_settings.hocon b/rel/i18n/emqx_authz_api_settings.hocon index b44580b34..1840848bd 100644 --- a/rel/i18n/emqx_authz_api_settings.hocon +++ b/rel/i18n/emqx_authz_api_settings.hocon @@ -1,15 +1,9 @@ emqx_authz_api_settings { - authorization_settings_get { - desc { - en: """Get authorization settings""" - zh: """获取授权配置""" - } - } - authorization_settings_put { - desc { - en: """Update authorization settings""" - zh: """更新授权配置""" - } - } +authorization_settings_get.desc: +"""Get authorization settings""" + +authorization_settings_put.desc: +"""Update authorization settings""" + } diff --git a/rel/i18n/emqx_authz_api_sources.hocon b/rel/i18n/emqx_authz_api_sources.hocon index c5f0eaad4..5d2dda69e 100644 --- a/rel/i18n/emqx_authz_api_sources.hocon +++ b/rel/i18n/emqx_authz_api_sources.hocon @@ -1,116 +1,48 @@ emqx_authz_api_sources { - authorization_sources_get { - desc { - en: """List all authorization sources""" - zh: """列出所有授权数据源""" - } - } - authorization_sources_post { - desc { - en: """Add a new source""" - zh: """添加授权数据源""" - } - } +authorization_sources_get.desc: +"""List all authorization sources""" - authorization_sources_type_get { - desc { - en: """Get a authorization source""" - zh: """获取指定类型的授权数据源""" - } - } +authorization_sources_post.desc: +"""Add a new source""" - authorization_sources_type_put { - desc { - en: """Update source""" - zh: """更新指定类型的授权数据源""" - } - } +authorization_sources_type_delete.desc: +"""Delete source""" - authorization_sources_type_delete { - desc { - en: """Delete source""" - zh: """删除指定类型的授权数据源""" - } - } +authorization_sources_type_get.desc: +"""Get a authorization source""" - authorization_sources_type_status_get { - desc { - en: """Get a authorization source""" - zh: """获取指定授权数据源的状态""" - } - } +authorization_sources_type_move_post.desc: +"""Change the exection order of sources""" - authorization_sources_type_move_post { - desc { - en: """Change the exection order of sources""" - zh: """更新授权数据源的优先执行顺序""" - } - } +authorization_sources_type_put.desc: +"""Update source""" - sources { - desc { - en: """Authorization source""" - zh: """授权数据源列表""" - } - label { - en: """sources""" - zh: """数据源列表""" - } - } +authorization_sources_type_status_get.desc: +"""Get a authorization source""" - sources { - desc { - en: """Authorization sources""" - zh: """授权数据源列表""" - } - label { - en: """sources""" - zh: """数据源列表""" - } - } +source.desc: +"""Authorization source""" - source_config { - desc { - en: """Source config""" - zh: """数据源配置""" - } - label { - en: """source_config""" - zh: """数据源配置""" - } - } +source.label: +"""source""" - source { - desc { - en: """Authorization source""" - zh: """授权数据源""" - } - label { - en: """source""" - zh: """数据源""" - } - } +source_config.desc: +"""Source config""" - source_config { - desc { - en: """Source config""" - zh: """数据源配置""" - } - label { - en: """source_config""" - zh: """数据源配置""" - } - } +source_config.label: +"""source_config""" + +source_type.desc: +"""Authorization type""" + +source_type.label: +"""source_type""" + +sources.desc: +"""Authorization sources""" + +sources.label: +"""sources""" - source_type { - desc { - en: """Authorization type""" - zh: """数据源类型""" - } - label { - en: """source_type""" - zh: """数据源类型""" - } - } } diff --git a/rel/i18n/emqx_authz_schema.hocon b/rel/i18n/emqx_authz_schema.hocon index e48c64b89..9e5339b2c 100644 --- a/rel/i18n/emqx_authz_schema.hocon +++ b/rel/i18n/emqx_authz_schema.hocon @@ -1,7 +1,251 @@ emqx_authz_schema { - sources { - desc { - en: """Authorization data sources.
+ +deny.desc: +"""The number of authentication failures.""" + +deny.label: +"""The Number of Authentication Failures""" + +redis_sentinel.desc: +"""Authorization using a Redis Sentinel.""" + +redis_sentinel.label: +"""redis_sentinel""" + +rate.desc: +"""The rate of matched, times/second.""" + +rate.label: +"""Rate""" + +status.desc: +"""The status of the resource.""" + +status.label: +"""Status""" + +method.desc: +"""HTTP method.""" + +method.label: +"""method""" + +query.desc: +"""Database query used to retrieve authorization data.""" + +query.label: +"""query""" + +metrics_total.desc: +"""The total number of times the authorization rule was triggered.""" + +metrics_total.label: +"""The Total Number of Times the Authorization Rule was Triggered""" + +redis_cluster.desc: +"""Authorization using a Redis cluster.""" + +redis_cluster.label: +"""redis_cluster""" + +mysql.desc: +"""Authorization using a MySQL database.""" + +mysql.label: +"""mysql""" + +postgresql.desc: +"""Authorization using a PostgreSQL database.""" + +postgresql.label: +"""postgresql""" + +mongo_rs.desc: +"""Authorization using a MongoDB replica set.""" + +mongo_rs.label: +"""mongo_rs""" + +type.desc: +"""Backend type.""" + +type.label: +"""type""" + +mongo_sharded.desc: +"""Authorization using a sharded MongoDB cluster.""" + +mongo_sharded.label: +"""mongo_sharded""" + +body.desc: +"""HTTP request body.""" + +body.label: +"""Request Body""" + +url.desc: +"""URL of the auth server.""" + +url.label: +"""URL""" + +node.desc: +"""Node name.""" + +node.label: +"""Node Name.""" + +headers.desc: +"""List of HTTP Headers.""" + +headers.label: +"""Headers""" + +rate_last5m.desc: +"""The average rate of matched in the last 5 minutes, times/second.""" + +rate_last5m.label: +"""Rate in Last 5min""" + +headers_no_content_type.desc: +"""List of HTTP headers (without content-type).""" + +headers_no_content_type.label: +"""headers_no_content_type""" + +node_error.desc: +"""The error of node.""" + +node_error.label: +"""Error in Node""" + +mnesia.desc: +"""Authorization using a built-in database (mnesia).""" + +mnesia.label: +"""mnesia""" + +enable.desc: +"""Set to true or false to disable this ACL provider""" + +enable.label: +"""enable""" + +matched.desc: +"""Count of this resource is queried.""" + +matched.label: +"""Matched""" + +node_status.desc: +"""The status of the resource for each node.""" + +node_status.label: +"""Resource Status in Node""" + +rate_max.desc: +"""The max rate of matched, times/second.""" + +rate_max.label: +"""Max Rate""" + +filter.desc: +"""Conditional expression that defines the filter condition in the query. +Filter supports the following placeholders
+ - ${username}: Will be replaced at runtime with Username used by the client when connecting
+ - ${clientid}: Will be replaced at runtime with Client ID used by the client when connecting""" + +filter.label: +"""Filter""" + +path.desc: +"""Path to the file which contains the ACL rules. +If the file provisioned before starting EMQX node, +it can be placed anywhere as long as EMQX has read access to it. +That is, EMQX will treat it as read only. + +In case the rule-set is created or updated from EMQX Dashboard or HTTP API, +a new file will be created and placed in `authz` subdirectory inside EMQX's `data_dir`, +and the old file will not be used anymore.""" + +path.label: +"""path""" + +redis_single.desc: +"""Authorization using a single Redis instance.""" + +redis_single.label: +"""redis_single""" + +failed.desc: +"""Count of query failed.""" + +failed.label: +"""Failed""" + +metrics.desc: +"""The metrics of the resource.""" + +metrics.label: +"""Metrics""" + +authorization.desc: +"""Configuration related to the client authorization.""" + +authorization.label: +"""authorization""" + +collection.desc: +"""`MongoDB` collection containing the authorization data.""" + +collection.label: +"""collection""" + +mongo_single.desc: +"""Authorization using a single MongoDB instance.""" + +mongo_single.label: +"""mongo_single""" + +file.desc: +"""Authorization using a static file.""" + +file.label: +"""file""" + +http_post.desc: +"""Authorization using an external HTTP server (via POST requests).""" + +http_post.label: +"""http_post""" + +request_timeout.desc: +"""HTTP request timeout.""" + +request_timeout.label: +"""Request Timeout""" + +allow.desc: +"""The number of times the authentication was successful.""" + +allow.label: +"""The Number of Times the Authentication was Successful""" + +cmd.desc: +"""Database query used to retrieve authorization data.""" + +cmd.label: +"""cmd""" + +nomatch.desc: +"""The number of times that no authorization rules were matched.""" + +nomatch.label: +"""The Number of Times that no Authorization Rules were Matched""" + +sources.desc: +"""Authorization data sources.
An array of authorization (ACL) data providers. It is designed as an array, not a hash-map, so the sources can be ordered to form a chain of access controls.
@@ -19,525 +263,25 @@ NOTE: The source elements are identified by their 'type'. It is NOT allowed to configure two or more sources of the same type.""" - zh: """授权数据源。
-授权（ACL）数据源的列表。 -它被设计为一个数组，而不是一个散列映射， -所以可以作为链式访问控制。
+sources.label: +"""sources""" -当授权一个 'publish' 或 'subscribe' 行为时， -该配置列表中的所有数据源将按顺序进行检查。 -如果在某个客户端未找到时(使用 ClientID 或 Username)。 -将会移动到下一个数据源。直至得到 'allow' 或 'deny' 的结果。
+node_metrics.desc: +"""The metrics of the resource for each node.""" -如果在任何数据源中都未找到对应的客户端信息。 -配置的默认行为 ('authorization.no_match') 将生效。
+node_metrics.label: +"""Resource Metrics in Node""" -注意： -数据源使用 'type' 进行标识。 -使用同一类型的数据源多于一次不被允许。""" - } - label { - en: """sources""" - zh: """数据源""" - } - } +success.desc: +"""Count of query success.""" - authorization { - desc { - en: """Configuration related to the client authorization.""" - zh: """客户端授权相关配置""" - } - label { - en: """authorization""" - zh: """授权""" - } - } +success.label: +"""Success""" - enable { - desc { - en: """Set to true or false to disable this ACL provider""" - zh: """设为 true 或 false 以启用或禁用此访问控制数据源""" - } - label { - en: """enable""" - zh: """enable""" - } - } +http_get.desc: +"""Authorization using an external HTTP server (via GET requests).""" - type { - desc { - en: """Backend type.""" - zh: """数据后端类型""" - } - label { - en: """type""" - zh: """type""" - } - } +http_get.label: +"""http_get""" -#==== authz_file - - file { - desc { - en: """Authorization using a static file.""" - zh: """使用静态文件授权""" - } - label { - en: """file""" - zh: """文件""" - } - } - - path { - desc { - en: """Path to the file which contains the ACL rules. -If the file provisioned before starting EMQX node, -it can be placed anywhere as long as EMQX has read access to it. -That is, EMQX will treat it as read only. - -In case the rule-set is created or updated from EMQX Dashboard or HTTP API, -a new file will be created and placed in `authz` subdirectory inside EMQX's `data_dir`, -and the old file will not be used anymore.""" - zh: """包含 ACL 规则的文件路径。 -如果在启动 EMQX 节点前预先配置该路径， -那么可以将该文件置于任何 EMQX 可以访问到的位置。 - -如果从 EMQX Dashboard 或 HTTP API 创建或修改了规则集， -那么EMQX将会生成一个新的文件并将它存放在 `data_dir` 下的 `authz` 子目录中， -并从此弃用旧的文件。""" - } - label { - en: """path""" - zh: """path""" - } - } - -#==== authz_http - - http_get { - desc { - en: """Authorization using an external HTTP server (via GET requests).""" - zh: """使用外部 HTTP 服务器授权(GET 请求)。""" - } - label { - en: """http_get""" - zh: """http_get""" - } - } - - http_post { - desc { - en: """Authorization using an external HTTP server (via POST requests).""" - zh: """使用外部 HTTP 服务器授权(POST 请求)。""" - } - label { - en: """http_post""" - zh: """http_post""" - } - } - - method { - desc { - en: """HTTP method.""" - zh: """HTTP 请求方法""" - } - label { - en: """method""" - zh: """method""" - } - } - - url { - desc { - en: """URL of the auth server.""" - zh: """授权 HTTP 服务器地址。""" - } - label { - en: """URL""" - zh: """URL""" - } - } - - headers { - desc { - en: """List of HTTP Headers.""" - zh: """HTTP Headers 列表""" - } - label { - en: """Headers""" - zh: """请求头""" - } - } - - headers_no_content_type { - desc { - en: """List of HTTP headers (without content-type).""" - zh: """HTTP Headers 列表 (无 content-type) 。""" - } - label { - en: """headers_no_content_type""" - zh: """请求头(无 content-type)""" - } - } - - body { - desc { - en: """HTTP request body.""" - zh: """HTTP request body。""" - } - label { - en: """Request Body""" - zh: """Request Body""" - } - } - - request_timeout { - desc { - en: """HTTP request timeout.""" - zh: """HTTP 请求超时时长。""" - } - label { - en: """Request Timeout""" - zh: """请求超时时间""" - } - } - -#==== authz_mnesia - - mnesia { - desc { - en: """Authorization using a built-in database (mnesia).""" - zh: """使用内部数据库授权（mnesia）。""" - } - label { - en: """mnesia""" - zh: """mnesia""" - } - } - -#==== authz_mongo - - mongo_single { - desc { - en: """Authorization using a single MongoDB instance.""" - zh: """使用 MongoDB 授权（单实例）。""" - } - label { - en: """mongo_single""" - zh: """mongo_single""" - } - } - - mongo_rs { - desc { - en: """Authorization using a MongoDB replica set.""" - zh: """使用 MongoDB 授权（副本集模式）""" - } - label { - en: """mongo_rs""" - zh: """mongo_rs""" - } - } - - mongo_sharded { - desc { - en: """Authorization using a sharded MongoDB cluster.""" - zh: """使用 MongoDB 授权（分片集群模式）。""" - } - label { - en: """mongo_sharded""" - zh: """mongo_sharded""" - } - } - - collection { - desc { - en: """`MongoDB` collection containing the authorization data.""" - zh: """`MongoDB` 授权数据集。""" - } - label { - en: """collection""" - zh: """数据集""" - } - } - - filter { - desc { - en: """Conditional expression that defines the filter condition in the query. -Filter supports the following placeholders
- - ${username}: Will be replaced at runtime with Username used by the client when connecting
- - ${clientid}: Will be replaced at runtime with Client ID used by the client when connecting""" - zh: """在查询中定义过滤条件的条件表达式。 -过滤器支持如下占位符：
-- ${username}：将在运行时被替换为客户端连接时使用的用户名
-- ${clientid}：将在运行时被替换为客户端连接时使用的客户端标识符""" - } - label { - en: """Filter""" - zh: """过滤器""" - } - } - -#==== authz_mysql - - mysql { - desc { - en: """Authorization using a MySQL database.""" - zh: """使用 MySOL 数据库授权""" - } - label { - en: """mysql""" - zh: """mysql""" - } - } - -#==== authz_pgsql - - postgresql { - desc { - en: """Authorization using a PostgreSQL database.""" - zh: """使用 PostgreSQL 数据库授权""" - } - label { - en: """postgresql""" - zh: """postgresql""" - } - } - -#==== authz_redis - - redis_single { - desc { - en: """Authorization using a single Redis instance.""" - zh: """使用 Redis 授权（单实例）。""" - } - label { - en: """redis_single""" - zh: """redis_single""" - } - } - - redis_sentinel { - desc { - en: """Authorization using a Redis Sentinel.""" - zh: """使用 Redis 授权（哨兵模式）。""" - } - label { - en: """redis_sentinel""" - zh: """redis_sentinel""" - } - } - - redis_cluster { - desc { - en: """Authorization using a Redis cluster.""" - zh: """使用 Redis 授权（集群模式）。""" - } - label { - en: """redis_cluster""" - zh: """redis_cluster""" - } - } - - cmd { - desc { - en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查查询命令""" - } - label { - en: """cmd""" - zh: """查询命令""" - } - } - -#==== common field for DBs (except redis) - - query { - desc { - en: """Database query used to retrieve authorization data.""" - zh: """访问控制数据查询语句/查询命令。""" - } - label { - en: """query""" - zh: """查询语句""" - } - } - -#==== metrics field - - metrics { - desc { - en: """The metrics of the resource.""" - zh: """资源统计指标。""" - } - label: { - en: """Metrics""" - zh: """指标""" - } - } - - node_metrics { - desc { - en: """The metrics of the resource for each node.""" - zh: """每个节点上资源的统计指标。""" - } - label: { - en: """Resource Metrics in Node""" - zh: """节点资源指标""" - } - } - - status { - desc { - en: """The status of the resource.""" - zh: """资源状态。""" - } - label: { - en: """Status""" - zh: """状态""" - } - } - - node_status { - desc { - en: """The status of the resource for each node.""" - zh: """每个节点上资源的状态。""" - } - label: { - en: """Resource Status in Node""" - zh: """节点资源状态""" - } - } - - node_error { - desc { - en: """The error of node.""" - zh: """节点上产生的错误。""" - } - label: { - en: """Error in Node""" - zh: """节点产生的错误""" - } - } - - matched { - desc { - en: """Count of this resource is queried.""" - zh: """请求命中次数。""" - } - label: { - en: """Matched""" - zh: """已命中""" - } - } - - success { - desc { - en: """Count of query success.""" - zh: """请求成功次数。""" - } - label: { - en: """Success""" - zh: """成功""" - } - } - - failed { - desc { - en: """Count of query failed.""" - zh: """请求失败次数。""" - } - label: { - en: """Failed""" - zh: """失败""" - } - } - - rate { - desc { - en: """The rate of matched, times/second.""" - zh: """命中速率，单位：次/秒。""" - } - label: { - en: """Rate""" - zh: """速率""" - } - } - - rate_max { - desc { - en: """The max rate of matched, times/second.""" - zh: """最大命中速率，单位：次/秒。""" - } - label: { - en: """Max Rate""" - zh: """最大速率""" - } - } - - rate_last5m { - desc { - en: """The average rate of matched in the last 5 minutes, times/second.""" - zh: """5分钟内平均命中速率，单位：次/秒。""" - } - label: { - en: """Rate in Last 5min""" - zh: """5分钟内速率""" - } - } - - node { - desc { - en: """Node name.""" - zh: """节点名称。""" - } - label: { - en: """Node Name.""" - zh: """节点名称。""" - } - } - - metrics_total { - desc { - en: """The total number of times the authorization rule was triggered.""" - zh: """授权实例被触发的总次数。""" - } - label: { - en: """The Total Number of Times the Authorization Rule was Triggered""" - zh: """授权实例被触发的总次数""" - } - } - - nomatch { - desc { - en: """The number of times that no authorization rules were matched.""" - zh: """没有匹配到任何授权规则的次数。""" - } - label: { - en: """The Number of Times that no Authorization Rules were Matched""" - zh: """没有匹配到任何授权规则的次数""" - } - } - - allow { - desc { - en: """The number of times the authentication was successful.""" - zh: """授权成功的次数。""" - } - label: { - en: """The Number of Times the Authentication was Successful""" - zh: """授权成功次数""" - } - } - - deny { - desc { - en: """The number of authentication failures.""" - zh: """授权失败的次数。""" - } - label: { - en: """The Number of Authentication Failures""" - zh: """授权失败次数""" - } - } } diff --git a/rel/i18n/emqx_auto_subscribe_api.hocon b/rel/i18n/emqx_auto_subscribe_api.hocon index b8d043e9a..df8e87e1a 100644 --- a/rel/i18n/emqx_auto_subscribe_api.hocon +++ b/rel/i18n/emqx_auto_subscribe_api.hocon @@ -1,23 +1,12 @@ emqx_auto_subscribe_api { - list_auto_subscribe_api { - desc { - en: """Get auto subscribe topic list""" - zh: """获取自动订阅主题列表""" - } - } - update_auto_subscribe_api { - desc { - en: """Update auto subscribe topic list""" - zh: """更新自动订阅主题列表""" - } - } +list_auto_subscribe_api.desc: +"""Get auto subscribe topic list""" - update_auto_subscribe_api_response409 { - desc { - en: """Auto Subscribe topics max limit""" - zh: """超出自定订阅主题列表长度限制""" - } - } +update_auto_subscribe_api.desc: +"""Update auto subscribe topic list""" + +update_auto_subscribe_api_response409.desc: +"""Auto Subscribe topics max limit""" } diff --git a/rel/i18n/emqx_auto_subscribe_schema.hocon b/rel/i18n/emqx_auto_subscribe_schema.hocon index ae8c40303..e26ed2546 100644 --- a/rel/i18n/emqx_auto_subscribe_schema.hocon +++ b/rel/i18n/emqx_auto_subscribe_schema.hocon @@ -1,85 +1,48 @@ emqx_auto_subscribe_schema { - auto_subscribe { - desc { - en: """After the device logs in successfully, the subscription is automatically completed for the device through the pre-defined subscription representation. Supports the use of placeholders.""" - zh: """设备登录成功之后，通过预设的订阅表示符，为设备自动完成订阅。支持使用占位符。""" - } - label { - en: """Auto Subscribe""" - zh: """自动订阅""" - } - } - topic { - desc { - en: """Topic name, placeholders are supported. For example: client/${clientid}/username/${username}/host/${host}/port/${port} -Required field, and cannot be empty string""" - zh: """订阅标识符，支持使用占位符，例如 client/${clientid}/username/${username}/host/${host}/port/${port} -必填，且不可为空字符串""" - } - label { - en: """Topic""" - zh: """订阅标识符""" - } - } +auto_subscribe.desc: +"""After the device logs in successfully, the subscription is automatically completed for the device through the pre-defined subscription representation. Supports the use of placeholders.""" - qos { - desc { - en: """Default value 0. Quality of service. +auto_subscribe.label: +"""Auto Subscribe""" + +nl.desc: +"""Default value 0. +MQTT v3.1.1： if you subscribe to the topic published by yourself, you will receive all messages that you published. +MQTT v5: if you set this option as 1 when subscribing, the server will not forward the message you published to you.""" + +nl.label: +"""No Local""" + +qos.desc: +"""Default value 0. Quality of service. At most once (0) At least once (1) Exactly once (2)""" - zh: """缺省值为 0，服务质量， -QoS 0：消息最多传递一次，如果当时客户端不可用，则会丢失该消息。 -QoS 1：消息传递至少 1 次。 -QoS 2：消息仅传送一次。""" - } - label { - en: """Quality of Service""" - zh: """服务质量""" - } - } - rh { - desc { - en: """Default value 0. This option is used to specify whether the server forwards the retained message to the client when establishing a subscription. +qos.label: +"""Quality of Service""" + +rap.desc: +"""Default value 0. This option is used to specify whether the server retains the RETAIN mark when forwarding messages to the client, and this option does not affect the RETAIN mark in the retained message. Therefore, when the option Retain As Publish is set to 0, the client will directly distinguish whether this is a normal forwarded message or a retained message according to the RETAIN mark in the message, instead of judging whether this message is the first received after subscribing(the forwarded message may be sent before the retained message, which depends on the specific implementation of different brokers).""" + +rap.label: +"""Retain As Publish""" + +rh.desc: +"""Default value 0. This option is used to specify whether the server forwards the retained message to the client when establishing a subscription. Retain Handling is equal to 0, as long as the client successfully subscribes, the server will send the retained message. Retain Handling is equal to 1, if the client successfully subscribes and this subscription does not exist previously, the server sends the retained message. After all, sometimes the client re-initiate the subscription just to change the QoS, but it does not mean that it wants to receive the reserved messages again. Retain Handling is equal to 2, even if the client successfully subscribes, the server does not send the retained message.""" - zh: """指定订阅建立时服务端是否向客户端发送保留消息， -可选值 0：只要客户端订阅成功，服务端就发送保留消息。 -可选值 1：客户端订阅成功且该订阅此前不存在，服务端才发送保留消息。毕竟有些时候客户端重新发起订阅可能只是为了改变一下 QoS，并不意味着它想再次接收保留消息。 -可选值 2：即便客户订阅成功，服务端也不会发送保留消息。""" - } - label { - en: """Retain Handling""" - zh: """Retain Handling""" - } - } - rap { - desc { - en: """Default value 0. This option is used to specify whether the server retains the RETAIN mark when forwarding messages to the client, and this option does not affect the RETAIN mark in the retained message. Therefore, when the option Retain As Publish is set to 0, the client will directly distinguish whether this is a normal forwarded message or a retained message according to the RETAIN mark in the message, instead of judging whether this message is the first received after subscribing(the forwarded message may be sent before the retained message, which depends on the specific implementation of different brokers).""" - zh: """缺省值为 0，这一选项用来指定服务端向客户端转发消息时是否要保留其中的 RETAIN 标识，注意这一选项不会影响保留消息中的 RETAIN 标识。因此当 Retain As Publish 选项被设置为 0 时，客户端直接依靠消息中的 RETAIN 标识来区分这是一个正常的转发消息还是一个保留消息，而不是去判断消息是否是自己订阅后收到的第一个消息（转发消息甚至可能会先于保留消息被发送，视不同 Broker 的具体实现而定）。""" - } - label { - en: """Retain As Publish""" - zh: """Retain As Publish""" - } - } +rh.label: +"""Retain Handling""" + +topic.desc: +"""Topic name, placeholders are supported. For example: client/${clientid}/username/${username}/host/${host}/port/${port} +Required field, and cannot be empty string""" + +topic.label: +"""Topic""" - nl { - desc { - en: """Default value 0. -MQTT v3.1.1： if you subscribe to the topic published by yourself, you will receive all messages that you published. -MQTT v5: if you set this option as 1 when subscribing, the server will not forward the message you published to you.""" - zh: """缺省值为0， -MQTT v3.1.1：如果设备订阅了自己发布消息的主题，那么将收到自己发布的所有消息。 -MQTT v5：如果设备在订阅时将此选项设置为 1，那么服务端将不会向设备转发自己发布的消息""" - } - label { - en: """No Local""" - zh: """No Local""" - } - } } diff --git a/rel/i18n/emqx_bridge_api.hocon b/rel/i18n/emqx_bridge_api.hocon index 66960619a..8b7950cdc 100644 --- a/rel/i18n/emqx_bridge_api.hocon +++ b/rel/i18n/emqx_bridge_api.hocon @@ -1,180 +1,100 @@ emqx_bridge_api { - desc_param_path_operation_cluster { - desc { - en: """Operations can be one of: stop, restart""" - zh: """集群可用操作：停止、重新启动""" - } - label: { - en: "Cluster Operation" - zh: "集群可用操作" - } - } +desc_api1.desc: +"""List all created bridges""" - desc_param_path_operation_on_node { - desc { - en: """Operations can be one of: stop, restart""" - zh: """节点可用操作：停止、重新启动""" - } - label: { - en: "Node Operation " - zh: "节点可用操作" - } - } +desc_api1.label: +"""List All Bridges""" - desc_param_path_node { - desc { - en: """The node name, e.g. emqx@127.0.0.1""" - zh: """节点名，比如 emqx@127.0.0.1""" - } - label: { - en: "The node name" - zh: "节点名" - } - } +desc_api2.desc: +"""Create a new bridge by type and name""" - desc_param_path_id { - desc { - en: """The bridge Id. Must be of format {type}:{name}""" - zh: """Bridge ID ， 格式为 {type}:{name}""" - } - label: { - en: "Bridge ID" - zh: "Bridge ID" - } - } +desc_api2.label: +"""Create Bridge""" - desc_param_path_enable { - desc { - en: """Whether to enable this bridge""" - zh: """是否启用桥接""" - } - label: { - en: "Enable bridge" - zh: "启用桥接" - } - } - desc_api1 { - desc { - en: """List all created bridges""" - zh: """列出所有 Bridge""" - } - label: { - en: "List All Bridges" - zh: "列出所有 Bridge" - } - } +desc_api3.desc: +"""Get a bridge by Id""" - desc_api2 { - desc { - en: """Create a new bridge by type and name""" - zh: """通过类型和名字创建 Bridge""" - } - label: { - en: "Create Bridge" - zh: "创建 Bridge" - } - } +desc_api3.label: +"""Get Bridge""" - desc_api3 { - desc { - en: """Get a bridge by Id""" - zh: """通过 ID 获取 Bridge""" - } - label: { - en: "Get Bridge" - zh: "获取 Bridge" - } - } +desc_api4.desc: +"""Update a bridge by Id""" - desc_api4 { - desc { - en: """Update a bridge by Id""" - zh: """通过 ID 更新 Bridge""" - } - label: { - en: "Update Bridge" - zh: "更新 Bridge" - } - } +desc_api4.label: +"""Update Bridge""" - desc_api5 { - desc { - en: """Delete a bridge by Id""" - zh: """通过 ID 删除 Bridge""" - } - label: { - en: "Delete Bridge" - zh: "删除 Bridge" - } - } +desc_api5.desc: +"""Delete a bridge by Id""" - desc_api6 { - desc { - en: """Reset a bridge metrics by Id""" - zh: """通过 ID 重置 Bridge 的计数""" - } - label: { - en: "Reset Bridge Metrics" - zh: "重置 Bridge 计数" - } - } +desc_api5.label: +"""Delete Bridge""" - desc_api7 { - desc { - en: """Stop/Restart bridges on all nodes in the cluster.""" - zh: """停止或启用所有节点上的桥接""" - } - label: { - en: "Cluster Bridge Operate" - zh: "集群 Bridge 操作" - } - } +desc_api6.desc: +"""Reset a bridge metrics by Id""" - desc_api8 { - desc { - en: """Stop/Restart bridges on a specific node.""" - zh: """在某个节点上停止/重新启动 Bridge。""" - } - label: { - en: "Node Bridge Operate" - zh: "单节点 Bridge 操作" - } - } +desc_api6.label: +"""Reset Bridge Metrics""" - desc_api9 { - desc { - en: """Test creating a new bridge by given ID
+desc_api7.desc: +"""Stop/Restart bridges on all nodes in the cluster.""" + +desc_api7.label: +"""Cluster Bridge Operate""" + +desc_api8.desc: +"""Stop/Restart bridges on a specific node.""" + +desc_api8.label: +"""Node Bridge Operate""" + +desc_api9.desc: +"""Test creating a new bridge by given ID
The ID must be of format '{type}:{name}'""" - zh: """通过给定的 ID 测试创建一个新的桥接。
-ID 的格式必须为 ’{type}:{name}”""" - } - label: { - en: "Test Bridge Creation" - zh: "测试桥接创建" - } - } - desc_bridge_metrics { - desc { - en: """Get bridge metrics by Id""" - zh: """通过 Id 来获取桥接的指标信息""" - } - label: { - en: "Get Bridge Metrics" - zh: "获取桥接的指标" - } - } +desc_api9.label: +"""Test Bridge Creation""" - desc_enable_bridge { - desc { - en: """Enable or Disable bridges on all nodes in the cluster.""" - zh: """启用或禁用所有节点上的桥接""" - } - label: { - en: "Cluster Bridge Enable" - zh: "是否启用集群内的桥接" - } - } +desc_bridge_metrics.desc: +"""Get bridge metrics by Id""" + +desc_bridge_metrics.label: +"""Get Bridge Metrics""" + +desc_enable_bridge.desc: +"""Enable or Disable bridges on all nodes in the cluster.""" + +desc_enable_bridge.label: +"""Cluster Bridge Enable""" + +desc_param_path_enable.desc: +"""Whether to enable this bridge""" + +desc_param_path_enable.label: +"""Enable bridge""" + +desc_param_path_id.desc: +"""The bridge Id. Must be of format {type}:{name}""" + +desc_param_path_id.label: +"""Bridge ID""" + +desc_param_path_node.desc: +"""The node name, e.g. emqx@127.0.0.1""" + +desc_param_path_node.label: +"""The node name""" + +desc_param_path_operation_cluster.desc: +"""Operations can be one of: stop, restart""" + +desc_param_path_operation_cluster.label: +"""Cluster Operation""" + +desc_param_path_operation_on_node.desc: +"""Operations can be one of: stop, restart""" + +desc_param_path_operation_on_node.label: +"""Node Operation """ } diff --git a/rel/i18n/emqx_bridge_kafka.hocon b/rel/i18n/emqx_bridge_kafka.hocon index 2f1811269..ef2e27972 100644 --- a/rel/i18n/emqx_bridge_kafka.hocon +++ b/rel/i18n/emqx_bridge_kafka.hocon @@ -1,667 +1,361 @@ emqx_bridge_kafka { - config_enable { - desc { - en: "Enable (true) or disable (false) this Kafka bridge." - zh: "启用（true）或停用该（false）Kafka 数据桥接。" - } - label { - en: "Enable or Disable" - zh: "启用或停用" - } - } - desc_config { - desc { - en: """Configuration for a Kafka bridge.""" - zh: """Kafka 桥接配置""" - } - label { - en: "Kafka Bridge Configuration" - zh: "Kafka 桥接配置" - } - } - desc_type { - desc { - en: """The Bridge Type""" - zh: """桥接类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } - kafka_producer { - desc { - en: "Kafka Producer configuration." - zh: "Kafka Producer 配置。" - } - label { - en: "Kafka Producer" - zh: "Kafka Producer" - } - } - producer_opts { - desc { - en: "Local MQTT data source and Kafka bridge configs." - zh: "本地 MQTT 数据源和 Kafka 桥接的配置。" - } - label { - en: "MQTT to Kafka" - zh: "MQTT 到 Kafka" - } - } - mqtt_topic { - desc { - en: "MQTT topic or topic filter as data source (bridge input). If rule action is used as data source, this config should be left empty, otherwise messages will be duplicated in Kafka." - zh: "MQTT 主题数据源由桥接指定，或留空由规则动作指定。" - } - label { - en: "Source MQTT Topic" - zh: "源 MQTT 主题" - } - } - producer_kafka_opts { - desc { - en: "Kafka producer configs." - zh: "Kafka 生产者参数。" - } - label { - en: "Kafka Producer" - zh: "生产者参数" - } - } - bootstrap_hosts { - desc { - en: "A comma separated list of Kafka host[:port] endpoints to bootstrap the client. Default port number is 9092." - zh: "用逗号分隔的 host[:port] 主机列表。默认端口号为 9092。" - } - label { - en: "Bootstrap Hosts" - zh: "主机列表" - } - } - connect_timeout { - desc { - en: "Maximum wait time for TCP connection establishment (including authentication time if enabled)." - zh: "建立 TCP 连接时的最大等待时长（若启用认证，这个等待时长也包含完成认证所需时间）。" - } - label { - en: "Connect Timeout" - zh: "连接超时" - } - } - min_metadata_refresh_interval { - desc { - en: "Minimum time interval the client has to wait before refreshing Kafka broker and topic metadata. " - "Setting too small value may add extra load on Kafka." - zh: "刷新 Kafka broker 和 Kafka 主题元数据段最短时间间隔。设置太小可能会增加 Kafka 压力。" - } - label { - en: "Min Metadata Refresh Interval" - zh: "元数据刷新最小间隔" - } - } - metadata_request_timeout { - desc { - en: "Maximum wait time when fetching metadata from Kafka." - zh: "刷新元数据时最大等待时长。" - } - label { - en: "Metadata Request Timeout" - zh: "元数据请求超时" - } - } - authentication { - desc { - en: "Authentication configs." - zh: "认证参数。" - } - label { - en: "Authentication" - zh: "认证" - } - } - socket_opts { - desc { - en: "Extra socket options." - zh: "更多 Socket 参数设置。" - } - label { - en: "Socket Options" - zh: "Socket 参数" - } - } - auth_sasl_mechanism { - desc { - en: "SASL authentication mechanism." - zh: "SASL 认证方法名称。" - } - label { - en: "Mechanism" - zh: "认证方法" - } - } - auth_sasl_username { - desc { - en: "SASL authentication username." - zh: "SASL 认证的用户名。" - } - label { - en: "Username" - zh: "用户名" - } - } - auth_sasl_password { - desc { - en: "SASL authentication password." - zh: "SASL 认证的密码。" - } - label { - en: "Password" - zh: "密码" - } - } - auth_kerberos_principal { - desc { - en: "SASL GSSAPI authentication Kerberos principal. " - "For example client_name@MY.KERBEROS.REALM.MYDOMAIN.COM, " - "NOTE: The realm in use has to be configured in /etc/krb5.conf in EMQX nodes." - zh: "SASL GSSAPI 认证方法的 Kerberos principal，" - "例如 client_name@MY.KERBEROS.REALM.MYDOMAIN.COM" - "注意：这里使用的 realm 需要配置在 EMQX 服务器的 /etc/krb5.conf 中" - } - label { - en: "Kerberos Principal" - zh: "Kerberos Principal" - } - } - auth_kerberos_keytab_file { - desc { - en: "SASL GSSAPI authentication Kerberos keytab file path. " - "NOTE: This file has to be placed in EMQX nodes, and the EMQX service runner user requires read permission." - zh: "SASL GSSAPI 认证方法的 Kerberos keytab 文件。" - "注意：该文件需要上传到 EMQX 服务器中，且运行 EMQX 服务的系统账户需要有读取权限。" - } - label { - en: "Kerberos keytab file" - zh: "Kerberos keytab 文件" - } - } - socket_send_buffer { - desc { - en: "Fine tune the socket send buffer. The default value is tuned for high throughput." - zh: "TCP socket 的发送缓存调优。默认值是针对高吞吐量的一个推荐值。" - } - label { - en: "Socket Send Buffer Size" - zh: "Socket 发送缓存大小" - } - } - socket_receive_buffer { - desc { - en: "Fine tune the socket receive buffer. The default value is tuned for high throughput." - zh: "TCP socket 的收包缓存调优。默认值是针对高吞吐量的一个推荐值。" - } - label { - en: "Socket Receive Buffer Size" - zh: "Socket 收包缓存大小" - } - } - # hidden - socket_nodelay { - desc { - en: "When set to 'true', TCP buffer is sent as soon as possible. " - "Otherwise, the OS kernel may buffer small TCP packets for a while (40 ms by default)." - zh: "设置‘true’让系统内核立即发送。否则当需要发送的内容很少时，可能会有一定延迟（默认 40 毫秒）。" - } - label { - en: "No Delay" - zh: "是否关闭延迟发送" - } - } - kafka_topic { - desc { - en: "Kafka topic name" - zh: "Kafka 主题名称" - } - label { - en: "Kafka Topic Name" - zh: "Kafka 主题名称" - } - } - kafka_message { - desc { - en: "Template to render a Kafka message." - zh: "用于生成 Kafka 消息的模版。" - } - label { - en: "Kafka Message Template" - zh: "Kafka 消息模版" - } - } - kafka_message_key { - desc { - en: "Template to render Kafka message key. " - "If the template is rendered into a NULL value (i.e. there is no such data field in Rule Engine context) " - "then Kafka's NULL (but not empty string) is used." - zh: "生成 Kafka 消息 Key 的模版。如果模版生成后为空值，" - "则会使用 Kafka 的 NULL ，而非空字符串。" - } - label { - en: "Message Key" - zh: "消息的 Key" - } - } - kafka_message_value { - desc { - en: "Template to render Kafka message value. " - "If the template is rendered " - "into a NULL value (i.e. there is no such data field in Rule Engine context) " - "then Kafka's NULL (but not empty string) is used." - zh: "生成 Kafka 消息 Value 的模版。如果模版生成后为空值，" - "则会使用 Kafka 的 NULL，而非空字符串。" - } - label { - en: "Message Value" - zh: "消息的 Value" - } - } - kafka_message_timestamp { - desc { - en: "Which timestamp to use. " - "The timestamp is expected to be a millisecond precision Unix epoch " - "which can be in string format, e.g. 1661326462115 or " - "'1661326462115'. " - "When the desired data field for this template is not found, " - "or if the found data is not a valid integer, " - "the current system timestamp will be used." - zh: "生成 Kafka 消息时间戳的模版。" - "该时间必需是一个整型数值（可以是字符串格式）例如 1661326462115 " - "或 '1661326462115'。" - "当所需的输入字段不存在，或不是一个整型时，" - "则会使用当前系统时间。" - } - label { - en: "Message Timestamp" - zh: "消息的时间戳" - } - } - max_batch_bytes { - desc { - en: "Maximum bytes to collect in a Kafka message batch. " - "Most of the Kafka brokers default to a limit of 1 MB batch size. " - "EMQX's default value is less than 1 MB in order to compensate " - "Kafka message encoding overheads (especially when each individual message is very small). " - "When a single message is over the limit, it is still sent (as a single element batch)." - zh: "最大消息批量字节数。" - "大多数 Kafka 环境的默认最低值是 1 MB，EMQX 的默认值比 1 MB 更小是因为需要" - "补偿 Kafka 消息编码所需要的额外字节（尤其是当每条消息都很小的情况下）。" - "当单个消息的大小超过该限制时，它仍然会被发送，（相当于该批量中只有单个消息）。" - } - label { - en: "Max Batch Bytes" - zh: "最大批量字节数" - } - } - compression { - desc { - en: "Compression method." - zh: "压缩方法。" - } - label { - en: "Compression" - zh: "压缩" - } - } - partition_strategy { - desc { - en: "Partition strategy is to tell the producer how to dispatch messages to Kafka partitions.\n\n" - "random: Randomly pick a partition for each message\n" - "key_dispatch: Hash Kafka message key to a partition number" - zh: "设置消息发布时应该如何选择 Kafka 分区。\n\n" - "random: 为每个消息随机选择一个分区。\n" - "key_dispatch: Hash Kafka message key to a partition number" - } - label { - en: "Partition Strategy" - zh: "分区选择策略" - } - } - required_acks { - desc { - en: "Required acknowledgements for Kafka partition leader to wait for its followers " - "before it sends back the acknowledgement to EMQX Kafka producer\n\n" - "all_isr: Require all in-sync replicas to acknowledge.\n" - "leader_only: Require only the partition-leader's acknowledgement.\n" - "none: No need for Kafka to acknowledge at all." - zh: "设置 Kafka leader 在返回给 EMQX 确认之前需要等待多少个 follower 的确认。\n\n" - "all_isr: 需要所有的在线复制者都确认。\n" - "leader_only: 仅需要分区 leader 确认。\n" - "none: 无需 Kafka 回复任何确认。" - } - label { - en: "Required Acks" - zh: "Kafka 确认数量" - } - } - partition_count_refresh_interval { - desc { - en: "The time interval for Kafka producer to discover increased number of partitions.\n" - "After the number of partitions is increased in Kafka, EMQX will start taking the \n" - "discovered partitions into account when dispatching messages per partition_strategy." - zh: "配置 Kafka 刷新分区数量的时间间隔。\n" - "EMQX 发现 Kafka 分区数量增加后，会开始按 partition_strategy 配置，把消息发送到新的分区中。" - } - label { - en: "Partition Count Refresh Interval" - zh: "分区数量刷新间隔" - } - } - max_inflight { - desc { - en: "Maximum number of batches allowed for Kafka producer (per-partition) to send before receiving acknowledgement from Kafka. " - "Greater value typically means better throughput. However, there can be a risk of message reordering when this " - "value is greater than 1." - zh: "设置 Kafka 生产者（每个分区一个）在收到 Kafka 的确认前最多发送多少个请求（批量）。" - "调大这个值通常可以增加吞吐量，但是，当该值设置大于 1 时存在消息乱序的风险。" - } - label { - en: "Max Inflight" - zh: "飞行窗口" - } - } - producer_buffer { - desc { - en: "Configure producer message buffer.\n\n" - "Tell Kafka producer how to buffer messages when EMQX has more messages to send than " - "Kafka can keep up, or when Kafka is down." - zh: "配置消息缓存的相关参数。\n\n" - "当 EMQX 需要发送的消息超过 Kafka 处理能力，或者当 Kafka 临时下线时，EMQX 内部会将消息缓存起来。" - } - label { - en: "Message Buffer" - zh: "消息缓存" - } - } - buffer_mode { - desc { - en: "Message buffer mode.\n\n" - "memory: Buffer all messages in memory. The messages will be lost in case of EMQX node restart\n" - "disk: Buffer all messages on disk. The messages on disk are able to survive EMQX node restart.\n" - "hybrid: Buffer message in memory first, when up to certain limit " - "(see segment_bytes config for more information), then start offloading " - "messages to disk, Like memory mode, the messages will be lost in case of " - "EMQX node restart." - zh: "消息缓存模式。\n" - "memory: 所有的消息都缓存在内存里。如果 EMQX 服务重启，缓存的消息会丢失。\n" - "disk: 缓存到磁盘上。EMQX 重启后会继续发送重启前未发送完成的消息。\n" - "hybrid: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" - "（配置项 segment_bytes 描述了该限制）后，后续的消息会缓存到磁盘上。" - "与 memory 模式一样，如果 EMQX 服务重启，缓存的消息会丢失。" - } - label { - en: "Buffer Mode" - zh: "缓存模式" - } - } - buffer_per_partition_limit { - desc { - en: "Number of bytes allowed to buffer for each Kafka partition. " - "When this limit is exceeded, old messages will be dropped in a trade for credits " - "for new messages to be buffered." - zh: "为每个 Kafka 分区设置的最大缓存字节数。当超过这个上限之后，老的消息会被丢弃，" - "为新的消息腾出空间。" - } - label { - en: "Per-partition Buffer Limit" - zh: "Kafka 分区缓存上限" - } - } - buffer_segment_bytes { - desc { - en: "Applicable when buffer mode is set to disk or hybrid.\n" - "This value is to specify the size of each on-disk buffer file." - zh: "当缓存模式是 disk 或 hybrid 时适用。" - "该配置用于指定缓存到磁盘上的文件的大小。" - } - label { - en: "Segment File Bytes" - zh: "缓存文件大小" - } - } - buffer_memory_overload_protection { - desc { - en: "Applicable when buffer mode is set to memory\n" - "EMQX will drop old buffered messages under high memory pressure. " - "The high memory threshold is defined in config sysmon.os.sysmem_high_watermark. " - "NOTE: This config only works on Linux." - zh: "缓存模式是 memory 或 hybrid 时适用。" - "当系统处于高内存压力时，从队列中丢弃旧的消息以减缓内存增长。" - "内存压力值由配置项 sysmon.os.sysmem_high_watermark 决定。" - "注意，该配置仅在 Linux 系统中有效。" - } - label { - en: "Memory Overload Protection" - zh: "内存过载保护" - } - } - auth_username_password { - desc { - en: "Username/password based authentication." - zh: "基于用户名密码的认证。" - } - label { - en: "Username/password Auth" - zh: "用户名密码认证" - } - } - auth_gssapi_kerberos { - desc { - en: "Use GSSAPI/Kerberos authentication." - zh: "使用 GSSAPI/Kerberos 认证。" - } - label { - en: "GSSAPI/Kerberos" - zh: "GSSAPI/Kerberos" - } - } - kafka_consumer { - desc { - en: "Kafka Consumer configuration." - zh: "Kafka 消费者配置。" - } - label { - en: "Kafka Consumer" - zh: "Kafka 消费者" - } - } - consumer_opts { - desc { - en: "Local MQTT publish and Kafka consumer configs." - zh: "本地 MQTT 转发 和 Kafka 消费者配置。" - } - label { - en: "MQTT to Kafka" - zh: "MQTT 到 Kafka" - } - } - consumer_kafka_opts { - desc { - en: "Kafka consumer configs." - zh: "Kafka消费者配置。" - } - label { - en: "Kafka Consumer" - zh: "Kafka 消费者" - } - } - consumer_mqtt_opts { - desc { - en: "Local MQTT message publish." - zh: "本地 MQTT 消息转发。" - } - label { - en: "MQTT publish" - zh: "MQTT 转发" - } - } - consumer_mqtt_topic { - desc { - en: "Local topic to which consumed Kafka messages should be published to." - zh: "设置 Kafka 消息向哪个本地 MQTT 主题转发消息。" - } - label { - en: "MQTT Topic" - zh: "MQTT主题" - } - } - consumer_mqtt_qos { - desc { - en: "MQTT QoS used to publish messages consumed from Kafka." - zh: "转发 MQTT 消息时使用的 QoS。" - } - label { - en: "QoS" - zh: "QoS" - } - } - consumer_mqtt_payload { - desc { - en: "The template for transforming the incoming Kafka message." - " By default, it will use JSON format to serialize" - " inputs from the Kafka message. Such fields are:\n" - "headers: an object containing string key-value pairs.\n" - "key: Kafka message key (uses the chosen key encoding).\n" - "offset: offset for the message.\n" - "topic: Kafka topic.\n" - "ts: message timestamp.\n" - "ts_type: message timestamp type, which is one of" - " create, append or undefined.\n" - "value: Kafka message value (uses the chosen value encoding)." - zh: "用于转换收到的 Kafka 消息的模板。 " - "默认情况下，它将使用 JSON 格式来序列化来自 Kafka 的所有字段。 " - "这些字段包括：" - "headers：一个包含字符串键值对的 JSON 对象。\n" - "key：Kafka 消息的键（使用选择的编码方式编码）。\n" - "offset：消息的偏移量。\n" - "topic：Kafka 主题。\n" - "ts: 消息的时间戳。\n" - "ts_type：消息的时间戳类型，值可能是：" - " create， append 或 undefined。\n" - "value: Kafka 消息值（使用选择的编码方式编码）。" +connect_timeout.desc: +"""Maximum wait time for TCP connection establishment (including authentication time if enabled).""" + +connect_timeout.label: +"""Connect Timeout""" + +producer_opts.desc: +"""Local MQTT data source and Kafka bridge configs.""" + +producer_opts.label: +"""MQTT to Kafka""" + +min_metadata_refresh_interval.desc: +"""Minimum time interval the client has to wait before refreshing Kafka broker and topic metadata. Setting too small value may add extra load on Kafka.""" + +min_metadata_refresh_interval.label: +"""Min Metadata Refresh Interval""" + +kafka_producer.desc: +"""Kafka Producer configuration.""" + +kafka_producer.label: +"""Kafka Producer""" + +producer_buffer.desc: +"""Configure producer message buffer. + +Tell Kafka producer how to buffer messages when EMQX has more messages to send than Kafka can keep up, or when Kafka is down.""" + +producer_buffer.label: +"""Message Buffer""" + +socket_send_buffer.desc: +"""Fine tune the socket send buffer. The default value is tuned for high throughput.""" + +socket_send_buffer.label: +"""Socket Send Buffer Size""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +consumer_offset_commit_interval_seconds.desc: +"""Defines the time interval between two offset commit requests sent for each consumer group.""" + +consumer_offset_commit_interval_seconds.label: +"""Offset Commit Interval""" + +consumer_max_batch_bytes.desc: +"""Set how many bytes to pull from Kafka in each fetch request. Please note that if the configured value is smaller than the message size in Kafka, it may negatively impact the fetch performance.""" + +consumer_max_batch_bytes.label: +"""Fetch Bytes""" + +socket_receive_buffer.desc: +"""Fine tune the socket receive buffer. The default value is tuned for high throughput.""" + +socket_receive_buffer.label: +"""Socket Receive Buffer Size""" + +consumer_topic_mapping.desc: +"""Defines the mapping between Kafka topics and MQTT topics. Must contain at least one item.""" + +consumer_topic_mapping.label: +"""Topic Mapping""" + +producer_kafka_opts.desc: +"""Kafka producer configs.""" + +producer_kafka_opts.label: +"""Kafka Producer""" + +kafka_topic.desc: +"""Kafka topic name""" + +kafka_topic.label: +"""Kafka Topic Name""" + +consumer_kafka_topic.desc: +"""Kafka topic to consume from.""" + +consumer_kafka_topic.label: +"""Kafka Topic""" + +auth_username_password.desc: +"""Username/password based authentication.""" + +auth_username_password.label: +"""Username/password Auth""" + +auth_sasl_password.desc: +"""SASL authentication password.""" + +auth_sasl_password.label: +"""Password""" + +kafka_message_timestamp.desc: +"""Which timestamp to use. The timestamp is expected to be a millisecond precision Unix epoch which can be in string format, e.g. 1661326462115 or '1661326462115'. When the desired data field for this template is not found, or if the found data is not a valid integer, the current system timestamp will be used.""" + +kafka_message_timestamp.label: +"""Message Timestamp""" + +buffer_mode.desc: +"""Message buffer mode. + +memory: Buffer all messages in memory. The messages will be lost in case of EMQX node restart +disk: Buffer all messages on disk. The messages on disk are able to survive EMQX node restart. +hybrid: Buffer message in memory first, when up to certain limit (see segment_bytes config for more information), then start offloading messages to disk, Like memory mode, the messages will be lost in case of EMQX node restart.""" + +buffer_mode.label: +"""Buffer Mode""" + +consumer_mqtt_qos.desc: +"""MQTT QoS used to publish messages consumed from Kafka.""" + +consumer_mqtt_qos.label: +"""QoS""" + +consumer_key_encoding_mode.desc: +"""Defines how the key from the Kafka message is encoded before being forwarded via MQTT. +none Uses the key from the Kafka message unchanged. Note: in this case, the key must be a valid UTF-8 string. +base64 Uses base-64 encoding on the received key.""" + +consumer_key_encoding_mode.label: +"""Key Encoding Mode""" + +auth_gssapi_kerberos.desc: +"""Use GSSAPI/Kerberos authentication.""" + +auth_gssapi_kerberos.label: +"""GSSAPI/Kerberos""" + +consumer_mqtt_opts.desc: +"""Local MQTT message publish.""" + +consumer_mqtt_opts.label: +"""MQTT publish""" + +auth_kerberos_principal.desc: +"""SASL GSSAPI authentication Kerberos principal. For example client_name@MY.KERBEROS.REALM.MYDOMAIN.COM, NOTE: The realm in use has to be configured in /etc/krb5.conf in EMQX nodes.""" + +auth_kerberos_principal.label: +"""Kerberos Principal""" + +socket_opts.desc: +"""Extra socket options.""" + +socket_opts.label: +"""Socket Options""" + +consumer_mqtt_topic.desc: +"""Local topic to which consumed Kafka messages should be published to.""" + +consumer_mqtt_topic.label: +"""MQTT Topic""" + +consumer_offset_reset_policy.desc: +"""Defines from which offset a consumer should start fetching when there is no commit history or when the commit history becomes invalid.""" + +consumer_offset_reset_policy.label: +"""Offset Reset Policy""" + +partition_count_refresh_interval.desc: +"""The time interval for Kafka producer to discover increased number of partitions. +After the number of partitions is increased in Kafka, EMQX will start taking the +discovered partitions into account when dispatching messages per partition_strategy.""" + +partition_count_refresh_interval.label: +"""Partition Count Refresh Interval""" + +max_batch_bytes.desc: +"""Maximum bytes to collect in a Kafka message batch. Most of the Kafka brokers default to a limit of 1 MB batch size. EMQX's default value is less than 1 MB in order to compensate Kafka message encoding overheads (especially when each individual message is very small). When a single message is over the limit, it is still sent (as a single element batch).""" + +max_batch_bytes.label: +"""Max Batch Bytes""" + +required_acks.desc: +"""Required acknowledgements for Kafka partition leader to wait for its followers before it sends back the acknowledgement to EMQX Kafka producer + +all_isr: Require all in-sync replicas to acknowledge. +leader_only: Require only the partition-leader's acknowledgement. +none: No need for Kafka to acknowledge at all.""" + +required_acks.label: +"""Required Acks""" + +metadata_request_timeout.desc: +"""Maximum wait time when fetching metadata from Kafka.""" + +metadata_request_timeout.label: +"""Metadata Request Timeout""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +socket_nodelay.desc: +"""When set to 'true', TCP buffer is sent as soon as possible. Otherwise, the OS kernel may buffer small TCP packets for a while (40 ms by default).""" + +socket_nodelay.label: +"""No Delay""" + +authentication.desc: +"""Authentication configs.""" + +authentication.label: +"""Authentication""" + +buffer_memory_overload_protection.desc: +"""Applicable when buffer mode is set to memory +EMQX will drop old buffered messages under high memory pressure. The high memory threshold is defined in config sysmon.os.sysmem_high_watermark. NOTE: This config only works on Linux.""" + +buffer_memory_overload_protection.label: +"""Memory Overload Protection""" + +auth_sasl_mechanism.desc: +"""SASL authentication mechanism.""" + +auth_sasl_mechanism.label: +"""Mechanism""" + +config_enable.desc: +"""Enable (true) or disable (false) this Kafka bridge.""" + +config_enable.label: +"""Enable or Disable""" + +consumer_mqtt_payload.desc: +"""The template for transforming the incoming Kafka message. By default, it will use JSON format to serialize inputs from the Kafka message. Such fields are: +headers: an object containing string key-value pairs. +key: Kafka message key (uses the chosen key encoding). +offset: offset for the message. +topic: Kafka topic. +ts: message timestamp. +ts_type: message timestamp type, which is one of create, append or undefined. +value: Kafka message value (uses the chosen value encoding).""" + +consumer_mqtt_payload.label: +"""MQTT Payload Template""" + +consumer_opts.desc: +"""Local MQTT publish and Kafka consumer configs.""" + +consumer_opts.label: +"""MQTT to Kafka""" + +kafka_consumer.desc: +"""Kafka Consumer configuration.""" + +kafka_consumer.label: +"""Kafka Consumer""" + +desc_config.desc: +"""Configuration for a Kafka bridge.""" + +desc_config.label: +"""Kafka Bridge Configuration""" + +consumer_value_encoding_mode.desc: +"""Defines how the value from the Kafka message is encoded before being forwarded via MQTT. +none Uses the value from the Kafka message unchanged. Note: in this case, the value must be a valid UTF-8 string. +base64 Uses base-64 encoding on the received value.""" + +consumer_value_encoding_mode.label: +"""Value Encoding Mode""" + +buffer_per_partition_limit.desc: +"""Number of bytes allowed to buffer for each Kafka partition. When this limit is exceeded, old messages will be dropped in a trade for credits for new messages to be buffered.""" + +buffer_per_partition_limit.label: +"""Per-partition Buffer Limit""" + +bootstrap_hosts.desc: +"""A comma separated list of Kafka host[:port] endpoints to bootstrap the client. Default port number is 9092.""" + +bootstrap_hosts.label: +"""Bootstrap Hosts""" + +consumer_max_rejoin_attempts.desc: +"""Maximum number of times allowed for a member to re-join the group. If the consumer group can not reach balance after this configured number of attempts, the consumer group member will restart after a delay.""" + +consumer_max_rejoin_attempts.label: +"""Max Rejoin Attempts""" + +kafka_message_key.desc: +"""Template to render Kafka message key. If the template is rendered into a NULL value (i.e. there is no such data field in Rule Engine context) then Kafka's NULL (but not empty string) is used.""" + +kafka_message_key.label: +"""Message Key""" + +kafka_message.desc: +"""Template to render a Kafka message.""" + +kafka_message.label: +"""Kafka Message Template""" + +mqtt_topic.desc: +"""MQTT topic or topic filter as data source (bridge input). If rule action is used as data source, this config should be left empty, otherwise messages will be duplicated in Kafka.""" + +mqtt_topic.label: +"""Source MQTT Topic""" + +kafka_message_value.desc: +"""Template to render Kafka message value. If the template is rendered into a NULL value (i.e. there is no such data field in Rule Engine context) then Kafka's NULL (but not empty string) is used.""" + +kafka_message_value.label: +"""Message Value""" + +partition_strategy.desc: +"""Partition strategy is to tell the producer how to dispatch messages to Kafka partitions. + +random: Randomly pick a partition for each message +key_dispatch: Hash Kafka message key to a partition number""" + +partition_strategy.label: +"""Partition Strategy""" + +buffer_segment_bytes.desc: +"""Applicable when buffer mode is set to disk or hybrid. +This value is to specify the size of each on-disk buffer file.""" + +buffer_segment_bytes.label: +"""Segment File Bytes""" + +consumer_kafka_opts.desc: +"""Kafka consumer configs.""" + +consumer_kafka_opts.label: +"""Kafka Consumer""" + +max_inflight.desc: +"""Maximum number of batches allowed for Kafka producer (per-partition) to send before receiving acknowledgement from Kafka. Greater value typically means better throughput. However, there can be a risk of message reordering when this value is greater than 1.""" + +max_inflight.label: +"""Max Inflight""" + +auth_sasl_username.desc: +"""SASL authentication username.""" + +auth_sasl_username.label: +"""Username""" + +auth_kerberos_keytab_file.desc: +"""SASL GSSAPI authentication Kerberos keytab file path. NOTE: This file has to be placed in EMQX nodes, and the EMQX service runner user requires read permission.""" + +auth_kerberos_keytab_file.label: +"""Kerberos keytab file""" + +compression.desc: +"""Compression method.""" + +compression.label: +"""Compression""" - } - label { - en: "MQTT Payload Template" - zh: "MQTT Payload Template" - } - } - consumer_kafka_topic { - desc { - en: "Kafka topic to consume from." - zh: "指定从哪个 Kafka 主题消费消息。" - } - label { - en: "Kafka Topic" - zh: "Kafka 主题" - } - } - consumer_max_batch_bytes { - desc { - en: "Set how many bytes to pull from Kafka in each fetch request. " - "Please note that if the configured value is smaller than the message size in Kafka, it may negatively impact the fetch performance." - zh: "设置每次从 Kafka 拉取数据的字节数。" - "如该配置小于 Kafka 消息的大小，可能会影响消费性能。" - } - label { - en: "Fetch Bytes" - zh: "拉取字节数" - } - } - # hidden - consumer_max_rejoin_attempts { - desc { - en: "Maximum number of times allowed for a member to re-join the group. If the consumer group can not reach balance after this configured number of attempts, the consumer group member will restart after a delay." - zh: "消费组成员允许重新加入小组的最大次数。如超过该配置次数后仍未能成功加入消费组，则会在等待一段时间后重试。" - } - label { - en: "Max Rejoin Attempts" - zh: "最大的重新加入尝试" - } - } - consumer_offset_reset_policy { - desc { - en: "Defines from which offset a consumer should start fetching when there " - "is no commit history or when the commit history becomes invalid." - zh: "如不存在偏移量历史记录或历史记录失效，消费者应使用哪个偏移量开始消费。" - } - label { - en: "Offset Reset Policy" - zh: "偏移重置策略" - } - } - consumer_offset_commit_interval_seconds { - desc { - en: "Defines the time interval between two offset commit requests sent for each consumer group." - zh: "指定 Kafka 消费组偏移量提交的时间间隔。" - } - label { - en: "Offset Commit Interval" - zh: "偏移提交间隔" - } - } - consumer_topic_mapping { - desc { - en: "Defines the mapping between Kafka topics and MQTT topics. Must contain at least one item." - zh: "指定 Kafka 主题和 MQTT 主题之间的映射关系。 应至少包含一项。" - } - label { - en: "Topic Mapping" - zh: "主题映射关系" - } - } - consumer_key_encoding_mode { - desc { - en: "Defines how the key from the Kafka message is" - " encoded before being forwarded via MQTT.\n" - "none Uses the key from the Kafka message unchanged." - " Note: in this case, the key must be a valid UTF-8 string.\n" - "base64 Uses base-64 encoding on the received key." - zh: "通过 MQTT 转发之前，如何处理 Kafka 消息的 Key。" - "none 使用 Kafka 消息中的 Key 原始值，不进行编码。" - " 注意：在这种情况下，Key 必须是一个有效的 UTF-8 字符串。\n" - "base64 对收到的密钥或值使用 base-64 编码。" - } - label { - en: "Key Encoding Mode" - zh: "Key 编码模式" - } - } - consumer_value_encoding_mode { - desc { - en: "Defines how the value from the Kafka message is" - " encoded before being forwarded via MQTT.\n" - "none Uses the value from the Kafka message unchanged." - " Note: in this case, the value must be a valid UTF-8 string.\n" - "base64 Uses base-64 encoding on the received value." - zh: "通过 MQTT 转发之前，如何处理 Kafka 消息的 Value。" - "none 使用 Kafka 消息中的 Value 原始值，不进行编码。" - " 注意：在这种情况下，Value 必须是一个有效的 UTF-8 字符串。\n" - "base64 对收到的 Value 使用 base-64 编码。" - } - label { - en: "Value Encoding Mode" - zh: "Value 编码模式" - } - } } diff --git a/rel/i18n/emqx_bridge_mqtt_schema.hocon b/rel/i18n/emqx_bridge_mqtt_schema.hocon index b935b360c..e05c4fb0a 100644 --- a/rel/i18n/emqx_bridge_mqtt_schema.hocon +++ b/rel/i18n/emqx_bridge_mqtt_schema.hocon @@ -1,34 +1,21 @@ emqx_bridge_mqtt_schema { - config { - desc { - en: """The config for MQTT Bridges.""" - zh: """MQTT Bridge 的配置。""" - } - label: { - en: "Config" - zh: "配置" - } - } - desc_type { - desc { - en: """The bridge type.""" - zh: """Bridge 的类型""" - } - label: { - en: "Bridge Type" - zh: "Bridge 类型" - } - } - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """Bridge 名字，Bridge 的可读描述""" - } - label: { - en: "Bridge Name" - zh: "Bridge 名字" - } - } +config.desc: +"""The config for MQTT Bridges.""" + +config.label: +"""Config""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The bridge type.""" + +desc_type.label: +"""Bridge Type""" } diff --git a/rel/i18n/emqx_bridge_schema.hocon b/rel/i18n/emqx_bridge_schema.hocon index de4ceb0d5..1d3053f73 100644 --- a/rel/i18n/emqx_bridge_schema.hocon +++ b/rel/i18n/emqx_bridge_schema.hocon @@ -1,324 +1,158 @@ emqx_bridge_schema { - desc_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用 Bridge""" - } - label: { - en: "Enable Or Disable Bridge" - zh: "启用/禁用 Bridge" - } - } +bridges_mqtt.desc: +"""MQTT bridges to/from another MQTT broker""" - desc_metrics { - desc { - en: """The metrics of the bridge""" - zh: """Bridge 的计数""" - } - label: { - en: "Bridge Metrics" - zh: "Bridge 计数" - } - } +bridges_mqtt.label: +"""MQTT Bridge""" - desc_node_metrics { - desc { - en: """The metrics of the bridge for each node""" - zh: """每个节点的 Bridge 计数""" - } - label: { - en: "Each Node Bridge Metircs" - zh: "每个节点的 Bridge 计数" - } - } +bridges_webhook.desc: +"""WebHook to an HTTP server.""" - desc_status { - desc { - en: """The status of the bridge
+bridges_webhook.label: +"""WebHook""" + +desc_bridges.desc: +"""Configuration for MQTT bridges.""" + +desc_bridges.label: +"""MQTT Bridge Configuration""" + +desc_enable.desc: +"""Enable or disable this bridge""" + +desc_enable.label: +"""Enable Or Disable Bridge""" + +desc_metrics.desc: +"""Bridge metrics.""" + +desc_metrics.label: +"""Bridge Metrics""" + +desc_node_metrics.desc: +"""Node metrics.""" + +desc_node_metrics.label: +"""Node Metrics""" + +desc_node_name.desc: +"""The node name.""" + +desc_node_name.label: +"""Node Name""" + +desc_node_status.desc: +"""Node status.""" + +desc_node_status.label: +"""Node Status""" + +desc_status.desc: +"""The status of the bridge
- connecting: the initial state before any health probes were made.
- connected: when the bridge passes the health probes.
- disconnected: when the bridge can not pass health probes.
- stopped: when the bridge resource is requested to be stopped.
- inconsistent: When not all the nodes are at the same status.""" - zh: """Bridge 的连接状态
-- connecting: 启动时的初始状态。
-- connected: 桥接驱动健康检查正常。
-- disconnected: 当桥接无法通过健康检查。
-- stopped: 桥接处于停用状态。
-- inconsistent: 集群中有各节点汇报的状态不一致。""" - } - label: { - en: "Bridge Status" - zh: "Bridge 状态" - } - } - desc_status_reason { - desc { - en: "This is the reason given in case a bridge is failing to connect." - zh: "桥接连接失败的原因。" - } - label: { - en: "Failure reason" - zh: "失败原因" - } - } +desc_status.label: +"""Bridge Status""" - desc_node_status { - desc { - en: """The status of the bridge for each node. -- connecting: the initial state before any health probes were made.
-- connected: when the bridge passes the health probes.
-- disconnected: when the bridge can not pass health probes.
-- stopped: when the bridge resource is requested to be stopped.""" - zh: """每个节点的 Bridge 状态 -- connecting: 启动时的初始状态。
-- connected: 桥接驱动健康检查正常。
-- disconnected: 当桥接无法通过健康检查。
-- stopped: 桥接处于停用状态。""" - } - label: { - en: "Node Bridge Status" - zh: "每个节点的 Bridge 状态" - } - } +desc_status_reason.desc: +"""This is the reason given in case a bridge is failing to connect.""" - bridges_webhook { - desc { - en: """WebHook to an HTTP server.""" - zh: """转发消息到 HTTP 服务器的 WebHook""" - } - label: { - en: "WebHook" - zh: "WebHook" - } - } +desc_status_reason.label: +"""Failure reason""" +metric_dropped.desc: +"""Count of messages dropped.""" - bridges_mqtt { - desc { - en: """MQTT bridges to/from another MQTT broker""" - zh: """桥接到另一个 MQTT Broker 的 MQTT Bridge""" - } - label: { - en: "MQTT Bridge" - zh: "MQTT Bridge" - } - } +metric_dropped.label: +"""Dropped""" - metric_dropped { - desc { - en: """Count of messages dropped.""" - zh: """被丢弃的消息个数。""" - } - label: { - en: "Dropped" - zh: "丢弃" - } - } +metric_dropped_other.desc: +"""Count of messages dropped due to other reasons.""" - metric_dropped_other { - desc { - en: """Count of messages dropped due to other reasons.""" - zh: """因为其他原因被丢弃的消息个数。""" - } - label: { - en: "Dropped Other" - zh: "其他丢弃" - } - } - metric_dropped_queue_full { - desc { - en: """Count of messages dropped due to the queue is full.""" - zh: """因为队列已满被丢弃的消息个数。""" - } - label: { - en: "Dropped Queue Full" - zh: "队列已满被丢弃" - } - } - metric_dropped_resource_not_found { - desc { - en: """Count of messages dropped due to the resource is not found.""" - zh: """因为资源不存在被丢弃的消息个数。""" - } - label: { - en: "Dropped Resource NotFound" - zh: "资源不存在被丢弃" - } - } - metric_dropped_resource_stopped { - desc { - en: """Count of messages dropped due to the resource is stopped.""" - zh: """因为资源已停用被丢弃的消息个数。""" - } - label: { - en: "Dropped Resource Stopped" - zh: "资源停用被丢弃" - } - } - metric_matched { - desc { - en: """Count of this bridge is matched and queried.""" - zh: """Bridge 被匹配到（被请求）的次数。""" - } - label: { - en: "Matched" - zh: "匹配次数" - } - } +metric_dropped_other.label: +"""Dropped Other""" - metric_queuing { - desc { - en: """Count of messages that are currently queuing.""" - zh: """当前被缓存到磁盘队列的消息个数。""" - } - label: { - en: "Queued" - zh: "被缓存" - } - } - metric_retried { - desc { - en: """Times of retried.""" - zh: """重试的次数。""" - } - label: { - en: "Retried" - zh: "已重试" - } - } +metric_dropped_queue_full.desc: +"""Count of messages dropped due to the queue is full.""" - metric_sent_failed { - desc { - en: """Count of messages that sent failed.""" - zh: """发送失败的消息个数。""" - } - label: { - en: "Sent Failed" - zh: "发送失败" - } - } +metric_dropped_queue_full.label: +"""Dropped Queue Full""" - metric_inflight { - desc { - en: """Count of messages that were sent asynchronously but ACKs are not yet received.""" - zh: """已异步地发送但没有收到 ACK 的消息个数。""" - } - label: { - en: "Sent Inflight" - zh: "已发送未确认" - } - } - metric_sent_success { - desc { - en: """Count of messages that sent successfully.""" - zh: """已经发送成功的消息个数。""" - } - label: { - en: "Sent Success" - zh: "发送成功" - } - } +metric_dropped_resource_not_found.desc: +"""Count of messages dropped due to the resource is not found.""" - metric_rate { - desc { - en: """The rate of matched, times/second""" - zh: """执行操作的速率，次/秒""" - } - label: { - en: "Rate" - zh: "速率" - } - } +metric_dropped_resource_not_found.label: +"""Dropped Resource NotFound""" - metric_rate_max { - desc { - en: """The max rate of matched, times/second""" - zh: """执行操作的最大速率，次/秒""" - } - label: { - en: "Max Rate Of Matched" - zh: "执行操作的最大速率" - } - } +metric_dropped_resource_stopped.desc: +"""Count of messages dropped due to the resource is stopped.""" - metric_rate_last5m { - desc { - en: """The average rate of matched in the last 5 minutes, times/second""" - zh: """5 分钟平均速率，次/秒""" - } - label: { - en: "Last 5 Minutes Rate" - zh: "5 分钟平均速率" - } - } +metric_dropped_resource_stopped.label: +"""Dropped Resource Stopped""" - metric_received { - desc { - en: """Count of messages that is received from the remote system.""" - zh: """从远程系统收到的消息个数。""" - } - label: { - en: "Received" - zh: "已接收" - } - } +metric_inflight.desc: +"""Count of messages that were sent asynchronously but ACKs are not yet received.""" - desc_bridges { - desc { - en: """Configuration for MQTT bridges.""" - zh: """MQTT Bridge 配置""" - } - label: { - en: "MQTT Bridge Configuration" - zh: "MQTT Bridge 配置" - } - } +metric_inflight.label: +"""Sent Inflight""" - desc_metrics { - desc { - en: """Bridge metrics.""" - zh: """Bridge 计数""" - } - label: { - en: "Bridge Metrics" - zh: "Bridge 计数" - } - } +metric_matched.desc: +"""Count of this bridge is matched and queried.""" - desc_node_metrics { - desc { - en: """Node metrics.""" - zh: """节点的计数器""" - } - label: { - en: "Node Metrics" - zh: "节点的计数器" - } - } +metric_matched.label: +"""Matched""" - desc_node_status { - desc { - en: """Node status.""" - zh: """节点的状态""" - } - label: { - en: "Node Status" - zh: "节点的状态" - } - } +metric_queuing.desc: +"""Count of messages that are currently queuing.""" - desc_node_name { - desc { - en: """The node name.""" - zh: """节点的名字""" - } - label: { - en: "Node Name" - zh: "节点名字" - } - } +metric_queuing.label: +"""Queued""" + +metric_rate.desc: +"""The rate of matched, times/second""" + +metric_rate.label: +"""Rate""" + +metric_rate_last5m.desc: +"""The average rate of matched in the last 5 minutes, times/second""" + +metric_rate_last5m.label: +"""Last 5 Minutes Rate""" + +metric_rate_max.desc: +"""The max rate of matched, times/second""" + +metric_rate_max.label: +"""Max Rate Of Matched""" + +metric_received.desc: +"""Count of messages that is received from the remote system.""" + +metric_received.label: +"""Received""" + +metric_retried.desc: +"""Times of retried.""" + +metric_retried.label: +"""Retried""" + +metric_sent_failed.desc: +"""Count of messages that sent failed.""" + +metric_sent_failed.label: +"""Sent Failed""" + +metric_sent_success.desc: +"""Count of messages that sent successfully.""" + +metric_sent_success.label: +"""Sent Success""" } diff --git a/rel/i18n/emqx_bridge_webhook_schema.hocon b/rel/i18n/emqx_bridge_webhook_schema.hocon index cf86b63e0..4037893bd 100644 --- a/rel/i18n/emqx_bridge_webhook_schema.hocon +++ b/rel/i18n/emqx_bridge_webhook_schema.hocon @@ -1,162 +1,92 @@ emqx_bridge_webhook_schema { - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用 Bridge""" - } - label: { - en: "Enable Or Disable Bridge" - zh: "启用/禁用 Bridge" - } - } - config_direction { - desc { - en: """Deprecated, The direction of this bridge, MUST be 'egress'""" - zh: """已废弃，Bridge 的方向，必须是 egress""" - } - label: { - en: "Bridge Direction" - zh: "Bridge 方向" - } - } - config_url { - desc { - en: """The URL of the HTTP Bridge.
-Template with variables is allowed in the path, but variables cannot be used in the scheme, host, -or port part.
-For example, http://localhost:9901/${topic} is allowed, but - http://${host}:9901/message or http://localhost:${port}/message -is not allowed.""" - zh: """HTTP Bridge 的 URL。
-路径中允许使用带变量的模板，但是 host， port 不允许使用变量模板。
-例如， http://localhost:9901/${topic} 是允许的， -但是 http://${host}:9901/message -或 http://localhost:${port}/message -不允许。""" - } - label: { - en: "HTTP Bridge" - zh: "HTTP Bridge" - } - } - - config_local_topic { - desc { - en: """The MQTT topic filter to be forwarded to the HTTP server. All MQTT 'PUBLISH' messages with the topic -matching the local_topic will be forwarded.
-NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is -configured, then both the data got from the rule and the MQTT messages that match local_topic -will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 HTTP 服务器。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HTTP 服务器。""" - } - label: { - en: "Local Topic" - zh: "本地 Topic" - } - } - - config_method { - desc { - en: """The method of the HTTP request. All the available methods are: post, put, get, delete.
-Template with variables is allowed.""" - zh: """HTTP 请求的方法。 所有可用的方法包括：post、put、get、delete。
-允许使用带有变量的模板。""" - } - label: { - en: "HTTP Method" - zh: "HTTP 请求方法" - } - } - - config_headers { - desc { - en: """The headers of the HTTP request.
-Template with variables is allowed.""" - zh: """HTTP 请求的标头。
-允许使用带有变量的模板。""" - } - label: { - en: "HTTP Header" - zh: "HTTP 请求标头" - } - } - - config_body { - desc { - en: """The body of the HTTP request.
+config_body.desc: +"""The body of the HTTP request.
If not provided, the body will be a JSON object of all the available fields.
There, 'all the available fields' means the context of a MQTT message when this webhook is triggered by receiving a MQTT message (the `local_topic` is set), or the context of the event when this webhook is triggered by a rule (i.e. this webhook is used as an action of a rule).
Template with variables is allowed.""" - zh: """HTTP 请求的正文。
-如果没有设置该字段，请求正文将是包含所有可用字段的 JSON object。
-如果该 webhook 是由于收到 MQTT 消息触发的，'所有可用字段' 将是 MQTT 消息的 -上下文信息；如果该 webhook 是由于规则触发的，'所有可用字段' 则为触发事件的上下文信息。
-允许使用带有变量的模板。""" - } - label: { - en: "HTTP Body" - zh: "HTTP 请求正文" - } - } - config_request_timeout { - desc { - en: """HTTP request timeout.""" - zh: """HTTP 请求超时""" - } - label: { - en: "HTTP Request Timeout" - zh: "HTTP 请求超时" - } - } +config_body.label: +"""HTTP Body""" - config_max_retries { - desc { - en: """HTTP request max retry times if failed.""" - zh: """HTTP 请求失败最大重试次数""" - } - label: { - en: "HTTP Request Max Retries" - zh: "HTTP 请求重试次数" - } - } +config_direction.desc: +"""Deprecated, The direction of this bridge, MUST be 'egress'""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label: { - en: "Bridge Type" - zh: "Bridge 类型" - } - } +config_direction.label: +"""Bridge Direction""" - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """Bridge 名字，Bridge 的可读描述""" - } - label: { - en: "Bridge Name" - zh: "Bridge 名字" - } - } +config_enable.desc: +"""Enable or disable this bridge""" - desc_config { - desc { - en: """Configuration for an HTTP bridge.""" - zh: """HTTP Bridge 配置""" - } - label: { - en: "HTTP Bridge Configuration" - zh: "HTTP Bridge 配置" - } - } +config_enable.label: +"""Enable Or Disable Bridge""" + +config_headers.desc: +"""The headers of the HTTP request.
+Template with variables is allowed.""" + +config_headers.label: +"""HTTP Header""" + +config_local_topic.desc: +"""The MQTT topic filter to be forwarded to the HTTP server. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded.""" + +config_local_topic.label: +"""Local Topic""" + +config_max_retries.desc: +"""HTTP request max retry times if failed.""" + +config_max_retries.label: +"""HTTP Request Max Retries""" + +config_method.desc: +"""The method of the HTTP request. All the available methods are: post, put, get, delete.
+Template with variables is allowed.""" + +config_method.label: +"""HTTP Method""" + +config_request_timeout.desc: +"""HTTP request timeout.""" + +config_request_timeout.label: +"""HTTP Request Timeout""" + +config_url.desc: +"""The URL of the HTTP Bridge.
+Template with variables is allowed in the path, but variables cannot be used in the scheme, host, +or port part.
+For example, http://localhost:9901/${topic} is allowed, but + http://${host}:9901/message or http://localhost:${port}/message +is not allowed.""" + +config_url.label: +"""HTTP Bridge""" + +desc_config.desc: +"""Configuration for an HTTP bridge.""" + +desc_config.label: +"""HTTP Bridge Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" } diff --git a/rel/i18n/emqx_coap_api.hocon b/rel/i18n/emqx_coap_api.hocon index 77ca40c00..14f644a87 100644 --- a/rel/i18n/emqx_coap_api.hocon +++ b/rel/i18n/emqx_coap_api.hocon @@ -1,58 +1,27 @@ emqx_coap_api { - send_coap_request { - desc { - en: """Send a CoAP request message to the client""" - zh: """发送 CoAP 消息到指定客户端""" - } - } +content_type.desc: +"""Payload type""" - token { - desc { - en: """Message token, can be empty""" - zh: """消息 Token, 可以为空""" - } - } +message_id.desc: +"""Message ID""" - method { - desc { - en: """Request method type""" - zh: """请求 Method 类型""" - } - } +method.desc: +"""Request method type""" - timeout { - desc { - en: """Timespan for response""" - zh: """请求超时时间""" - } - } +payload.desc: +"""The content of the payload""" - content_type { - desc { - en: """Payload type""" - zh: """Payload 类型""" - } - } +response_code.desc: +"""Response code""" - payload { - desc { - en: """The content of the payload""" - zh: """Payload 内容""" - } - } +send_coap_request.desc: +"""Send a CoAP request message to the client""" - message_id { - desc { - en: """Message ID""" - zh: """消息 ID""" - } - } +timeout.desc: +"""Timespan for response""" + +token.desc: +"""Message token, can be empty""" - response_code { - desc { - en: """Response code""" - zh: """应答码""" - } - } } diff --git a/rel/i18n/emqx_coap_schema.hocon b/rel/i18n/emqx_coap_schema.hocon index 1e6452e49..322003a3f 100644 --- a/rel/i18n/emqx_coap_schema.hocon +++ b/rel/i18n/emqx_coap_schema.hocon @@ -1,80 +1,38 @@ emqx_coap_schema { - coap { - desc { - en: """The CoAP Gateway configuration. + +coap.desc: +"""The CoAP Gateway configuration. This gateway is implemented based on RFC-7252 and https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" - zh: """CoAP 网关配置。 -该网关的实现基于 RFC-7252 和 https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" - } - } - coap_heartbeat { - desc { - en: """The gateway server required minimum heartbeat interval. -When connection mode is enabled, this parameter is used to set the minimum heartbeat interval for the connection to be alive""" - zh: """CoAP 网关要求客户端的最小心跳间隔时间。 -当 connection_required 开启后，该参数用于检查客户端连接是否存活""" - } - } - - coap_connection_required { - desc { - en: """Enable or disable connection mode. +coap_connection_required.desc: +"""Enable or disable connection mode. Connection mode is a feature of non-standard protocols. When connection mode is enabled, it is necessary to maintain the creation, authentication and alive of connection resources""" - zh: """是否开启连接模式。 -连接模式是非标准协议的功能。它维护 CoAP 客户端上线、认证、和连接状态的保持""" - } - } - coap_notify_type { - desc { - en: """The Notification Message will be delivered to the CoAP client if a new message received on an observed topic. +coap_heartbeat.desc: +"""The gateway server required minimum heartbeat interval. +When connection mode is enabled, this parameter is used to set the minimum heartbeat interval for the connection to be alive""" + +coap_notify_type.desc: +"""The Notification Message will be delivered to the CoAP client if a new message received on an observed topic. The type of delivered coap message can be set to:
- non: Non-confirmable;
- con: Confirmable;
- qos: Mapping from QoS type of received message, QoS0 -> non, QoS1,2 -> con""" - zh: """投递给 CoAP 客户端的通知消息类型。当客户端 Observe 一个资源（或订阅某个主题）时，网关会向客户端推送新产生的消息。其消息类型可设置为：
- - non: 不需要客户端返回确认消息;
- - con: 需要客户端返回一个确认消息;
- - qos: 取决于消息的 QoS 等级; QoS 0 会以 `non` 类型下发，QoS 1/2 会以 `con` 类型下发""" - } - } - coap_subscribe_qos { - desc { - en: """The Default QoS Level indicator for subscribe request. -This option specifies the QoS level for the CoAP Client when establishing a subscription membership, if the subscribe request is not carried `qos` option. The indicator can be set to:
- - qos0, qos1, qos2: Fixed default QoS level
- - coap: Dynamic QoS level by the message type of subscribe request
- * qos0: If the subscribe request is non-confirmable
- * qos1: If the subscribe request is confirmable""" - - zh: """客户端订阅请求的默认 QoS 等级。 -当 CoAP 客户端发起订阅请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
- - qos0、 qos1、qos2: 设置为固定的 QoS 等级
- - coap: 依据订阅操作的 CoAP 报文类型来动态决定
- * 当订阅请求为 `non-confirmable` 类型时，取值为 qos0
- * 当订阅请求为 `confirmable` 类型时，取值为 qos1""" - } - } - - coap_publish_qos { - desc { - en: """The Default QoS Level indicator for publish request. +coap_publish_qos.desc: +"""The Default QoS Level indicator for publish request. This option specifies the QoS level for the CoAP Client when publishing a message to EMQX PUB/SUB system, if the publish request is not carried `qos` option. The indicator can be set to:
- qos0, qos1, qos2: Fixed default QoS level
- coap: Dynamic QoS level by the message type of publish request
* qos0: If the publish request is non-confirmable
* qos1: If the publish request is confirmable""" - zh: """客户端发布请求的默认 QoS 等级。 -当 CoAP 客户端发起发布请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
- - qos0、qos1、qos2: 设置为固定的 QoS 等级
- - coap: 依据发布操作的 CoAP 报文类型来动态决定
- * 当发布请求为 `non-confirmable` 类型时，取值为 qos0
- * 当发布请求为 `confirmable` 类型时，取值为 qos1""" - } - } - +coap_subscribe_qos.desc: +"""The Default QoS Level indicator for subscribe request. +This option specifies the QoS level for the CoAP Client when establishing a subscription membership, if the subscribe request is not carried `qos` option. The indicator can be set to:
+ - qos0, qos1, qos2: Fixed default QoS level
+ - coap: Dynamic QoS level by the message type of subscribe request
+ * qos0: If the subscribe request is non-confirmable
+ * qos1: If the subscribe request is confirmable""" } diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index b252353f8..f4016d32e 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -1,411 +1,456 @@ emqx_conf_schema { - cluster_name { - desc { - en: """Human-friendly name of the EMQX cluster.""" - zh: """EMQX集群名称。每个集群都有一个唯一的名称。服务发现时会用于做路径的一部分。""" - } - label { - en: "Cluster Name" - zh: "集群名称" - } - } +common_handler_drop_mode_qlen.desc: +"""When the number of buffered log events is larger than this value, the new log events are dropped. +When drop mode is activated or deactivated, a message is printed in the logs.""" - process_limit { - desc { - en: """Maximum number of simultaneously existing processes for this Erlang system. -The actual maximum chosen may be much larger than the Number passed. -For more information, see: https://www.erlang.org/doc/man/erl.html""" +common_handler_drop_mode_qlen.label: +"""Queue Length before Entering Drop Mode""" - zh: """Erlang系统同时存在的最大进程数。 -实际选择的最大值可能比设置的数字大得多。 -参考: https://www.erlang.org/doc/man/erl.html""" - } - label { - en: "Erlang Process Limit" - zh: "Erlang 最大进程数" - } - } +cluster_mcast_addr.desc: +"""Multicast IPv4 address.""" - max_ports { - desc { - en: """Maximum number of simultaneously existing ports for this Erlang system. -The actual maximum chosen may be much larger than the Number passed. -For more information, see: https://www.erlang.org/doc/man/erl.html""" +cluster_mcast_addr.label: +"""Cluster Multicast Address""" - zh: """Erlang系统同时存在的最大端口数。 -实际选择的最大值可能比设置的数字大得多。 -参考: https://www.erlang.org/doc/man/erl.html""" - } - label { - en: "Erlang Port Limit" - zh: "Erlang 最大端口数" - } - } +desc_cluster_dns.desc: +"""Service discovery via DNS SRV records.""" - dist_buffer_size { - desc { - en: """Erlang's distribution buffer busy limit in kilobytes.""" - zh: """Erlang分布式缓冲区的繁忙阈值，单位是KB。""" - } - label { - en: "Erlang's dist buffer size(KB)" - zh: "Erlang分布式缓冲区的繁忙阈值(KB)" - } - } +desc_cluster_dns.label: +"""Cluster DNS""" - max_ets_tables { - desc { - en: """Max number of ETS tables""" - zh: """Erlang ETS 表的最大数量""" - } - label { - en: "Max number of ETS tables" - zh: "Erlang 表的最大数量" - } - } +cluster_dns_name.desc: +"""The domain name from which to discover peer EMQX nodes' IP addresses. +Applicable when cluster.discovery_strategy = dns""" - cluster_discovery_strategy { - desc { - en: """Service discovery method for the cluster nodes. Possible values are: -- manual: Use emqx ctl cluster command to manage cluster.
-- static: Configure static nodes list by setting seeds in config file.
-- dns: Use DNS A record to discover peer nodes.
-- etcd: Use etcd to discover peer nodes.
-- k8s: Use Kubernetes API to discover peer pods.""" +cluster_dns_name.label: +"""Cluster Dns Name""" - zh: """集群节点发现方式。可选值为: -- manual: 使用 emqx ctl cluster 命令管理集群。
-- static: 配置静态节点。配置几个固定的节点，新节点通过连接固定节点中的某一个来加入集群。
-- dns: 使用 DNS A 记录的方式发现节点。
-- etcd: 使用 etcd 发现节点。
-- k8s: 使用 Kubernetes API 发现节点。""" - } - label { - en: "Cluster Discovery Strategy" - zh: "集群服务发现策略" - } - } +rpc_keyfile.desc: +"""Path to the private key file for the rpc.certfile.
+Note: contents of this file are secret, so it's necessary to set permissions to 600.""" - cluster_autoclean { - desc { - en: """Remove disconnected nodes from the cluster after this interval.""" - zh: """指定多久之后从集群中删除离线节点。""" - } - label { - en: "Cluster Auto Clean" - zh: "自动删除离线节点时间" - } - } +rpc_keyfile.label: +"""RPC Keyfile""" - cluster_autoheal { - desc { - en: """If true, the node will try to heal network partitions automatically.""" - zh: """集群脑裂自动恢复机制开关。""" - } - label { - en: "Cluster Auto Heal" - zh: "节点脑裂自动修复机制" - } - } +cluster_mcast_recbuf.desc: +"""Size of the kernel-level buffer for incoming datagrams.""" - cluster_proto_dist { - desc { - en: """The Erlang distribution protocol for the cluster.
+cluster_mcast_recbuf.label: +"""Cluster Muticast Sendbuf""" + +cluster_autoheal.desc: +"""If true, the node will try to heal network partitions automatically.""" + +cluster_autoheal.label: +"""Cluster Auto Heal""" + +log_overload_kill_enable.desc: +"""Enable log handler overload kill feature.""" + +log_overload_kill_enable.label: +"""Log Handler Overload Kill""" + +node_etc_dir.desc: +"""etc dir for the node""" + +node_etc_dir.label: +"""Etc Dir""" + +cluster_proto_dist.desc: +"""The Erlang distribution protocol for the cluster.
- inet_tcp: IPv4 TCP
- inet_tls: IPv4 TLS, works together with etc/ssl_dist.conf""" - zh: """分布式 Erlang 集群协议类型。可选值为:
-- inet_tcp: 使用 IPv4
-- inet_tls: 使用 TLS，需要配合 etc/ssl_dist.conf 一起使用。""" - } - label { - en: "Cluster Protocol Distribution" - zh: "集群内部通信协议" - } - } - cluster_static_seeds { - desc { - en: """List EMQX node names in the static cluster. See node.name.""" - zh: """集群中的EMQX节点名称列表， -指定固定的节点列表，多个节点间使用逗号 , 分隔。 -当 cluster.discovery_strategy 为 static 时，此配置项才有效。 -适合于节点数量较少且固定的集群。""" - } - label { - en: "Cluster Static Seeds" - zh: "集群静态节点" - } - } +cluster_proto_dist.label: +"""Cluster Protocol Distribution""" - cluster_mcast_addr { - desc { - en: """Multicast IPv4 address.""" - zh: """指定多播 IPv4 地址。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Multicast Address" - zh: "多播地址" - } - } +log_burst_limit_enable.desc: +"""Enable log burst control feature.""" - cluster_mcast_ports { - desc { - en: """List of UDP ports used for service discovery.
-Note: probe messages are broadcast to all the specified ports.""" +log_burst_limit_enable.label: +"""Enable Burst""" - zh: """指定多播端口。如有多个端口使用逗号 , 分隔。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Multicast Ports" - zh: "多播端口" - } - } +dist_buffer_size.desc: +"""Erlang's distribution buffer busy limit in kilobytes.""" - cluster_mcast_iface { - desc { - en: """Local IP address the node discovery service needs to bind to.""" - zh: """指定节点发现服务需要绑定到本地 IP 地址。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Multicast Interface" - zh: "多播绑定地址" - } - } +dist_buffer_size.label: +"""Erlang's dist buffer size(KB)""" - cluster_mcast_ttl { - desc { - en: """Time-to-live (TTL) for the outgoing UDP datagrams.""" - zh: """指定多播的 Time-To-Live 值。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Multicast TTL" - zh: "多播TTL" - } - } +common_handler_max_depth.desc: +"""Maximum depth for Erlang term log formatting and Erlang process message queue inspection.""" - cluster_mcast_loop { - desc { - en: """If true, loop UDP datagrams back to the local socket.""" - zh: """设置多播的报文是否投递到本地回环地址。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Multicast Loop" - zh: "多播回环开关" - } - } +common_handler_max_depth.label: +"""Max Depth""" - cluster_mcast_sndbuf { - desc { - en: """Size of the kernel-level buffer for outgoing datagrams.""" - zh: """外发数据报的内核级缓冲区的大小。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Muticast Sendbuf" - zh: "多播发送缓存区" - } - } +desc_log.desc: +"""EMQX logging supports multiple sinks for the log events. +Each sink is represented by a _log handler_, which can be configured independently.""" - cluster_mcast_recbuf { - desc { - en: """Size of the kernel-level buffer for incoming datagrams.""" - zh: """接收数据报的内核级缓冲区的大小。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Muticast Sendbuf" - zh: "多播接收数据缓冲区" - } - } +desc_log.label: +"""Log""" - cluster_mcast_buffer { - desc { - en: """Size of the user-level buffer.""" - zh: """用户级缓冲区的大小。 -当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" - } - label { - en: "Cluster Muticast Buffer" - zh: "多播用户级缓冲区" - } - } +common_handler_flush_qlen.desc: +"""If the number of buffered log events grows larger than this threshold, a flush (delete) operation takes place. +To flush events, the handler discards the buffered log messages without logging.""" - cluster_dns_name { - desc { - en: """The domain name from which to discover peer EMQX nodes' IP addresses. -Applicable when cluster.discovery_strategy = dns""" - zh: """指定 DNS A 记录的名字。emqx 会通过访问这个 DNS A 记录来获取 IP 地址列表。 -当cluster.discovery_strategy 为 dns 时有效。""" - } - label { - en: "Cluster Dns Name" - zh: "DNS名称" - } - } +common_handler_flush_qlen.label: +"""Flush Threshold""" - cluster_dns_record_type { - desc { - en: """DNS record type.""" - zh: """DNS 记录类型。""" - } - label { - en: "DNS Record Type" - zh: "DNS记录类型" - } - } +common_handler_chars_limit.desc: +"""Set the maximum length of a single log message. If this length is exceeded, the log message will be truncated. +NOTE: Restrict char limiter if formatter is JSON , it will get a truncated incomplete JSON data, which is not recommended.""" - cluster_etcd_server { - desc { - en: """List of endpoint URLs of the etcd cluster""" - zh: """指定 etcd 服务的地址。如有多个服务使用逗号 , 分隔。 -当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" - } - label { - en: "Cluster Etcd Server" - zh: "Etcd 服务器地址" - } - } +common_handler_chars_limit.label: +"""Single Log Max Length""" - cluster_etcd_prefix { - desc { - en: """Key prefix used for EMQX service discovery.""" - zh: """指定 etcd 路径的前缀。每个节点在 etcd 中都会创建一个路径: -v2/keys///
-当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" - } - label { - en: "Cluster Etcd Prefix" - zh: "Etcd 路径前缀" - } - } +cluster_k8s_namespace.desc: +"""Kubernetes namespace.""" - cluster_etcd_node_ttl { - desc { - en: """Expiration time of the etcd key associated with the node. -It is refreshed automatically, as long as the node is alive.""" +cluster_k8s_namespace.label: +"""K8s Namespace""" - zh: """指定 etcd 中节点信息的过期时间。 -当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" - } - label { - en: "Cluster Etcd Node TTL" - zh: "Etcd 节点过期时间" - } - } +node_name.desc: +"""Unique name of the EMQX node. It must follow %name%@FQDN or +%name%@IPv4 format.""" - cluster_etcd_ssl { - desc { - en: """Options for the TLS connection to the etcd cluster.""" - zh: """当使用 TLS 连接 etcd 时的配置选项。 -当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" - } - label { - en: "Cluster Etcd SSL Option" - zh: "Etcd SSL 选项" - } - } +node_name.label: +"""Node Name""" - cluster_k8s_apiserver { - desc { - en: """Kubernetes API endpoint URL.""" - zh: """指定 Kubernetes API Server。如有多个 Server 使用逗号 , 分隔。 -当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。""" - } - label { - en: "Cluster k8s ApiServer" - zh: "K8s 服务地址" - } - } +rpc_port_discovery.desc: +"""manual: discover ports by tcp_server_port.
+stateless: discover ports in a stateless manner, using the following algorithm. +If node name is emqxN@127.0.0.1, where the N is an integer, +then the listening port will be 5370 + N.""" - cluster_k8s_service_name { - desc { - en: """EMQX broker service name.""" - zh: """指定 Kubernetes 中 EMQX 的服务名。 -当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。""" - } - label { - en: "K8s Service Name" - zh: "K8s 服务别名" - } - } +rpc_port_discovery.label: +"""RRC Port Discovery""" - cluster_k8s_address_type { - desc { - en: """Address type used for connecting to the discovered nodes. +log_overload_kill_restart_after.desc: +"""The handler restarts automatically after a delay in the event of termination, unless the value `infinity` is set, which blocks any subsequent restarts.""" + +log_overload_kill_restart_after.label: +"""Handler Restart Timer""" + +log_file_handler_max_size.desc: +"""This parameter controls log file rotation. The value `infinity` means the log file will grow indefinitely, otherwise the log file will be rotated once it reaches `max_size` in bytes.""" + +log_file_handler_max_size.label: +"""Rotation Size""" + +desc_log_file_handler.desc: +"""Log handler that prints log events to files.""" + +desc_log_file_handler.label: +"""Files Log Handler""" + +rpc_socket_keepalive_count.desc: +"""How many times the keepalive probe message can fail to receive a reply +until the RPC connection is considered lost.""" + +rpc_socket_keepalive_count.label: +"""RPC Socket Keepalive Count""" + +cluster_etcd_server.desc: +"""List of endpoint URLs of the etcd cluster""" + +cluster_etcd_server.label: +"""Cluster Etcd Server""" + +db_backend.desc: +"""Select the backend for the embedded database.
+rlog is the default backend, +that is suitable for very large clusters.
+mnesia is a backend that offers decent performance in small clusters.""" + +db_backend.label: +"""DB Backend""" + +desc_authorization.desc: +"""Settings that control client authorization.""" + +desc_authorization.label: +"""Authorization""" + +cluster_etcd_ssl.desc: +"""Options for the TLS connection to the etcd cluster.""" + +cluster_etcd_ssl.label: +"""Cluster Etcd SSL Option""" + +rpc_insecure_fallback.desc: +"""Enable compatibility with old RPC authentication.""" + +rpc_insecure_fallback.label: +"""RPC insecure fallback""" + +cluster_mcast_buffer.desc: +"""Size of the user-level buffer.""" + +cluster_mcast_buffer.label: +"""Cluster Muticast Buffer""" + +rpc_authentication_timeout.desc: +"""Timeout for the remote node authentication.""" + +rpc_authentication_timeout.label: +"""RPC Authentication Timeout""" + +cluster_call_retry_interval.desc: +"""Time interval to retry after a failed call.""" + +cluster_call_retry_interval.label: +"""Cluster Call Retry Interval""" + +cluster_mcast_sndbuf.desc: +"""Size of the kernel-level buffer for outgoing datagrams.""" + +cluster_mcast_sndbuf.label: +"""Cluster Muticast Sendbuf""" + +rpc_driver.desc: +"""Transport protocol used for inter-broker communication""" + +rpc_driver.label: +"""RPC dirver""" + +max_ets_tables.desc: +"""Max number of ETS tables""" + +max_ets_tables.label: +"""Max number of ETS tables""" + +desc_db.desc: +"""Settings for the embedded database.""" + +desc_db.label: +"""Database""" + +desc_cluster_etcd.desc: +"""Service discovery using 'etcd' service.""" + +desc_cluster_etcd.label: +"""Cluster Etcd""" + +cluster_name.desc: +"""Human-friendly name of the EMQX cluster.""" + +cluster_name.label: +"""Cluster Name""" + +log_rotation_enable.desc: +"""Enable log rotation feature.""" + +log_rotation_enable.label: +"""Rotation Enable""" + +cluster_call_cleanup_interval.desc: +"""Time interval to clear completed but stale transactions. +Ensure that the number of completed transactions is less than the max_history.""" + +cluster_call_cleanup_interval.label: +"""Clean Up Interval""" + +desc_cluster_static.desc: +"""Service discovery via static nodes. +The new node joins the cluster by connecting to one of the bootstrap nodes.""" + +desc_cluster_static.label: +"""Cluster Static""" + +db_default_shard_transport.desc: +"""Defines the default transport for pushing transaction logs.
+This may be overridden on a per-shard basis in db.shard_transports. +gen_rpc uses the gen_rpc library, +distr uses the Erlang distribution.""" + +db_default_shard_transport.label: +"""Default Shard Transport""" + +cluster_static_seeds.desc: +"""List EMQX node names in the static cluster. See node.name.""" + +cluster_static_seeds.label: +"""Cluster Static Seeds""" + +log_overload_kill_qlen.desc: +"""Maximum allowed queue length.""" + +log_overload_kill_qlen.label: +"""Max Queue Length""" + +node_backtrace_depth.desc: +"""Maximum depth of the call stack printed in error messages and +process_info.""" + +node_backtrace_depth.label: +"""BackTrace Depth""" + +desc_log_burst_limit.desc: +"""Large bursts of log events produced in a short time can potentially cause problems, such as: + - Log files grow very large + - Log files are rotated too quickly, and useful information gets overwritten + - Overall performance impact on the system + +Log burst limit feature can temporarily disable logging to avoid these issues.""" + +desc_log_burst_limit.label: +"""Log Burst Limit""" + +common_handler_enable.desc: +"""Enable this log handler.""" + +common_handler_enable.label: +"""Enable Log Handler""" + +cluster_k8s_service_name.desc: +"""EMQX broker service name.""" + +cluster_k8s_service_name.label: +"""K8s Service Name""" + +log_rotation_count.desc: +"""Maximum number of log files.""" + +log_rotation_count.label: +"""Max Log Files Number""" + +node_cookie.desc: +"""Secret cookie is a random string that should be the same on all nodes in +the given EMQX cluster, but unique per EMQX cluster. It is used to prevent EMQX nodes that +belong to different clusters from accidentally connecting to each other.""" + +node_cookie.label: +"""Node Cookie""" + +db_role.desc: +"""Select a node role.
+core nodes provide durability of the data, and take care of writes. +It is recommended to place core nodes in different racks or different availability zones.
+replicant nodes are ephemeral worker nodes. Removing them from the cluster +doesn't affect database redundancy
+It is recommended to have more replicant nodes than core nodes.
+Note: this parameter only takes effect when the backend is set +to rlog.""" + +db_role.label: +"""DB Role""" + +rpc_tcp_server_port.desc: +"""Listening port used by RPC local service.
+Note that this config only takes effect when rpc.port_discovery is set to manual.""" + +rpc_tcp_server_port.label: +"""RPC TCP Server Port""" + +desc_console_handler.desc: +"""Log handler that prints log events to the EMQX console.""" + +desc_console_handler.label: +"""Console Handler""" + +node_applications.desc: +"""List of Erlang applications that shall be rebooted when the EMQX broker joins the cluster.""" + +node_applications.label: +"""Application""" + +log_burst_limit_max_count.desc: +"""Maximum number of log events to handle within a `window_time` interval. After the limit is reached, successive events are dropped until the end of the `window_time`.""" + +log_burst_limit_max_count.label: +"""Events Number""" + +rpc_tcp_client_num.desc: +"""Set the maximum number of RPC communication channels initiated by this node to each remote node.""" + +rpc_tcp_client_num.label: +"""RPC TCP Client Num""" + +cluster_k8s_address_type.desc: +"""Address type used for connecting to the discovered nodes. Setting cluster.k8s.address_type to ip will make EMQX to discover IP addresses of peer nodes from Kubernetes API.""" - zh: """当使用 k8s 方式集群时，address_type 用来从 Kubernetes 接口的应答里获取什么形式的 Host 列表。 -指定 cluster.k8s.address_type 为 ip，则将从 Kubernetes 接口中获取集群中其他节点 -的IP地址。""" - } - label { - en: "K8s Address Type" - zh: "K8s 地址类型" - } - } +cluster_k8s_address_type.label: +"""K8s Address Type""" - cluster_k8s_namespace { - desc { - en: """Kubernetes namespace.""" - zh: """当使用 k8s 方式并且 cluster.k8s.address_type 指定为 dns 类型时， -可设置 emqx 节点名的命名空间。与 cluster.k8s.suffix 一起使用用以拼接得到节点名列表。""" - } - label { - en: "K8s Namespace" - zh: "K8s 命名空间" - } - } +rpc_socket_sndbuf.desc: +"""TCP tuning parameters. TCP sending buffer size.""" - cluster_k8s_suffix { - desc { - en: """Node name suffix.
-Note: this parameter is only relevant when address_type is dns -or hostname.""" - zh: """当使用 k8s 方式并且 cluster.k8s.address_type 指定为 dns 类型时，可设置 emqx 节点名的后缀。 -与 cluster.k8s.namespace 一起使用用以拼接得到节点名列表。""" - } - label { - en: "K8s Suffix" - zh: "K8s 前缀" - } - } +rpc_socket_sndbuf.label: +"""RPC Socket Sndbuf""" - node_name { - desc { - en: """Unique name of the EMQX node. It must follow %name%@FQDN or -%name%@IPv4 format.""" - zh: """节点名。格式为 \@\。其中 可以是 IP 地址，也可以是 FQDN。 -详见 http://erlang.org/doc/reference_manual/distributed.html。""" - } - label { - en: "Node Name" - zh: "节点名" - } - } +cluster_mcast_ttl.desc: +"""Time-to-live (TTL) for the outgoing UDP datagrams.""" - node_cookie { - desc { - en: """Secret cookie is a random string that should be the same on all nodes in -the given EMQX cluster, but unique per EMQX cluster. It is used to prevent EMQX nodes that -belong to different clusters from accidentally connecting to each other.""" - zh: """分布式 Erlang 集群使用的 cookie 值。集群间保持一致""" - } - label { - en: "Node Cookie" - zh: "节点 Cookie" - } - } +cluster_mcast_ttl.label: +"""Cluster Multicast TTL""" - node_data_dir { - desc { - en: """Path to the persistent data directory.
+db_core_nodes.desc: +"""List of core nodes that the replicant will connect to.
+Note: this parameter only takes effect when the backend is set +to rlog and the role is set to replicant.
+This value needs to be defined for manual or static cluster discovery mechanisms.
+If an automatic cluster discovery mechanism is being used (such as etcd), +there is no need to set this value.""" + +db_core_nodes.label: +"""Db Core Node""" + +log_file_handler_file.desc: +"""Name the log file.""" + +log_file_handler_file.label: +"""Log File Name""" + +node_dist_net_ticktime.desc: +"""This is the approximate time an EMQX node may be unresponsive until it is considered down and thereby disconnected.""" + +node_dist_net_ticktime.label: +"""Dist Net TickTime""" + +desc_cluster_k8s.desc: +"""Service discovery via Kubernetes API server.""" + +desc_cluster_k8s.label: +"""Cluster Kubernetes""" + +desc_cluster_mcast.desc: +"""Service discovery via UDP multicast.""" + +desc_cluster_mcast.label: +"""Cluster Multicast""" + +rpc_cacertfile.desc: +"""Path to certification authority TLS certificate file used to validate rpc.certfile.
+Note: certificates of all nodes in the cluster must be signed by the same CA.""" + +rpc_cacertfile.label: +"""RPC Cacertfile""" + +desc_node.desc: +"""Node name, cookie, config & data directories and the Erlang virtual machine (BEAM) boot parameters.""" + +desc_node.label: +"""Node""" + +cluster_k8s_apiserver.desc: +"""Kubernetes API endpoint URL.""" + +cluster_k8s_apiserver.label: +"""Cluster k8s ApiServer""" + +common_handler_supervisor_reports.desc: +"""Type of supervisor reports that are logged. Defaults to error
+ - error: only log errors in the Erlang processes
. + - progress: log process startup.""" + +common_handler_supervisor_reports.label: +"""Report Type""" + +node_data_dir.desc: +"""Path to the persistent data directory.
Possible auto-created subdirectories are:
- `mnesia/`: EMQX's built-in database directory.
For example, `mnesia/emqx@127.0.0.1`.
@@ -418,822 +463,85 @@ the old dir should be deleted first.
**NOTE**: One data dir cannot be shared by two or more EMQX nodes.""" - zh: """节点数据存放目录，可能会自动创建的子目录如下：
-- `mnesia/`。EMQX的内置数据库目录。例如，`mnesia/emqx@127.0.0.1`。
-如果节点要被重新命名（例如，`emqx@10.0.1.1`）。旧目录应该首先被删除。
-- `configs`。在启动时生成的配置，以及集群/本地覆盖的配置。
-- `patches`: 热补丁文件将被放在这里。
-- `trace`: 日志跟踪文件。
+node_data_dir.label: +"""Node Data Dir""" -**注意**: 一个数据dir不能被两个或更多的EMQX节点同时使用。""" +cluster_k8s_suffix.desc: +"""Node name suffix.
+Note: this parameter is only relevant when address_type is dns +or hostname.""" - } - label { - en: "Node Data Dir" - zh: "节点数据目录" - } - } +cluster_k8s_suffix.label: +"""K8s Suffix""" - node_global_gc_interval { - desc { - en: """Periodic garbage collection interval. Set to disabled to have it disabled.""" - zh: """系统调优参数，设置节点运行多久强制进行一次全局垃圾回收。禁用设置为 disabled。""" - } - label { - en: "Global GC Interval" - zh: "全局垃圾回收" - } - } +db_rpc_module.desc: +"""Protocol used for pushing transaction logs to the replicant nodes.""" - node_crash_dump_file { - desc { - en: """Location of the crash dump file.""" - zh: """设置 Erlang crash_dump 文件的存储路径和文件名。""" - } - label { - en: "Crash Dump File" - zh: "节点崩溃时的Dump文件" - } - } +db_rpc_module.label: +"""RPC Module""" - node_crash_dump_seconds { - desc { - en: """This variable gives the number of seconds that the emulator is allowed to spend writing a crash dump. When the given number of seconds have elapsed, the emulator is terminated.
-- If setting to 0 seconds, the runtime system does not even attempt to write the crash dump file. It only terminates.
-- If setting to a positive value S, wait for S seconds to complete the crash dump file and then terminates the runtime system with a SIGALRM signal.
-- A negative value causes the termination of the runtime system to wait indefinitely until the crash dump file has been completely written.""" +cluster_etcd_prefix.desc: +"""Key prefix used for EMQX service discovery.""" - zh: """该配置给出了运行时系统允许花费的写入崩溃转储的秒数。当给定的秒数已经过去，运行时系统将被终止。
-- 如果设置为0秒，运行时会立即终止，不会尝试写入崩溃转储文件。
-- 如果设置为一个正数 S，节点会等待 S 秒来完成崩溃转储文件，然后用SIGALRM信号终止运行时系统。
-- 如果设置为一个负值导致运行时系统的终止等待无限期地直到崩溃转储文件已经完全写入。""" - } - label { - en: "Crash Dump Seconds" - zh: "保存崩溃文件最长时间" - } - } +cluster_etcd_prefix.label: +"""Cluster Etcd Prefix""" - node_crash_dump_bytes { - desc { - en: """This variable sets the maximum size of a crash dump file in bytes. -The crash dump will be truncated if this limit is exceeded. -If setting it to 0, the runtime system does not even attempt to write a crash dump file.""" +cluster_mcast_iface.desc: +"""Local IP address the node discovery service needs to bind to.""" - zh: """限制崩溃文件的大小，当崩溃时节点内存太大， -如果为了保存现场，需要全部存到崩溃文件中，此处限制最多能保存多大的文件。 -如果超过此限制，崩溃转储将被截断。如果设置为0，系统不会尝试写入崩溃转储文件。""" - } - label { - en: "Crash Dump Bytes" - zh: "崩溃文件最大容量" - } - } +cluster_mcast_iface.label: +"""Cluster Multicast Interface""" - node_dist_net_ticktime { - desc { - en: """This is the approximate time an EMQX node may be unresponsive until it is considered down and thereby disconnected.""" - zh: """系统调优参数，此配置将覆盖 vm.args 文件里的 -kernel net_ticktime 参数。当一个节点持续无响应多久之后，认为其已经宕机并断开连接。""" - } - label { - en: "Dist Net TickTime" - zh: "节点间心跳间隔" - } - } +log_burst_limit_window_time.desc: +"""See max_count.""" - node_backtrace_depth { - desc { - en: """Maximum depth of the call stack printed in error messages and -process_info.""" - zh: """错误信息中打印的最大堆栈层数""" - } - label { - en: "BackTrace Depth" - zh: "最大堆栈导数" - } - } +log_burst_limit_window_time.label: +"""Window Time""" - # TODO: check if this is still needed - node_applications { - desc { - en: """List of Erlang applications that shall be rebooted when the EMQX broker joins the cluster.""" - zh: """当新EMQX 加入集群时，应重启的Erlang应用程序的列表。""" - } - label { - en: "Application" - zh: "应用" - } - } +cluster_dns_record_type.desc: +"""DNS record type.""" - # deprecated, TODO: remove - node_etc_dir { - desc { - en: """etc dir for the node""" - zh: """etc 存放目录""" - } - label { - en: "Etc Dir" - zh: "Etc 目录" - } - } +cluster_dns_record_type.label: +"""DNS Record Type""" - db_backend { - desc { - en: """Select the backend for the embedded database.
-rlog is the default backend, -that is suitable for very large clusters.
-mnesia is a backend that offers decent performance in small clusters.""" - zh: """配置后端数据库驱动，默认值为 rlog 它适用于大规模的集群。 -mnesia 是备选数据库，适合中小集群。""" - } - label { - en: "DB Backend" - zh: "内置数据库" - } - } +cluster_autoclean.desc: +"""Remove disconnected nodes from the cluster after this interval.""" - db_role { - desc { - en: """Select a node role.
-core nodes provide durability of the data, and take care of writes. -It is recommended to place core nodes in different racks or different availability zones.
-replicant nodes are ephemeral worker nodes. Removing them from the cluster -doesn't affect database redundancy
-It is recommended to have more replicant nodes than core nodes.
-Note: this parameter only takes effect when the backend is set -to rlog.""" - zh: """选择节点的角色。
-core 节点提供数据的持久性，并负责写入。建议将核心节点放置在不同的机架或不同的可用区。
-repliant 节点是临时工作节点。 从集群中删除它们，不影响数据库冗余
-建议复制节点多于核心节点。
-注意：该参数仅在设置backend时生效到 rlog。""" - } - label { - en: "DB Role" - zh: "数据库角色" - } - } +cluster_autoclean.label: +"""Cluster Auto Clean""" - db_core_nodes { - desc { - en: """List of core nodes that the replicant will connect to.
-Note: this parameter only takes effect when the backend is set -to rlog and the role is set to replicant.
-This value needs to be defined for manual or static cluster discovery mechanisms.
-If an automatic cluster discovery mechanism is being used (such as etcd), -there is no need to set this value.""" - zh: """当前节点连接的核心节点列表。
-注意：该参数仅在设置backend时生效到 rlog -并且设置role为replicant时生效。
-该值需要在手动或静态集群发现机制下设置。
-如果使用了自动集群发现机制（如etcd），则不需要设置该值。""" - } - label { - en: "Db Core Node" - zh: "数据库核心节点" - } - } +process_limit.desc: +"""Maximum number of simultaneously existing processes for this Erlang system. +The actual maximum chosen may be much larger than the Number passed. +For more information, see: https://www.erlang.org/doc/man/erl.html""" - db_rpc_module { - desc { - en: """Protocol used for pushing transaction logs to the replicant nodes.""" - zh: """集群间推送事务日志到复制节点使用的协议。""" - } - label { - en: "RPC Module" - zh: "RPC协议" - } - } +process_limit.label: +"""Erlang Process Limit""" - db_tlog_push_mode { - desc { - en: """In sync mode the core node waits for an ack from the replicant nodes before sending the next -transaction log entry.""" - zh: """同步模式下，核心节点等待复制节点的确认信息，然后再发送下一条事务日志。""" - } - label { - en: "Tlog Push Mode" - zh: "Tlog推送模式" - } - } +max_ports.desc: +"""Maximum number of simultaneously existing ports for this Erlang system. +The actual maximum chosen may be much larger than the Number passed. +For more information, see: https://www.erlang.org/doc/man/erl.html""" - db_default_shard_transport { - desc { - en: """Defines the default transport for pushing transaction logs.
-This may be overridden on a per-shard basis in db.shard_transports. -gen_rpc uses the gen_rpc library, -distr uses the Erlang distribution.""" - zh: """定义用于推送事务日志的默认传输。
-这可以在 db.shard_transports 中基于每个分片被覆盖。 -gen_rpc 使用 gen_rpc 库， -distr 使用 Erlang 发行版。""" - } - label { - en: "Default Shard Transport" - zh: "事务日志传输默认协议" - } - } +max_ports.label: +"""Erlang Port Limit""" - db_shard_transports { - desc { - en: """Allows to tune the transport method used for transaction log replication, on a per-shard basis.
-gen_rpc uses the gen_rpc library, -distr uses the Erlang distribution.
If not specified, -the default is to use the value set in db.default_shard_transport.""" - zh: """允许为每个 shard 下的事务日志复制操作的传输方法进行调优。
-gen_rpc 使用 gen_rpc 库， -distr 使用 Erlang 自带的 rpc 库。
如果未指定， -默认是使用 db.default_shard_transport 中设置的值。""" - } - label { - en: "Shard Transports" - zh: "事务日志传输协议" - } - } +desc_log_rotation.desc: +"""By default, the logs are stored in `./log` directory (for installation from zip file) or in `/var/log/emqx` (for binary installation).
+This section of the configuration controls the number of files kept for each log handler.""" - cluster_call_retry_interval { - desc { - en: """Time interval to retry after a failed call.""" - zh: """当集群间调用出错时，多长时间重试一次。""" - } - label { - en: "Cluster Call Retry Interval" - zh: "重试时间间隔" - } - } +desc_log_rotation.label: +"""Log Rotation""" - cluster_call_max_history { - desc { - en: """Retain the maximum number of completed transactions (for queries).""" - zh: """集群间调用最多保留的历史记录数。只用于排错时查看。""" - } - label { - en: "Cluster Call Max History" - zh: "最大历史记录" - } - } +desc_log_overload_kill.desc: +"""Log overload kill features an overload protection that activates when the log handlers use too much memory or have too many buffered log messages.
+When the overload is detected, the log handler is terminated and restarted after a cooldown period.""" - cluster_call_cleanup_interval { - desc { - en: """Time interval to clear completed but stale transactions. -Ensure that the number of completed transactions is less than the max_history.""" - zh: """清理过期事务的时间间隔""" - } - label { - en: "Clean Up Interval" - zh: "清理间隔" - } - } +desc_log_overload_kill.label: +"""Log Overload Kill""" - rpc_mode { - desc { - en: """In sync mode the sending side waits for the ack from the receiving side.""" - zh: """在 sync 模式下，发送端等待接收端的 ack信号。""" - } - label { - en: "RPC Mode" - zh: "RPC 模式" - } - } - - rpc_driver { - desc { - en: """Transport protocol used for inter-broker communication""" - zh: """集群间通信使用的传输协议。""" - } - label { - en: "RPC dirver" - zh: "RPC 驱动" - } - } - - rpc_async_batch_size { - desc { - en: """The maximum number of batch messages sent in asynchronous mode. - Note that this configuration does not work in synchronous mode.""" - zh: """异步模式下，发送的批量消息的最大数量。""" - } - label { - en: "Async Batch Size" - zh: "异步模式下的批量消息数量" - } - } - - rpc_port_discovery { - desc { - en: """manual: discover ports by tcp_server_port.
-stateless: discover ports in a stateless manner, using the following algorithm. -If node name is emqxN@127.0.0.1, where the N is an integer, -then the listening port will be 5370 + N.""" - zh: """manual: 通过 tcp_server_port 来发现端口。 -
stateless: 使用无状态的方式来发现端口，使用如下算法。如果节点名称是 -emqxN@127.0.0.1, N 是一个数字，那么监听端口就是 5370 + N。""" - } - label { - en: "RRC Port Discovery" - zh: "RPC 端口发现策略" - } - } - - rpc_tcp_server_port { - desc { - en: """Listening port used by RPC local service.
-Note that this config only takes effect when rpc.port_discovery is set to manual.""" - zh: """RPC 本地服务使用的 TCP 端口。
-只有当 rpc.port_discovery 设置为 manual 时，此配置才会生效。""" - } - label { - en: "RPC TCP Server Port" - zh: "RPC TCP 服务监听端口" - } - } - - rpc_ssl_server_port { - desc { - en: """Listening port used by RPC local service.
-Note that this config only takes effect when rpc.port_discovery is set to manual -and driver is set to ssl.""" - zh: """RPC 本地服务使用的监听SSL端口。
-只有当 rpc.port_discovery 设置为 manual 且 dirver 设置为 ssl， -此配置才会生效。""" - } - label { - en: "RPC SSL Server Port" - zh: "RPC SSL 服务监听端口" - } - } - - rpc_tcp_client_num { - desc { - en: """Set the maximum number of RPC communication channels initiated by this node to each remote node.""" - zh: """设置本节点与远程节点之间的 RPC 通信通道的最大数量。""" - } - label { - en: "RPC TCP Client Num" - zh: "RPC TCP 客户端数量" - } - } - - rpc_connect_timeout { - desc { - en: """Timeout for establishing an RPC connection.""" - zh: """建立 RPC 连接的超时时间。""" - } - label { - en: "RPC Connect Timeout" - zh: "RPC 连接超时时间" - } - } - - rpc_certfile { - desc { - en: """Path to TLS certificate file used to validate identity of the cluster nodes. -Note that this config only takes effect when rpc.driver is set to ssl.""" - - zh: """TLS 证书文件的路径，用于验证集群节点的身份。 -只有当 rpc.driver 设置为 ssl 时，此配置才会生效。""" - } - label { - en: "RPC Certfile" - zh: "RPC 证书文件" - } - } - - rpc_keyfile { - desc { - en: """Path to the private key file for the rpc.certfile.
-Note: contents of this file are secret, so it's necessary to set permissions to 600.""" - zh: """rpc.certfile 的私钥文件的路径。
-注意：此文件内容是私钥，所以需要设置权限为 600。""" - } - label { - en: "RPC Keyfile" - zh: "RPC 私钥文件" - } - } - - rpc_cacertfile { - desc { - en: """Path to certification authority TLS certificate file used to validate rpc.certfile.
-Note: certificates of all nodes in the cluster must be signed by the same CA.""" - zh: """验证 rpc.certfile 的 CA 证书文件的路径。
-注意：集群中所有节点的证书必须使用同一个 CA 签发。""" - } - label { - en: "RPC Cacertfile" - zh: "RPC CA 证书文件" - } - } - - rpc_send_timeout { - desc { - en: """Timeout for sending the RPC request.""" - zh: """发送 RPC 请求的超时时间。""" - } - label { - en: "RPC Send Timeout" - zh: "RPC 发送超时时间" - } - } - - rpc_authentication_timeout { - desc { - en: """Timeout for the remote node authentication.""" - zh: """远程节点认证的超时时间。""" - } - label { - en: "RPC Authentication Timeout" - zh: "RPC 认证超时时间" - } - } - - rpc_call_receive_timeout { - desc { - en: """Timeout for the reply to a synchronous RPC.""" - zh: """同步 RPC 的回复超时时间。""" - } - label { - en: "RPC Call Receive Timeout" - zh: "RPC 调用接收超时时间" - } - } - - rpc_socket_keepalive_idle { - desc { - en: """How long the connections between the brokers should remain open after the last message is sent.""" - zh: """broker 之间的连接在最后一条消息发送后保持打开的时间。""" - } - label { - en: "RPC Socket Keepalive Idle" - zh: "RPC Socket Keepalive Idle" - } - } - - rpc_socket_keepalive_interval { - desc { - en: """The interval between keepalive messages.""" - zh: """keepalive 消息的间隔。""" - } - label { - en: "RPC Socket Keepalive Interval" - zh: "RPC Socket Keepalive 间隔" - } - } - - rpc_socket_keepalive_count { - desc { - en: """How many times the keepalive probe message can fail to receive a reply -until the RPC connection is considered lost.""" - zh: """keepalive 探测消息发送失败的次数，直到 RPC 连接被认为已经断开。""" - } - label { - en: "RPC Socket Keepalive Count" - zh: "RPC Socket Keepalive 次数" - } - } - - rpc_socket_sndbuf { - desc { - en: """TCP tuning parameters. TCP sending buffer size.""" - zh: """TCP 调节参数。TCP 发送缓冲区大小。""" - } - label { - en: "RPC Socket Sndbuf" - zh: "RPC 套接字发送缓冲区大小" - } - } - - rpc_socket_recbuf { - desc { - en: """TCP tuning parameters. TCP receiving buffer size.""" - zh: """TCP 调节参数。TCP 接收缓冲区大小。""" - } - label { - en: "RPC Socket Recbuf" - zh: "RPC 套接字接收缓冲区大小" - } - } - - rpc_socket_buffer { - desc { - en: """TCP tuning parameters. Socket buffer size in user mode.""" - zh: """TCP 调节参数。用户模式套接字缓冲区大小。""" - } - label { - en: "RPC Socket Buffer" - zh: "RPC 套接字缓冲区大小" - } - } - - rpc_insecure_fallback { - desc { - en: """Enable compatibility with old RPC authentication.""" - zh: """兼容旧的无鉴权模式""" - } - label { - en: "RPC insecure fallback" - zh: "向后兼容旧的无鉴权模式" - } - } - - log_file_handlers { - desc { - en: """File-based log handlers.""" - zh: """输出到文件的日志处理进程列表""" - } - label { - en: "File Handler" - zh: "File Handler" - } - } - - common_handler_enable { - desc { - en: """Enable this log handler.""" - zh: """启用此日志处理进程。""" - } - label { - en: "Enable Log Handler" - zh: "启用日志处理进程" - } - } - - common_handler_level { - desc { - en: """The log level for the current log handler. -Defaults to warning.""" - zh: """当前日志处理进程的日志级别。 -默认为 warning 级别。""" - } - label { - en: "Log Level" - zh: "日志级别" - } - } - - common_handler_time_offset { - desc { - en: """The time offset to be used when formatting the timestamp. -Can be one of: - - system: the time offset used by the local system - - utc: the UTC time offset - - +-[hh]:[mm]: user specified time offset, such as "-02:00" or "+00:00" -Defaults to: system.""" - zh: """日志中的时间戳使用的时间偏移量。 -可选值为： - - system: 本地系统使用的时区偏移量 - - utc: 0 时区的偏移量 - - +-[hh]:[mm]: 自定义偏移量，比如 "-02:00" 或者 "+00:00" -默认值为本地系统的时区偏移量：system。""" - } - label { - en: "Time Offset" - zh: "时间偏移量" - } - } - - common_handler_chars_limit { - desc { - en: """Set the maximum length of a single log message. If this length is exceeded, the log message will be truncated. -NOTE: Restrict char limiter if formatter is JSON , it will get a truncated incomplete JSON data, which is not recommended.""" - zh: """设置单个日志消息的最大长度。 如果超过此长度，则日志消息将被截断。最小可设置的长度为100。 -注意：如果日志格式为 JSON，限制字符长度可能会导致截断不完整的 JSON 数据。""" - } - label { - en: "Single Log Max Length" - zh: "单条日志长度限制" - } - } - - common_handler_formatter { - desc { - en: """Choose log formatter. text for free text, and json for structured logging.""" - zh: """选择日志格式类型。 text 用于纯文本，json 用于结构化日志记录。""" - } - label { - en: "Log Formatter" - zh: "日志格式类型" - } - } - - common_handler_single_line { - desc { - en: """Print logs in a single line if set to true. Otherwise, log messages may span multiple lines.""" - zh: """如果设置为 true，则单行打印日志。 否则，日志消息可能跨越多行。""" - } - label { - en: "Single Line Mode" - zh: "单行模式" - } - } - - common_handler_sync_mode_qlen { - desc { - en: """As long as the number of buffered log events is lower than this value, -all log events are handled asynchronously. This means that the client process sending the log event, -by calling a log function in the Logger API, does not wait for a response from the handler -but continues executing immediately after the event is sent. -It is not affected by the time it takes the handler to print the event to the log device. -If the message queue grows larger than this value, -the handler starts handling log events synchronously instead, -meaning that the client process sending the event must wait for a response. -When the handler reduces the message queue to a level below the sync_mode_qlen threshold, -asynchronous operation is resumed.""" - zh: """只要缓冲的日志事件的数量低于这个值，所有的日志事件都会被异步处理。 -这意味着，日志落地速度不会影响正常的业务进程，因为它们不需要等待日志处理进程的响应。 -如果消息队列的增长超过了这个值，处理程序开始同步处理日志事件。也就是说，发送事件的客户进程必须等待响应。 -当处理程序将消息队列减少到低于sync_mode_qlen阈值的水平时，异步操作就会恢复。 -默认为100条信息，当等待的日志事件大于100条时，就开始同步处理日志。""" - } - label { - en: "Queue Length before Entering Sync Mode" - zh: "进入异步模式的队列长度" - } - } - - common_handler_drop_mode_qlen { - desc { - en: """When the number of buffered log events is larger than this value, the new log events are dropped. -When drop mode is activated or deactivated, a message is printed in the logs.""" - zh: """当缓冲的日志事件数大于此值时，新的日志事件将被丢弃。起到过载保护的功能。 -为了使过载保护算法正常工作必须要： sync_mode_qlen =< drop_mode_qlen =< flush_qlen 且 drop_mode_qlen > 1 -要禁用某些模式，请执行以下操作。 -- 如果sync_mode_qlen被设置为0，所有的日志事件都被同步处理。也就是说，异步日志被禁用。 -- 如果sync_mode_qlen被设置为与drop_mode_qlen相同的值，同步模式被禁用。也就是说，处理程序总是以异步模式运行，除非调用drop或flushing。 -- 如果drop_mode_qlen被设置为与flush_qlen相同的值，则drop模式被禁用，永远不会发生。""" - } - label { - en: "Queue Length before Entering Drop Mode" - zh: "进入丢弃模式的队列长度" - } - } - - common_handler_flush_qlen { - desc { - en: """If the number of buffered log events grows larger than this threshold, a flush (delete) operation takes place. -To flush events, the handler discards the buffered log messages without logging.""" - zh: """如果缓冲日志事件的数量增长大于此阈值，则会发生冲刷（删除）操作。 日志处理进程会丢弃缓冲的日志消息。 -来缓解自身不会由于内存瀑涨而影响其它业务进程。日志内容会提醒有多少事件被删除。""" - } - label { - en: "Flush Threshold" - zh: "冲刷阈值" - } - } - - common_handler_supervisor_reports { - desc { - en: """Type of supervisor reports that are logged. Defaults to error
- - error: only log errors in the Erlang processes
. - - progress: log process startup.""" - - zh: """Supervisor 报告的类型。默认为 error 类型。
- - error：仅记录 Erlang 进程中的错误。 - - progress：除了 error 信息外，还需要记录进程启动的详细信息。""" - } - label { - en: "Report Type" - zh: "报告类型" - } - } - - common_handler_max_depth { - desc { - en: """Maximum depth for Erlang term log formatting and Erlang process message queue inspection.""" - zh: """Erlang 内部格式日志格式化和 Erlang 进程消息队列检查的最大深度。""" - } - label { - en: "Max Depth" - zh: "最大深度" - } - } - - log_file_handler_file { - desc { - en: """Name the log file.""" - zh: """日志文件路径及名字。""" - } - label { - en: "Log File Name" - zh: "日志文件名字" - } - } - - log_file_handler_max_size { - desc { - en: """This parameter controls log file rotation. The value `infinity` means the log file will grow indefinitely, otherwise the log file will be rotated once it reaches `max_size` in bytes.""" - zh: """此参数控制日志文件轮换。 `infinity` 意味着日志文件将无限增长，否则日志文件将在达到 `max_size`（以字节为单位）时进行轮换。 -与 rotation count配合使用。如果 counter 为 10，则是10个文件轮换。""" - } - label { - en: "Rotation Size" - zh: "日志文件轮换大小" - } - } - - log_rotation_enable { - desc { - en: """Enable log rotation feature.""" - zh: """启用日志轮换功能。启动后生成日志文件后缀会加上对应的索引数字，比如：log/emqx.log.1。 -系统会默认生成*.siz/*.idx用于记录日志位置，请不要手动修改这两个文件。""" - } - label { - en: "Rotation Enable" - zh: "日志轮换" - } - } - - log_rotation_count { - desc { - en: """Maximum number of log files.""" - zh: """轮换的最大日志文件数。""" - } - label { - en: "Max Log Files Number" - zh: "最大日志文件数" - } - } - - log_overload_kill_enable { - desc { - en: """Enable log handler overload kill feature.""" - zh: """日志处理进程过载时为保护自己节点其它的业务能正常，强制杀死日志处理进程。""" - } - label { - en: "Log Handler Overload Kill" - zh: "日志处理进程过载保护" - } - } - - log_overload_kill_mem_size { - desc { - en: """Maximum memory size that the log handler process is allowed to use.""" - zh: """日志处理进程允许使用的最大内存。""" - } - label { - en: "Log Handler Max Memory Size" - zh: "日志处理进程允许使用的最大内存" - } - } - - log_overload_kill_qlen { - desc { - en: """Maximum allowed queue length.""" - zh: """允许的最大队列长度。""" - } - label { - en: "Max Queue Length" - zh: "最大队列长度" - } - } - - log_overload_kill_restart_after { - desc { - en: """The handler restarts automatically after a delay in the event of termination, unless the value `infinity` is set, which blocks any subsequent restarts.""" - zh: """处理进程停止后，会在该延迟时间后自动重新启动。除非该值设置为 infinity，这会阻止任何后续的重启。""" - } - label { - en: "Handler Restart Timer" - zh: "处理进程重启延迟" - } - } - - log_burst_limit_enable { - desc { - en: """Enable log burst control feature.""" - zh: """启用日志限流保护机制。""" - } - label { - en: "Enable Burst" - zh: "日志限流保护" - } - } - - log_burst_limit_max_count { - desc { - en: """Maximum number of log events to handle within a `window_time` interval. After the limit is reached, successive events are dropped until the end of the `window_time`.""" - zh: """在 `window_time` 间隔内处理的最大日志事件数。 达到限制后，将丢弃连续事件，直到 `window_time` 结束。""" - } - label { - en: "Events Number" - zh: "日志事件数" - } - } - - log_burst_limit_window_time { - desc { - en: """See max_count.""" - zh: """参考 max_count。""" - } - label { - en: "Window Time" - zh: "Window Time" - } - } - - authorization { - desc { - en: """Authorization a.k.a. ACL.
+authorization.desc: +"""Authorization a.k.a. ACL.
In EMQX, MQTT client access control is extremely flexible.
An out-of-the-box set of authorization data sources are supported. For example,
@@ -1243,221 +551,238 @@ natively in the EMQX node;
'http' source to make EMQX call an external HTTP API to make the decision;
'PostgreSQL' etc. to look up clients or rules from external databases""" - zh: """授权（ACL）。EMQX 支持完整的客户端访问控制（ACL）。""" - } - label { - en: "Authorization" - zh: "授权" - } - } +authorization.label: +"""Authorization""" - desc_cluster { - desc { - en: """EMQX nodes can form a cluster to scale up the total capacity.
- Here holds the configs to instruct how individual nodes can discover each other.""" - zh: """EMQX 节点可以组成一个集群，以提高总容量。
这里指定了节点之间如何连接。""" - } - label { - en: "Cluster" - zh: "集群" - } - } +rpc_socket_keepalive_idle.desc: +"""How long the connections between the brokers should remain open after the last message is sent.""" - desc_cluster_static { - desc { - en: """Service discovery via static nodes. -The new node joins the cluster by connecting to one of the bootstrap nodes.""" - zh: """静态节点服务发现。新节点通过连接一个节点来加入集群。""" - } - label { - en: "Cluster Static" - zh: "静态节点服务发现" - } - } +rpc_socket_keepalive_idle.label: +"""RPC Socket Keepalive Idle""" - desc_cluster_mcast { - desc { - en: """Service discovery via UDP multicast.""" - zh: """UDP 组播服务发现。""" - } - label { - en: "Cluster Multicast" - zh: "UDP 组播服务发现" - } - } +desc_cluster_call.desc: +"""Options for the 'cluster call' feature that allows to execute a callback on all nodes in the cluster.""" - desc_cluster_dns { - desc { - en: """Service discovery via DNS SRV records.""" - zh: """DNS SRV 记录服务发现。""" - } - label { - en: "Cluster DNS" - zh: "DNS SRV 记录服务发现" - } - } +desc_cluster_call.label: +"""Cluster Call""" - desc_cluster_etcd { - desc { - en: """Service discovery using 'etcd' service.""" - zh: """使用 'etcd' 服务的服务发现。""" - } - label { - en: "Cluster Etcd" - zh: "'etcd' 服务的服务发现" - } - } +cluster_mcast_ports.desc: +"""List of UDP ports used for service discovery.
+Note: probe messages are broadcast to all the specified ports.""" - desc_cluster_k8s { - desc { - en: """Service discovery via Kubernetes API server.""" - zh: """Kubernetes 服务发现。""" - } - label { - en: "Cluster Kubernetes" - zh: "Kubernetes 服务发现" - } - } +cluster_mcast_ports.label: +"""Cluster Multicast Ports""" - desc_node { - desc { - en: """Node name, cookie, config & data directories and the Erlang virtual machine (BEAM) boot parameters.""" - zh: """节点名称、Cookie、配置文件、数据目录和 Erlang 虚拟机（BEAM）启动参数。""" - } - label { - en: "Node" - zh: "节点" - } - } +log_overload_kill_mem_size.desc: +"""Maximum memory size that the log handler process is allowed to use.""" - desc_db { - desc { - en: """Settings for the embedded database.""" - zh: """内置数据库的配置。""" - } - label { - en: "Database" - zh: "数据库" - } - } +log_overload_kill_mem_size.label: +"""Log Handler Max Memory Size""" - desc_cluster_call { - desc { - en: """Options for the 'cluster call' feature that allows to execute a callback on all nodes in the cluster.""" - zh: """集群调用功能的选项。""" - } - label { - en: "Cluster Call" - zh: "集群调用" - } - } +rpc_connect_timeout.desc: +"""Timeout for establishing an RPC connection.""" - desc_rpc { - desc { - en: """EMQX uses a library called gen_rpc for inter-broker communication.
+rpc_connect_timeout.label: +"""RPC Connect Timeout""" + +cluster_etcd_node_ttl.desc: +"""Expiration time of the etcd key associated with the node. +It is refreshed automatically, as long as the node is alive.""" + +cluster_etcd_node_ttl.label: +"""Cluster Etcd Node TTL""" + +rpc_call_receive_timeout.desc: +"""Timeout for the reply to a synchronous RPC.""" + +rpc_call_receive_timeout.label: +"""RPC Call Receive Timeout""" + +rpc_socket_recbuf.desc: +"""TCP tuning parameters. TCP receiving buffer size.""" + +rpc_socket_recbuf.label: +"""RPC Socket Recbuf""" + +db_tlog_push_mode.desc: +"""In sync mode the core node waits for an ack from the replicant nodes before sending the next +transaction log entry.""" + +db_tlog_push_mode.label: +"""Tlog Push Mode""" + +node_crash_dump_bytes.desc: +"""This variable sets the maximum size of a crash dump file in bytes. +The crash dump will be truncated if this limit is exceeded. +If setting it to 0, the runtime system does not even attempt to write a crash dump file.""" + +node_crash_dump_bytes.label: +"""Crash Dump Bytes""" + +rpc_certfile.desc: +"""Path to TLS certificate file used to validate identity of the cluster nodes. +Note that this config only takes effect when rpc.driver is set to ssl.""" + +rpc_certfile.label: +"""RPC Certfile""" + +node_crash_dump_seconds.desc: +"""This variable gives the number of seconds that the emulator is allowed to spend writing a crash dump. When the given number of seconds have elapsed, the emulator is terminated.
+- If setting to 0 seconds, the runtime system does not even attempt to write the crash dump file. It only terminates.
+- If setting to a positive value S, wait for S seconds to complete the crash dump file and then terminates the runtime system with a SIGALRM signal.
+- A negative value causes the termination of the runtime system to wait indefinitely until the crash dump file has been completely written.""" + +node_crash_dump_seconds.label: +"""Crash Dump Seconds""" + +log_file_handlers.desc: +"""File-based log handlers.""" + +log_file_handlers.label: +"""File Handler""" + +node_global_gc_interval.desc: +"""Periodic garbage collection interval. Set to disabled to have it disabled.""" + +node_global_gc_interval.label: +"""Global GC Interval""" + +common_handler_time_offset.desc: +"""The time offset to be used when formatting the timestamp. +Can be one of: + - system: the time offset used by the local system + - utc: the UTC time offset + - +-[hh]:[mm]: user specified time offset, such as "-02:00" or "+00:00" +Defaults to: system.""" + +common_handler_time_offset.label: +"""Time Offset""" + +rpc_mode.desc: +"""In sync mode the sending side waits for the ack from the receiving side.""" + +rpc_mode.label: +"""RPC Mode""" + +node_crash_dump_file.desc: +"""Location of the crash dump file.""" + +node_crash_dump_file.label: +"""Crash Dump File""" + +cluster_mcast_loop.desc: +"""If true, loop UDP datagrams back to the local socket.""" + +cluster_mcast_loop.label: +"""Cluster Multicast Loop""" + +rpc_socket_keepalive_interval.desc: +"""The interval between keepalive messages.""" + +rpc_socket_keepalive_interval.label: +"""RPC Socket Keepalive Interval""" + +common_handler_level.desc: +"""The log level for the current log handler. +Defaults to warning.""" + +common_handler_level.label: +"""Log Level""" + +desc_rpc.desc: +"""EMQX uses a library called gen_rpc for inter-broker communication.
Most of the time the default config should work, but in case you need to do performance fine-tuning or experiment a bit, this is where to look.""" - zh: """EMQX 使用 gen_rpc 库来实现跨节点通信。
-大多数情况下，默认的配置应该可以工作，但如果你需要做一些性能优化或者实验，可以尝试调整这些参数。""" - } - label { - en: "RPC" - zh: "RPC" - } - } - desc_log { - desc { - en: """EMQX logging supports multiple sinks for the log events. -Each sink is represented by a _log handler_, which can be configured independently.""" - zh: """EMQX 日志记录支持日志事件的多个接收器。 每个接收器由一个_log handler_表示，可以独立配置。""" - } - label { - en: "Log" - zh: "日志" - } - } +desc_rpc.label: +"""RPC""" - desc_console_handler { - desc { - en: """Log handler that prints log events to the EMQX console.""" - zh: """日志处理进程将日志事件打印到 EMQX 控制台。""" - } - label { - en: "Console Handler" - zh: "Console Handler" - } - } +rpc_ssl_server_port.desc: +"""Listening port used by RPC local service.
+Note that this config only takes effect when rpc.port_discovery is set to manual +and driver is set to ssl.""" - desc_log_file_handler { - desc { - en: """Log handler that prints log events to files.""" - zh: """日志处理进程将日志事件打印到文件。""" - } - label { - en: "Files Log Handler" - zh: "文件日志处理进程" - } - } +rpc_ssl_server_port.label: +"""RPC SSL Server Port""" - desc_log_rotation { - desc { - en: """By default, the logs are stored in `./log` directory (for installation from zip file) or in `/var/log/emqx` (for binary installation).
-This section of the configuration controls the number of files kept for each log handler.""" +desc_cluster.desc: +"""EMQX nodes can form a cluster to scale up the total capacity.
+ Here holds the configs to instruct how individual nodes can discover each other.""" - zh: """默认情况下，日志存储在 `./log` 目录（用于从 zip 文件安装）或 `/var/log/emqx`（用于二进制安装）。
-这部分配置，控制每个日志处理进程保留的文件数量。""" - } - label { - en: "Log Rotation" - zh: "日志轮换" - } - } +desc_cluster.label: +"""Cluster""" - desc_log_overload_kill { - desc { - en: """Log overload kill features an overload protection that activates when the log handlers use too much memory or have too many buffered log messages.
-When the overload is detected, the log handler is terminated and restarted after a cooldown period.""" +common_handler_sync_mode_qlen.desc: +"""As long as the number of buffered log events is lower than this value, +all log events are handled asynchronously. This means that the client process sending the log event, +by calling a log function in the Logger API, does not wait for a response from the handler +but continues executing immediately after the event is sent. +It is not affected by the time it takes the handler to print the event to the log device. +If the message queue grows larger than this value, +the handler starts handling log events synchronously instead, +meaning that the client process sending the event must wait for a response. +When the handler reduces the message queue to a level below the sync_mode_qlen threshold, +asynchronous operation is resumed.""" - zh: """日志过载终止，具有过载保护功能。当日志处理进程使用过多内存，或者缓存的日志消息过多时该功能被激活。
-检测到过载时，日志处理进程将终止，并在冷却期后重新启动。""" - } - label { - en: "Log Overload Kill" - zh: "日志过载保护" - } - } +common_handler_sync_mode_qlen.label: +"""Queue Length before Entering Sync Mode""" - desc_log_burst_limit { - desc { - en: """Large bursts of log events produced in a short time can potentially cause problems, such as: - - Log files grow very large - - Log files are rotated too quickly, and useful information gets overwritten - - Overall performance impact on the system +common_handler_formatter.desc: +"""Choose log formatter. text for free text, and json for structured logging.""" -Log burst limit feature can temporarily disable logging to avoid these issues.""" - zh: """短时间内产生的大量日志事件可能会导致问题，例如： - - 日志文件变得非常大 - - 日志文件轮换过快，有用信息被覆盖 - - 对系统的整体性能影响 +common_handler_formatter.label: +"""Log Formatter""" -日志突发限制功能可以暂时禁用日志记录以避免这些问题。""" - } - label { - en: "Log Burst Limit" - zh: "日志突发限制" - } - } +rpc_async_batch_size.desc: +"""The maximum number of batch messages sent in asynchronous mode. + Note that this configuration does not work in synchronous mode.""" + +rpc_async_batch_size.label: +"""Async Batch Size""" + +cluster_call_max_history.desc: +"""Retain the maximum number of completed transactions (for queries).""" + +cluster_call_max_history.label: +"""Cluster Call Max History""" + +cluster_discovery_strategy.desc: +"""Service discovery method for the cluster nodes. Possible values are: +- manual: Use emqx ctl cluster command to manage cluster.
+- static: Configure static nodes list by setting seeds in config file.
+- dns: Use DNS A record to discover peer nodes.
+- etcd: Use etcd to discover peer nodes.
+- k8s: Use Kubernetes API to discover peer pods.""" + +cluster_discovery_strategy.label: +"""Cluster Discovery Strategy""" + +rpc_send_timeout.desc: +"""Timeout for sending the RPC request.""" + +rpc_send_timeout.label: +"""RPC Send Timeout""" + +common_handler_single_line.desc: +"""Print logs in a single line if set to true. Otherwise, log messages may span multiple lines.""" + +common_handler_single_line.label: +"""Single Line Mode""" + +rpc_socket_buffer.desc: +"""TCP tuning parameters. Socket buffer size in user mode.""" + +rpc_socket_buffer.label: +"""RPC Socket Buffer""" + +db_shard_transports.desc: +"""Allows to tune the transport method used for transaction log replication, on a per-shard basis.
+gen_rpc uses the gen_rpc library, +distr uses the Erlang distribution.
If not specified, +the default is to use the value set in db.default_shard_transport.""" + +db_shard_transports.label: +"""Shard Transports""" - desc_authorization { - desc { - en: """Settings that control client authorization.""" - zh: """授权相关""" - } - label { - en: "Authorization" - zh: "授权" - } - } } diff --git a/rel/i18n/emqx_connector_api.hocon b/rel/i18n/emqx_connector_api.hocon index 8575f01aa..4942ce981 100644 --- a/rel/i18n/emqx_connector_api.hocon +++ b/rel/i18n/emqx_connector_api.hocon @@ -1,82 +1,46 @@ emqx_connector_api { - id { - desc { - en: "The connector ID. Must be of format {type}:{name}" - zh: "连接器 ID， 格式必须为 {type}:{name}" - } - label: { - en: "Connector ID" - zh: "连接器 ID" - } - } +conn_get.desc: +"""List all connectors""" - conn_test_post { - desc { - en: """Test creating a new connector by given ID
+conn_get.label: +"""List All Connectors""" + +conn_id_delete.desc: +"""Delete a connector by ID""" + +conn_id_delete.label: +"""Delete Connector""" + +conn_id_get.desc: +"""Get the connector by ID""" + +conn_id_get.label: +"""Get Connector""" + +conn_id_put.desc: +"""Update an existing connector by ID""" + +conn_id_put.label: +"""Update Connector""" + +conn_post.desc: +"""Create a new connector""" + +conn_post.label: +"""Create Connector""" + +conn_test_post.desc: +"""Test creating a new connector by given ID
The ID must be of format '{type}:{name}'""" - zh: """通过给定的 ID 测试创建一个新的连接器
-ID 的格式必须为“{type}:{name}”""" - } - label: { - en: "Create Test Connector" - zh: "创建测试连接器" - } - } - conn_get { - desc { - en: "List all connectors" - zh: "列出所有连接器" - } - label: { - en: "List All Connectors" - zh: "列出所有连接器" - } - } +conn_test_post.label: +"""Create Test Connector""" - conn_post { - desc { - en: "Create a new connector" - zh: "创建一个新的连接器" - } - label: { - en: "Create Connector" - zh: "创建连接器" - } - } +id.desc: +"""The connector ID. Must be of format {type}:{name}""" - conn_id_get { - desc { - en: "Get the connector by ID" - zh: "通过 ID 获取连接器" - } - label: { - en: "Get Connector" - zh: "获取连接器" - } - } - - conn_id_put { - desc { - en: "Update an existing connector by ID" - zh: "通过 ID 更新一个连接器" - } - label: { - en: "Update Connector" - zh: "更新连接器" - } - } - - conn_id_delete { - desc { - en: "Delete a connector by ID" - zh: "通过 ID 删除一个连接器" - } - label: { - en: "Delete Connector" - zh: "删除连接器" - } - } +id.label: +"""Connector ID""" } diff --git a/rel/i18n/emqx_connector_http.hocon b/rel/i18n/emqx_connector_http.hocon index c6efd03ca..70c644e33 100644 --- a/rel/i18n/emqx_connector_http.hocon +++ b/rel/i18n/emqx_connector_http.hocon @@ -1,139 +1,78 @@ emqx_connector_http { - base_url { - desc { - en: """The base URL is the URL includes only the scheme, host and port.
+ +base_url.desc: +"""The base URL is the URL includes only the scheme, host and port.
When send an HTTP request, the real URL to be used is the concatenation of the base URL and the path parameter
For example: `http://localhost:9901/`""" - zh: """base URL 只包含host和port。
-发送HTTP请求时，真实的URL是由base URL 和 path parameter连接而成。
-示例：`http://localhost:9901/`""" - } - label: { - en: "Base Url" - zh: "Base Url" - } - } - connect_timeout { - desc { - en: "The timeout when connecting to the HTTP server." - zh: "连接HTTP服务器的超时时间。" - } - label: { - en: "Connect Timeout" - zh: "连接超时" - } - } +base_url.label: +"""Base Url""" - max_retries { - desc { - en: "Max retry times if error on sending request." - zh: "请求出错时的最大重试次数。" - } - label: { - en: "Max Retries" - zh: "最大重试次数" - } - } +body.desc: +"""HTTP request body.""" - pool_type { - desc { - en: "The type of the pool. Can be one of `random`, `hash`." - zh: "连接池的类型，可用类型有`random`, `hash`。" - } - label: { - en: "Pool Type" - zh: "连接池类型" - } - } +body.label: +"""HTTP Body""" - pool_size { - desc { - en: "The pool size." - zh: "连接池大小。" - } - label: { - en: "Pool Size" - zh: "连接池大小" - } - } +connect_timeout.desc: +"""The timeout when connecting to the HTTP server.""" - enable_pipelining { - desc { - en: "A positive integer. Whether to send HTTP requests continuously, when set to 1, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request." - zh: "正整数，设置最大可发送的异步 HTTP 请求数量。当设置为 1 时，表示每次发送完成 HTTP 请求后都需要等待服务器返回，再继续发送下一个请求。" - } - label: { - en: "HTTP Pipelineing" - zh: "HTTP 管道" - } - } +connect_timeout.label: +"""Connect Timeout""" - request { - desc { - en: """Configure HTTP request parameters.""" - zh: """设置 HTTP 请求的参数。""" - } - label: { - en: "Request" - zh: "HTTP 请求" - } - } +enable_pipelining.desc: +"""A positive integer. Whether to send HTTP requests continuously, when set to 1, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request.""" - method { - desc { - en: "HTTP method." - zh: "HTTP 请求方法。" - } - label: { - en: "HTTP Method" - zh: "HTTP 请求方法" - } - } +enable_pipelining.label: +"""HTTP Pipelineing""" - path { - desc { - en: "URL path." - zh: "HTTP请求路径。" - } - label: { - en: "URL Path" - zh: "HTTP请求路径" - } - } +headers.desc: +"""List of HTTP headers.""" - body { - desc { - en: "HTTP request body." - zh: "HTTP请求报文主体。" - } - label: { - en: "HTTP Body" - zh: "HTTP请求报文主体" - } - } +headers.label: +"""HTTP Headers""" - headers { - desc { - en: "List of HTTP headers." - zh: "HTTP 头字段列表。" - } - label: { - en: "HTTP Headers" - zh: "HTTP 头字段列表" - } - } +max_retries.desc: +"""Max retry times if error on sending request.""" - request_timeout { - desc { - en: "HTTP request timeout." - zh: "HTTP 请求超时。" - } - label: { - en: "Request Timeout" - zh: "HTTP 请求超时" - } - } +max_retries.label: +"""Max Retries""" + +method.desc: +"""HTTP method.""" + +method.label: +"""HTTP Method""" + +path.desc: +"""URL path.""" + +path.label: +"""URL Path""" + +pool_size.desc: +"""The pool size.""" + +pool_size.label: +"""Pool Size""" + +pool_type.desc: +"""The type of the pool. Can be one of `random`, `hash`.""" + +pool_type.label: +"""Pool Type""" + +request.desc: +"""Configure HTTP request parameters.""" + +request.label: +"""Request""" + +request_timeout.desc: +"""HTTP request timeout.""" + +request_timeout.label: +"""Request Timeout""" } diff --git a/rel/i18n/emqx_connector_ldap.hocon b/rel/i18n/emqx_connector_ldap.hocon index 0bcb4869e..64a953816 100644 --- a/rel/i18n/emqx_connector_ldap.hocon +++ b/rel/i18n/emqx_connector_ldap.hocon @@ -1,37 +1,21 @@ emqx_connector_ldap { - bind_dn { - desc { - en: """LDAP's Binding Distinguished Name (DN)""" - zh: """LDAP 绑定的 DN 的值""" - } - label: { - en: "Bind DN" - zh: "Bind DN" - } - } +bind_dn.desc: +"""LDAP's Binding Distinguished Name (DN)""" - port { - desc { - en: """LDAP Port""" - zh: """LDAP 端口""" - } - label: { - en: "Port" - zh: "端口" - } - } +bind_dn.label: +"""Bind DN""" +port.desc: +"""LDAP Port""" - timeout { - desc { - en: """LDAP's query timeout""" - zh: """LDAP 查询超时时间""" - } - label: { - en: "timeout" - zh: "超时时间" - } - } +port.label: +"""Port""" + +timeout.desc: +"""LDAP's query timeout""" + +timeout.label: +"""timeout""" } diff --git a/rel/i18n/emqx_connector_mongo.hocon b/rel/i18n/emqx_connector_mongo.hocon index 6a2511ec8..bba26d736 100644 --- a/rel/i18n/emqx_connector_mongo.hocon +++ b/rel/i18n/emqx_connector_mongo.hocon @@ -1,277 +1,152 @@ emqx_connector_mongo { - single_mongo_type { - desc { - en: "Standalone instance. Must be set to 'single' when MongoDB server is running in standalone mode." - zh: "Standalone 模式。当 MongoDB 服务运行在 standalone 模式下，该配置必须设置为 'single'。" - } - label: { - en: "Standalone instance" - zh: "Standalone 模式" - } - } +auth_source.desc: +"""Database name associated with the user's credentials.""" - rs_mongo_type { - desc { - en: "Replica set. Must be set to 'rs' when MongoDB server is running in 'replica set' mode." - zh: "Replica set模式。当 MongoDB 服务运行在 replica-set 模式下，该配置必须设置为 'rs'。" - } - label: { - en: "Replica set" - zh: "Replica set 模式" - } - } +auth_source.label: +"""Auth Source""" - sharded_mongo_type { - desc { - en: "Sharded cluster. Must be set to 'sharded' when MongoDB server is running in 'sharded' mode." - zh: "Sharded cluster模式。当 MongoDB 服务运行在 sharded 模式下，该配置必须设置为 'sharded'。" - } - label: { - en: "Sharded cluster" - zh: "Sharded cluster 模式" - } - } +connect_timeout.desc: +"""The duration to attempt a connection before timing out.""" - auth_source { - desc { - en: "Database name associated with the user's credentials." - zh: "与用户证书关联的数据库名称。" - } - label: { - en: "Auth Source" - zh: "认证源" - } - } +connect_timeout.label: +"""Connect Timeout""" - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+desc_rs.desc: +"""Settings for replica set.""" + +desc_rs.label: +"""Setting Replica Set""" + +desc_sharded.desc: +"""Settings for sharded cluster.""" + +desc_sharded.label: +"""Setting Sharded Cluster""" + +desc_single.desc: +"""Settings for a single MongoDB instance.""" + +desc_single.label: +"""Setting Single MongoDB""" + +desc_topology.desc: +"""Topology of MongoDB.""" + +desc_topology.label: +"""Setting Topology""" + +heartbeat_period.desc: +"""Controls when the driver checks the state of the MongoDB deployment. Specify the interval between checks, counted from the end of the previous check until the beginning of the next one. If the number of connections is increased (which will happen, for example, if you increase the pool size), you may need to increase this period as well to avoid creating too many log entries in the MongoDB log file.""" + +heartbeat_period.label: +"""Heartbeat period""" + +local_threshold.desc: +"""The size of the latency window for selecting among multiple suitable MongoDB instances.""" + +local_threshold.label: +"""Local Threshold""" + +max_overflow.desc: +"""Max Overflow.""" + +max_overflow.label: +"""Max Overflow""" + +min_heartbeat_period.desc: +"""Controls the minimum amount of time to wait between heartbeats.""" + +min_heartbeat_period.label: +"""Minimum Heartbeat Period""" + +overflow_check_period.desc: +"""Period for checking if there are more workers than configured ("overflow").""" + +overflow_check_period.label: +"""Overflow Check Period""" + +overflow_ttl.desc: +"""Period of time before workers that exceed the configured pool size ("overflow") to be terminated.""" + +overflow_ttl.label: +"""Overflow TTL""" + +r_mode.desc: +"""Read mode.""" + +r_mode.label: +"""Read Mode""" + +replica_set_name.desc: +"""Name of the replica set.""" + +replica_set_name.label: +"""Replica Set Name""" + +rs_mongo_type.desc: +"""Replica set. Must be set to 'rs' when MongoDB server is running in 'replica set' mode.""" + +rs_mongo_type.label: +"""Replica set""" + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The MongoDB default port 27017 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 MongoDB 默认端口 27017。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } - servers { - desc { - en: """A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].` +server.label: +"""Server Host""" + +server_selection_timeout.desc: +"""Specifies how long to block for server selection before throwing an exception.""" + +server_selection_timeout.label: +"""Server Selection Timeout""" + +servers.desc: +"""A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].` For each Node should be: The IPv4 or IPv6 address or the hostname to connect to. A host entry has the following form: `Host[:Port]`. The MongoDB default port 27017 is used if `[:Port]` is not specified.""" - zh: """集群将要连接的节点列表。 节点之间用逗号分隔，如：`Node[,Node].` -每个节点的配置为：将要连接的 IPv4 或 IPv6 地址或主机名。 -主机名具有以下形式：`Host[:Port]`。 -如果未指定 `[:Port]`，则使用 MongoDB 默认端口 27017。""" - } - label: { - en: "Servers" - zh: "服务器列表" - } - } - w_mode { - desc { - en: "Write mode." - zh: "写模式。" - } - label: { - en: "Write Mode" - zh: "写模式" - } - } +servers.label: +"""Servers""" - r_mode { - desc { - en: "Read mode." - zh: "读模式。" - } - label: { - en: "Read Mode" - zh: "读模式" - } - } +sharded_mongo_type.desc: +"""Sharded cluster. Must be set to 'sharded' when MongoDB server is running in 'sharded' mode.""" - overflow_ttl { - desc { - en: "Period of time before workers that exceed the configured pool size (\"overflow\") to be terminated." - zh: "当池内工人太多时，等待多久清除多余工人。" - } - label { - en: "Overflow TTL" - zh: "溢出TTL" - } - } +sharded_mongo_type.label: +"""Sharded cluster""" - overflow_check_period { - desc { - en: "Period for checking if there are more workers than configured (\"overflow\")." - zh: "检查是否有超过配置的工人的周期（\"溢出\"）。" - } - label { - en: "Overflow Check Period" - zh: "溢出检查周期" - } - } +single_mongo_type.desc: +"""Standalone instance. Must be set to 'single' when MongoDB server is running in standalone mode.""" - local_threshold { - desc { - en: "The size of the latency window for selecting among multiple suitable MongoDB instances." - zh: "在多个合适的MongoDB实例中进行选择的延迟窗口的大小。" - } - label { - en: "Local Threshold" - zh: "本地阈值" - } - } +single_mongo_type.label: +"""Standalone instance""" - connect_timeout { - desc { - en: "The duration to attempt a connection before timing out." - zh: "超时重连的等待时间。" - } - label { - en: "Connect Timeout" - zh: "连接超时" - } - } +socket_timeout.desc: +"""The duration to attempt to send or to receive on a socket before the attempt times out.""" - socket_timeout { - desc { - en: "The duration to attempt to send or to receive on a socket before the attempt times out." - zh: "在尝试超时之前，在套接字上尝试发送或接收的持续时间。" - } - label { - en: "Socket Timeout" - zh: "套接字操作超时" - } - } +socket_timeout.label: +"""Socket Timeout""" - server_selection_timeout { - desc { - en: "Specifies how long to block for server selection before throwing an exception." - zh: "指定在抛出异常之前为服务器选择阻断多长时间。" - } - label { - en: "Server Selection Timeout" - zh: "服务器选择超时" - } - } +srv_record.desc: +"""Use DNS SRV record.""" - wait_queue_timeout { - desc { - en: "The maximum duration that a worker can wait for a connection to become available." - zh: "工作者等待连接可用的最长时间。" - } - label { - en: "Wait Queue Timeout" - zh: "等待队列超时" - } - } +srv_record.label: +"""Srv Record""" - heartbeat_period { - desc { - en: "Controls when the driver checks the state of the MongoDB deployment. Specify the interval between checks, counted from the end of the previous check until the beginning of the next one. If the number of connections is increased (which will happen, for example, if you increase the pool size), you may need to increase this period as well to avoid creating too many log entries in the MongoDB log file." - zh: "控制驱动程序何时检查MongoDB部署的状态。指定检查的间隔时间，从上一次检查结束到下一次检查开始计算。如果连接数增加（例如，如果你增加池子的大小，就会发生这种情况），你可能也需要增加这个周期，以避免在MongoDB日志文件中创建太多的日志条目。" - } - label { - en: "Heartbeat period" - zh: "心跳期" - } - } +w_mode.desc: +"""Write mode.""" - min_heartbeat_period { - desc { - en: "Controls the minimum amount of time to wait between heartbeats." - zh: "心跳间的最小间隙" - } - label { - en: "Minimum Heartbeat Period" - zh: "最小心跳周期" - } - } +w_mode.label: +"""Write Mode""" - max_overflow { - desc { - en: "Max Overflow." - zh: "最大溢出。" - } - label: { - en: "Max Overflow" - zh: "最大溢出" - } - } +wait_queue_timeout.desc: +"""The maximum duration that a worker can wait for a connection to become available.""" - replica_set_name { - desc { - en: "Name of the replica set." - zh: "副本集的名称。" - } - label: { - en: "Replica Set Name" - zh: "副本集名称" - } - } - - srv_record { - desc { - en: "Use DNS SRV record." - zh: "使用 DNS SRV 记录。" - } - label: { - en: "Srv Record" - zh: "SRV 记录" - } - } - - desc_single { - desc { - en: """Settings for a single MongoDB instance.""" - zh: """配置 Single 模式""" - } - label: { - en: "Setting Single MongoDB" - zh: "配置 Single 模式" - } - } - - desc_rs { - desc { - en: """Settings for replica set.""" - zh: """配置 Replica Set""" - } - label: { - en: "Setting Replica Set" - zh: "配置 Replica Set" - } - } - - desc_sharded { - desc { - en: """Settings for sharded cluster.""" - zh: """配置 Sharded Cluster""" - } - label: { - en: "Setting Sharded Cluster" - zh: "配置 Sharded Cluster" - } - } - - desc_topology { - desc { - en: """Topology of MongoDB.""" - zh: """配置 Topology""" - } - label: { - en: "Setting Topology" - zh: "配置 Topology" - } - } +wait_queue_timeout.label: +"""Wait Queue Timeout""" } diff --git a/rel/i18n/emqx_connector_mqtt.hocon b/rel/i18n/emqx_connector_mqtt.hocon index 5ade54670..80303b825 100644 --- a/rel/i18n/emqx_connector_mqtt.hocon +++ b/rel/i18n/emqx_connector_mqtt.hocon @@ -1,35 +1,21 @@ emqx_connector_mqtt { - num_of_bridges { - desc { - en: "The current number of bridges that are using this connector." - zh: "当前使用此连接器的网桥数量。" - } - label: { - en: "Num of Bridges" - zh: "网桥数量" - } - } - type { - desc { - en: "The Connector Type." - zh: "连接器类型。" - } - label: { - en: "Connector Type" - zh: "连接器类型" - } - } +name.desc: +"""Connector name, used as a human-readable description of the connector.""" - name { - desc { - en: "Connector name, used as a human-readable description of the connector." - zh: "连接器名称，人类可读的连接器描述。" - } - label: { - en: "Connector Name" - zh: "连接器名称" - } - } +name.label: +"""Connector Name""" + +num_of_bridges.desc: +"""The current number of bridges that are using this connector.""" + +num_of_bridges.label: +"""Num of Bridges""" + +type.desc: +"""The Connector Type.""" + +type.label: +"""Connector Type""" } diff --git a/rel/i18n/emqx_connector_mqtt_schema.hocon b/rel/i18n/emqx_connector_mqtt_schema.hocon index 0de97d84b..e37e87e49 100644 --- a/rel/i18n/emqx_connector_mqtt_schema.hocon +++ b/rel/i18n/emqx_connector_mqtt_schema.hocon @@ -1,319 +1,178 @@ emqx_connector_mqtt_schema { - ingress_desc { - desc { - en: """The ingress config defines how this bridge receive messages from the remote MQTT broker, and then + +bridge_mode.desc: +"""If enable bridge mode. +NOTE: This setting is only for MQTT protocol version older than 5.0, and the remote MQTT +broker MUST support this feature. +If bridge_mode is set to true, the bridge will indicate to the remote broker that it is a bridge not an ordinary client. +This means that loop detection will be more effective and that retained messages will be propagated correctly.""" + +bridge_mode.label: +"""Bridge Mode""" + +clean_start.desc: +"""Whether to start a clean session when reconnecting a remote broker for ingress bridge""" + +clean_start.label: +"""Clean Session""" + +clientid_prefix.desc: +"""Optional prefix to prepend to the clientid used by egress bridges.""" + +clientid_prefix.label: +"""Clientid Prefix""" + +egress_desc.desc: +"""The egress config defines how this bridge forwards messages from the local broker to the remote broker.
+Template with variables is allowed in 'remote.topic', 'local.qos', 'local.retain', 'local.payload'.
+NOTE: if this bridge is used as the action of a rule, and also 'local.topic' +is configured, then both the data got from the rule and the MQTT messages that matches +'local.topic' will be forwarded.""" + +egress_desc.label: +"""Egress Configs""" + +egress_local.desc: +"""The configs about receiving messages from local broker.""" + +egress_local.label: +"""Local Configs""" + +egress_local_topic.desc: +"""The local topic to be forwarded to the remote broker""" + +egress_local_topic.label: +"""Local Topic""" + +egress_remote.desc: +"""The configs about sending message to the remote broker.""" + +egress_remote.label: +"""Remote Configs""" + +egress_remote_qos.desc: +"""The QoS of the MQTT message to be sent.
+Template with variables is allowed.""" + +egress_remote_qos.label: +"""Remote QoS""" + +egress_remote_topic.desc: +"""Forward to which topic of the remote broker.
+Template with variables is allowed.""" + +egress_remote_topic.label: +"""Remote Topic""" + +ingress_desc.desc: +"""The ingress config defines how this bridge receive messages from the remote MQTT broker, and then send them to the local broker.
Template with variables is allowed in 'remote.qos', 'local.topic', 'local.qos', 'local.retain', 'local.payload'.
NOTE: if this bridge is used as the input of a rule, and also 'local.topic' is configured, then messages got from the remote broker will be sent to both the 'local.topic' and the rule.""" - zh: """入口配置定义了该桥接如何从远程 MQTT Broker 接收消息，然后将消息发送到本地 Broker。
- 以下字段中允许使用带有变量的模板：'remote.qos', 'local.topic', 'local.qos', 'local.retain', 'local.payload'。
- 注意：如果此桥接被用作规则的输入，并且配置了 'local.topic'，则从远程代理获取的消息将同时被发送到 'local.topic' 和规则。""" - } - label: { - en: "Ingress Configs" - zh: "入方向配置" - } - } - egress_desc { - desc { - en: """The egress config defines how this bridge forwards messages from the local broker to the remote broker.
-Template with variables is allowed in 'remote.topic', 'local.qos', 'local.retain', 'local.payload'.
-NOTE: if this bridge is used as the action of a rule, and also 'local.topic' -is configured, then both the data got from the rule and the MQTT messages that matches -'local.topic' will be forwarded.""" - zh: """出口配置定义了该桥接如何将消息从本地 Broker 转发到远程 Broker。 -以下字段中允许使用带有变量的模板：'remote.topic', 'local.qos', 'local.retain', 'local.payload'。
-注意：如果此桥接被用作规则的动作，并且配置了 'local.topic'，则从规则输出的数据以及匹配到 'local.topic' 的 MQTT 消息都会被转发。""" - } - label: { - en: "Egress Configs" - zh: "出方向配置" - } - } +ingress_desc.label: +"""Ingress Configs""" - ingress_remote { - desc { - en: """The configs about subscribing to the remote broker.""" - zh: """订阅远程 Broker 相关的配置。""" - } - label: { - en: "Remote Configs" - zh: "远程配置" - } - } +ingress_local.desc: +"""The configs about sending message to the local broker.""" - ingress_local { - desc { - en: """The configs about sending message to the local broker.""" - zh: """发送消息到本地 Broker 相关的配置。""" - } - label: { - en: "Local Configs" - zh: "本地配置" - } - } +ingress_local.label: +"""Local Configs""" - egress_remote { - desc { - en: """The configs about sending message to the remote broker.""" - zh: """发送消息到远程 Broker 相关的配置。""" - } - label: { - en: "Remote Configs" - zh: "远程配置" - } - } +ingress_local_qos.desc: +"""The QoS of the MQTT message to be sent.
+Template with variables is allowed.""" - egress_local { - desc { - en: """The configs about receiving messages from local broker.""" - zh: """如何从本地 Broker 接收消息相关的配置。""" - } - label: { - en: "Local Configs" - zh: "本地配置" - } - } +ingress_local_qos.label: +"""Local QoS""" - mode { - desc { - en: """The mode of the MQTT Bridge.
+ingress_local_topic.desc: +"""Send messages to which topic of the local broker.
+Template with variables is allowed.""" + +ingress_local_topic.label: +"""Local Topic""" + +ingress_remote.desc: +"""The configs about subscribing to the remote broker.""" + +ingress_remote.label: +"""Remote Configs""" + +ingress_remote_qos.desc: +"""The QoS level to be used when subscribing to the remote broker""" + +ingress_remote_qos.label: +"""Remote QoS""" + +ingress_remote_topic.desc: +"""Receive messages from which topic of the remote broker""" + +ingress_remote_topic.label: +"""Remote Topic""" + +max_inflight.desc: +"""Max inflight (sent, but un-acked) messages of the MQTT protocol""" + +max_inflight.label: +"""Max Inflight Message""" + +mode.desc: +"""The mode of the MQTT Bridge.
- cluster_shareload: create an MQTT connection on each node in the emqx cluster.
In 'cluster_shareload' mode, the incoming load from the remote broker is shared by using shared subscription.
Note that the 'clientid' is suffixed by the node name, this is to avoid clientid conflicts between different nodes. And we can only use shared subscription topic filters for remote.topic of ingress connections.""" - zh: """MQTT 桥的模式。
-- cluster_shareload：在 emqx 集群的每个节点上创建一个 MQTT 连接。
-在“cluster_shareload”模式下，来自远程代理的传入负载通过共享订阅的方式接收。
-请注意，clientid 以节点名称为后缀，这是为了避免不同节点之间的 clientid 冲突。 -而且对于入口连接的 remote.topic，我们只能使用共享订阅主题过滤器。""" - } - label: { - en: "MQTT Bridge Mode" - zh: "MQTT 桥接模式" - } - } - server { - desc { - en: "The host and port of the remote MQTT broker" - zh: "远程 MQTT Broker的主机和端口。" - } - label: { - en: "Broker IP And Port" - zh: "Broker主机和端口" - } - } +mode.label: +"""MQTT Bridge Mode""" - bridge_mode { - desc { - en: """If enable bridge mode. -NOTE: This setting is only for MQTT protocol version older than 5.0, and the remote MQTT -broker MUST support this feature. -If bridge_mode is set to true, the bridge will indicate to the remote broker that it is a bridge not an ordinary client. -This means that loop detection will be more effective and that retained messages will be propagated correctly.""" - zh: """是否启用 Bridge Mode。 -注意：此设置只针对 MQTT 协议版本 < 5.0 有效，并且需要远程 MQTT Broker 支持 Bridge Mode。 -如果设置为 true ，桥接会告诉远端服务器当前连接是一个桥接而不是一个普通的客户端。 -这意味着消息回环检测会更加高效，并且远端服务器收到的保留消息的标志位会透传给本地。""" - } - label { - en: "Bridge Mode" - zh: "Bridge 模式" - } - } +password.desc: +"""The password of the MQTT protocol""" - proto_ver { - desc { - en: "The MQTT protocol version" - zh: "MQTT 协议版本" - } - label: { - en: "Protocol Version" - zh: "协议版本" - } - } +password.label: +"""Password""" - username { - desc { - en: "The username of the MQTT protocol" - zh: "MQTT 协议的用户名" - } - label: { - en: "Username" - zh: "用户名" - } - } - - password { - desc { - en: "The password of the MQTT protocol" - zh: "MQTT 协议的密码" - } - label: { - en: "Password" - zh: "密码" - } - } - - clean_start { - desc { - en: "Whether to start a clean session when reconnecting a remote broker for ingress bridge" - zh: "与 ingress MQTT 桥的远程服务器重连时是否清除老的 MQTT 会话。" - } - label: { - en: "Clean Session" - zh: "清除会话" - } - } - - max_inflight { - desc { - en: "Max inflight (sent, but un-acked) messages of the MQTT protocol" - zh: "MQTT 协议的最大飞行（已发送但未确认）消息" - } - label: { - en: "Max Inflight Message" - zh: "最大飞行消息" - } - } - - ingress_remote_topic { - desc { - en: "Receive messages from which topic of the remote broker" - zh: "从远程broker的哪个topic接收消息" - } - label: { - en: "Remote Topic" - zh: "远程主题" - } - } - - ingress_remote_qos { - desc { - en: "The QoS level to be used when subscribing to the remote broker" - zh: "订阅远程borker时要使用的 QoS 级别" - } - label: { - en: "Remote QoS" - zh: "远程 QoS" - } - } - - ingress_local_topic { - desc { - en: """Send messages to which topic of the local broker.
+payload.desc: +"""The payload of the MQTT message to be sent.
Template with variables is allowed.""" - zh: """向本地broker的哪个topic发送消息。
-允许使用带有变量的模板。""" - } - label: { - en: "Local Topic" - zh: "本地主题" - } - } - ingress_local_qos { - desc { - en: """The QoS of the MQTT message to be sent.
+payload.label: +"""Payload""" + +proto_ver.desc: +"""The MQTT protocol version""" + +proto_ver.label: +"""Protocol Version""" + +retain.desc: +"""The 'retain' flag of the MQTT message to be sent.
Template with variables is allowed.""" - zh: """待发送 MQTT 消息的 QoS。
-允许使用带有变量的模板。""" - } - label: { - en: "Local QoS" - zh: "本地 QoS" - } - } - egress_local_topic { - desc { - en: "The local topic to be forwarded to the remote broker" - zh: "要转发到远程broker的本地主题" - } - label: { - en: "Local Topic" - zh: "本地主题" - } - } +retain.label: +"""Retain Flag""" - egress_remote_topic { - desc { - en: """Forward to which topic of the remote broker.
-Template with variables is allowed.""" - zh: """转发到远程broker的哪个topic。
-允许使用带有变量的模板。""" - } - label: { - en: "Remote Topic" - zh: "远程主题" - } - } +server.desc: +"""The host and port of the remote MQTT broker""" - egress_remote_qos { - desc { - en: """The QoS of the MQTT message to be sent.
-Template with variables is allowed.""" - zh: """待发送 MQTT 消息的 QoS。
-允许使用带有变量的模板。""" - } - label: { - en: "Remote QoS" - zh: "远程 QoS" - } - } +server.label: +"""Broker IP And Port""" - retain { - desc { - en: """The 'retain' flag of the MQTT message to be sent.
-Template with variables is allowed.""" - zh: """要发送的 MQTT 消息的“保留”标志。
-允许使用带有变量的模板。""" - } - label: { - en: "Retain Flag" - zh: "保留消息标志" - } - } +server_configs.desc: +"""Configs related to the server.""" - payload { - desc { - en: """The payload of the MQTT message to be sent.
-Template with variables is allowed.""" - zh: """要发送的 MQTT 消息的负载。
-允许使用带有变量的模板。""" - } - label: { - en: "Payload" - zh: "消息负载" - } - } +server_configs.label: +"""Server Configs""" - server_configs { - desc { - en: """Configs related to the server.""" - zh: """服务器相关的配置。""" - } - label: { - en: "Server Configs" - zh: "服务配置。" - } - } +username.desc: +"""The username of the MQTT protocol""" - clientid_prefix { - desc { - en: """Optional prefix to prepend to the clientid used by egress bridges.""" - zh: """可选的前缀，用于在出口网桥使用的clientid前加上前缀。""" - } - label: { - en: "Clientid Prefix" - zh: "客户ID前缀" - } - } +username.label: +"""Username""" } diff --git a/rel/i18n/emqx_connector_mysql.hocon b/rel/i18n/emqx_connector_mysql.hocon index 89e95534b..dd86b62c8 100644 --- a/rel/i18n/emqx_connector_mysql.hocon +++ b/rel/i18n/emqx_connector_mysql.hocon @@ -1,18 +1,11 @@ emqx_connector_mysql { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The MySQL default port 3306 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 MySQL 默认端口 3306。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } + +server.label: +"""Server Host""" } diff --git a/rel/i18n/emqx_connector_pgsql.hocon b/rel/i18n/emqx_connector_pgsql.hocon index 33246c844..485e666a0 100644 --- a/rel/i18n/emqx_connector_pgsql.hocon +++ b/rel/i18n/emqx_connector_pgsql.hocon @@ -1,18 +1,11 @@ emqx_connector_pgsql { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The PostgreSQL default port 5432 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 PostgreSQL 默认端口 5432。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } + +server.label: +"""Server Host""" } diff --git a/rel/i18n/emqx_connector_redis.hocon b/rel/i18n/emqx_connector_redis.hocon index 5915725a2..5dc887b72 100644 --- a/rel/i18n/emqx_connector_redis.hocon +++ b/rel/i18n/emqx_connector_redis.hocon @@ -1,90 +1,50 @@ emqx_connector_redis { - single { - desc { - en: "Single mode. Must be set to 'single' when Redis server is running in single mode." - zh: "单机模式。当 Redis 服务运行在单机模式下，该配置必须设置为 'single'。" - } - label: { - en: "Single Mode" - zh: "单机模式" - } - } +cluster.desc: +"""Cluster mode. Must be set to 'cluster' when Redis server is running in clustered mode.""" - cluster { - desc { - en: "Cluster mode. Must be set to 'cluster' when Redis server is running in clustered mode." - zh: "集群模式。当 Redis 服务运行在集群模式下，该配置必须设置为 'cluster'。" - } - label: { - en: "Cluster Mode" - zh: "集群模式" - } - } +cluster.label: +"""Cluster Mode""" - sentinel { - desc { - en: "Sentinel mode. Must be set to 'sentinel' when Redis server is running in sentinel mode." - zh: "哨兵模式。当 Redis 服务运行在哨兵模式下，该配置必须设置为 'sentinel'。" - } - label: { - en: "Sentinel Mode" - zh: "哨兵模式" - } - } +database.desc: +"""Redis database ID.""" - sentinel_desc { - desc { - en: "The cluster name in Redis sentinel mode." - zh: "Redis 哨兵模式下的集群名称。" - } - label: { - en: "Cluster Name" - zh: "集群名称" - } - } +database.label: +"""Database ID""" - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+sentinel.desc: +"""Sentinel mode. Must be set to 'sentinel' when Redis server is running in sentinel mode.""" + +sentinel.label: +"""Sentinel Mode""" + +sentinel_desc.desc: +"""The cluster name in Redis sentinel mode.""" + +sentinel_desc.label: +"""Cluster Name""" + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The Redis default port 6379 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 Redis 默认端口 6379。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } - servers { - desc { - en: """A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].` +server.label: +"""Server Host""" + +servers.desc: +"""A Node list for Cluster to connect to. The nodes should be separated with commas, such as: `Node[,Node].` For each Node should be: The IPv4 or IPv6 address or the hostname to connect to. A host entry has the following form: `Host[:Port]`. The Redis default port 6379 is used if `[:Port]` is not specified.""" - zh: """集群将要连接的节点列表。 节点之间用逗号分隔，如：`Node[,Node].` -每个节点的配置为：将要连接的 IPv4 或 IPv6 地址或主机名。 -主机名具有以下形式：`Host[:Port]`。 -如果未指定 `[:Port]`，则使用 Redis 默认端口 6379。""" - } - label: { - en: "Servers" - zh: "服务器列表" - } - } - database { - desc { - en: "Redis database ID." - zh: "Redis 数据库 ID。" - } - label: { - en: "Database ID" - zh: "数据库 ID" - } - } +servers.label: +"""Servers""" + +single.desc: +"""Single mode. Must be set to 'single' when Redis server is running in single mode.""" + +single.label: +"""Single Mode""" } diff --git a/rel/i18n/emqx_connector_schema_lib.hocon b/rel/i18n/emqx_connector_schema_lib.hocon index 1bc45c36d..0e8a2e9a3 100644 --- a/rel/i18n/emqx_connector_schema_lib.hocon +++ b/rel/i18n/emqx_connector_schema_lib.hocon @@ -1,80 +1,45 @@ emqx_connector_schema_lib { - ssl { - desc { - en: "SSL connection settings." - zh: "启用 SSL 连接。" - } - label: { - en: "Enable SSL" - zh: "启用SSL" - } - } +auto_reconnect.desc: +"""Deprecated. Enable automatic reconnect to the database.""" - prepare_statement { - desc { - en: "Key-value list of SQL prepared statements." - zh: "SQL 预处理语句列表。" - } - label: { - en: "SQL Prepared Statements List" - zh: "SQL 预处理语句列表" - } - } +auto_reconnect.label: +"""Deprecated. Auto Reconnect Database""" - database_desc { - desc { - en: "Database name." - zh: "数据库名字。" - } - label: { - en: "Database Name" - zh: "数据库名字" - } - } +database_desc.desc: +"""Database name.""" - pool_size { - desc { - en: "Size of the connection pool towards the bridge target service." - zh: "桥接远端服务时使用的连接池大小。" - } - label: { - en: "Connection Pool Size" - zh: "连接池大小" - } - } +database_desc.label: +"""Database Name""" - username { - desc { - en: "EMQX's username in the external database." - zh: "内部数据库的用户名。" - } - label: { - en: "Username" - zh: "用户名" - } - } +password.desc: +"""EMQX's password in the external database.""" - password { - desc { - en: "EMQX's password in the external database." - zh: "内部数据库密码。" - } - label: { - en: "Password" - zh: "密码" - } - } +password.label: +"""Password""" - auto_reconnect { - desc { - en: "Deprecated. Enable automatic reconnect to the database." - zh: "已弃用。自动重连数据库。" - } - label: { - en: "Deprecated. Auto Reconnect Database" - zh: "已弃用。自动重连数据库" - } - } +pool_size.desc: +"""Size of the connection pool towards the bridge target service.""" + +pool_size.label: +"""Connection Pool Size""" + +prepare_statement.desc: +"""Key-value list of SQL prepared statements.""" + +prepare_statement.label: +"""SQL Prepared Statements List""" + +ssl.desc: +"""SSL connection settings.""" + +ssl.label: +"""Enable SSL""" + +username.desc: +"""EMQX's username in the external database.""" + +username.label: +"""Username""" } diff --git a/rel/i18n/emqx_dashboard_api.hocon b/rel/i18n/emqx_dashboard_api.hocon index 9a6390a02..3e5bb6239 100644 --- a/rel/i18n/emqx_dashboard_api.hocon +++ b/rel/i18n/emqx_dashboard_api.hocon @@ -1,150 +1,66 @@ emqx_dashboard_api { - token { - desc { - en: """Dashboard Auth Token""" - zh: """Dashboard 认证 Token""" - } - } +change_pwd_api.desc: +"""Change dashboard user password""" - username { - desc { - en: """Dashboard Username""" - zh: """Dashboard 用户名""" - } - } +create_user_api.desc: +"""Create dashboard user""" - user_description { - desc { - en: """Dashboard User Description""" - zh: """Dashboard 用户描述""" - } - } +create_user_api_success.desc: +"""Create dashboard user success""" - password { - desc { - en: """Dashboard Password""" - zh: """Dashboard 密码""" - } - } +delete_user_api.desc: +"""Delete dashboard user""" - license { - desc { - en: """EMQX License. opensource or enterprise""" - zh: """EMQX 许可类型。可为 opensource 或 enterprise""" - } - } +license.desc: +"""EMQX License. opensource or enterprise""" - version { - desc { - en: """EMQX Version""" - zh: """EMQX 版本""" - } - } +list_users_api.desc: +"""Dashboard list users""" - login_api { - desc { - en: """Get Dashboard Auth Token.""" - zh: """获取 Dashboard 认证 Token。""" - } - } +login_api.desc: +"""Get Dashboard Auth Token.""" - login_success { - desc { - en: """Dashboard Auth Success""" - zh: """Dashboard 认证成功""" - } - } +login_failed401.desc: +"""Login failed. Bad username or password""" - login_failed401 { - desc { - en: """Login failed. Bad username or password""" - zh: """登录失败。用户名或密码错误""" - } - } +login_failed_response400.desc: +"""Login failed. Bad username or password""" - logout_api { - desc { - en: """Dashboard user logout""" - zh: """Dashboard 用户登出""" - } - } +login_success.desc: +"""Dashboard Auth Success""" - list_users_api { - desc { - en: """Dashboard list users""" - zh: """Dashboard 用户列表""" - } - } +logout_api.desc: +"""Dashboard user logout""" - create_user_api { - desc { - en: """Create dashboard user""" - zh: """创建 Dashboard 用户""" - } - } +new_pwd.desc: +"""New password""" - create_user_api_success { - desc { - en: """Create dashboard user success""" - zh: """创建 Dashboard 用户成功""" - } - } +old_pwd.desc: +"""Old password""" - update_user_api { - desc { - en: """Update dashboard user description""" - zh: """更新 Dashboard 用户描述""" - } - } +password.desc: +"""Dashboard Password""" - update_user_api200 { - desc { - en: """Update dashboard user success""" - zh: """更新 Dashboard 用户成功""" - } - } +token.desc: +"""Dashboard Auth Token""" - delete_user_api { - desc { - en: """Delete dashboard user""" - zh: """删除 Dashboard 用户""" - } - } +update_user_api.desc: +"""Update dashboard user description""" - users_api404 { - desc { - en: """Dashboard user not found""" - zh: """Dashboard 用户不存在""" - } - } +update_user_api200.desc: +"""Update dashboard user success""" - change_pwd_api { - desc { - en: """Change dashboard user password""" - zh: """更改 Dashboard 用户密码""" - } - } +user_description.desc: +"""Dashboard User Description""" - old_pwd { - desc { - en: """Old password""" - zh: """旧密码""" - } - } +username.desc: +"""Dashboard Username""" - new_pwd { - desc { - en: """New password""" - zh: """新密码""" - } - } +users_api404.desc: +"""Dashboard user not found""" - login_failed_response400 { - desc { - en: """Login failed. Bad username or password""" - zh: """登录失败。用户名或密码错误""" - } - } +version.desc: +"""EMQX Version""" } diff --git a/rel/i18n/emqx_dashboard_schema.hocon b/rel/i18n/emqx_dashboard_schema.hocon index f81816e63..6bd6ab016 100644 --- a/rel/i18n/emqx_dashboard_schema.hocon +++ b/rel/i18n/emqx_dashboard_schema.hocon @@ -1,225 +1,139 @@ emqx_dashboard_schema { - listeners { - desc { - en: """HTTP(s) listeners are identified by their protocol type and are + +backlog.desc: +"""Defines the maximum length that the queue of pending connections can grow to.""" + +backlog.label: +"""Backlog""" + +bind.desc: +"""Port without IP(18083) or port with specified IP(127.0.0.1:18083).""" + +bind.label: +"""Bind""" + +bootstrap_users_file.desc: +"""Deprecated, use api_key.bootstrap_file.""" + +bootstrap_users_file.label: +"""Deprecated""" + +cors.desc: +"""Support Cross-Origin Resource Sharing (CORS). +Allows a server to indicate any origins (domain, scheme, or port) other than +its own from which a browser should permit loading resources.""" + +cors.label: +"""CORS""" + +default_password.desc: +"""The initial default password for dashboard 'admin' user. +For safety, it should be changed as soon as possible. +This value is not valid when you log in to Dashboard for the first time via the web +and change to a complex password as prompted.""" + +default_password.label: +"""Default password""" + +default_username.desc: +"""The default username of the automatically created dashboard user.""" + +default_username.label: +"""Default username""" + +desc_dashboard.desc: +"""Configuration for EMQX dashboard.""" + +desc_dashboard.label: +"""Dashboard""" + +desc_http.desc: +"""Configuration for the dashboard listener (plaintext).""" + +desc_http.label: +"""HTTP""" + +desc_https.desc: +"""Configuration for the dashboard listener (TLS).""" + +desc_https.label: +"""HTTPS""" + +desc_listeners.desc: +"""Configuration for the dashboard listener.""" + +desc_listeners.label: +"""Listeners""" + +i18n_lang.desc: +"""Internationalization language support.""" + +i18n_lang.label: +"""I18n language""" + +inet6.desc: +"""Enable IPv6 support, default is false, which means IPv4 only.""" + +inet6.label: +"""IPv6""" + +ipv6_v6only.desc: +"""Disable IPv4-to-IPv6 mapping for the listener. +The configuration is only valid when the inet6 is true.""" + +ipv6_v6only.label: +"""IPv6 only""" + +listener_enable.desc: +"""Ignore or enable this listener""" + +listener_enable.label: +"""Enable""" + +listeners.desc: +"""HTTP(s) listeners are identified by their protocol type and are used to serve dashboard UI and restful HTTP API. Listeners must have a unique combination of port number and IP address. For example, an HTTP listener can listen on all configured IP addresses on a given port for a machine by specifying the IP address 0.0.0.0. Alternatively, the HTTP listener can specify a unique IP address for each listener, but use the same port.""" - zh: """Dashboard 监听器设置。监听器必须有唯一的端口号和IP地址的组合。 -例如，可以通过指定IP地址 0.0.0.0 来监听机器上给定端口上的所有配置的IP地址。 -或者，可以为每个监听器指定唯一的IP地址，但使用相同的端口。""" - } - label { - en: "Listeners" - zh: "监听器" - } - } - sample_interval { - desc { - en: """How often to update metrics displayed in the dashboard. + +listeners.label: +"""Listeners""" + +max_connections.desc: +"""Maximum number of simultaneous connections.""" + +max_connections.label: +"""Maximum connections""" + +num_acceptors.desc: +"""Socket acceptor pool size for TCP protocols. Default is the number of schedulers online""" + +num_acceptors.label: +"""Number of acceptors""" + +proxy_header.desc: +"""Enable support for `HAProxy` header. Be aware once enabled regular HTTP requests can't be handled anymore.""" + +proxy_header.label: +"""Enable support for HAProxy header""" + +sample_interval.desc: +"""How often to update metrics displayed in the dashboard. Note: `sample_interval` should be a divisor of 60, default is 10s.""" - zh: """Dashboard 中图表指标的时间间隔。必须小于60，且被60的整除，默认设置 10s。""" - } - } - token_expired_time { - desc { - en: "JWT token expiration time. Default is 60 minutes" - zh: "JWT token 过期时间。默认设置为 60 分钟。" - } - label { - en: "Token expired time" - zh: "JWT 过期时间" - } - } - num_acceptors { - desc { - en: "Socket acceptor pool size for TCP protocols. Default is the number of schedulers online" - zh: "TCP协议的Socket acceptor池大小, 默认设置在线的调度器数量（通常为 CPU 核数）" - } - label { - en: "Number of acceptors" - zh: "Acceptor 数量" - } - } - max_connections { - desc { - en: "Maximum number of simultaneous connections." - zh: "同时处理的最大连接数。" - } - label { - en: "Maximum connections" - zh: "最大连接数" - } - } - backlog { - desc { - en: "Defines the maximum length that the queue of pending connections can grow to." - zh: "排队等待连接的队列的最大长度。" - } - label { - en: "Backlog" - zh: "排队长度" - } - } - send_timeout { - desc { - en: "Send timeout for the socket." - zh: "Socket发送超时时间。" - } - label { - en: "Send timeout" - zh: "发送超时时间" - } - } - inet6 { - desc { - en: "Enable IPv6 support, default is false, which means IPv4 only." - zh: "启用IPv6， 如果机器不支持IPv6，请关闭此选项，否则会导致 Dashboard 无法使用。" - } - label { - en: "IPv6" - zh: "IPv6" - } - } - ipv6_v6only { - desc { - en: """Disable IPv4-to-IPv6 mapping for the listener. -The configuration is only valid when the inet6 is true.""" - zh: "当开启 inet6 功能的同时禁用 IPv4-to-IPv6 映射。该配置仅在 inet6 功能开启时有效。" - } - label { - en: "IPv6 only" - zh: "IPv6 only" - } - } - proxy_header { - desc { - en: "Enable support for `HAProxy` header. Be aware once enabled regular HTTP requests can't be handled anymore." - zh: "开启对 `HAProxy` 的支持，注意：一旦开启了这个功能，就无法再处理普通的 HTTP 请求了。" - } - label: { - en: "Enable support for HAProxy header" - zh: "开启对 `HAProxy` 的支持" - } - } - desc_dashboard { - desc { - en: "Configuration for EMQX dashboard." - zh: "EMQX Dashboard 配置。" - } - label { - en: "Dashboard" - zh: "Dashboard" - } - } - desc_listeners { - desc { - en: "Configuration for the dashboard listener." - zh: "Dashboard 监听器配置。" - } - label { - en: "Listeners" - zh: "监听器" - } - } - desc_http { - desc { - en: "Configuration for the dashboard listener (plaintext)." - zh: "Dashboard 监听器(HTTP)配置。" - } - label { - en: "HTTP" - zh: "HTTP" - } - } - desc_https { - desc { - en: "Configuration for the dashboard listener (TLS)." - zh: "Dashboard 监听器(HTTPS)配置。" - } - label { - en: "HTTPS" - zh: "HTTPS" - } - } - listener_enable { - desc { - en: "Ignore or enable this listener" - zh: "忽略或启用该监听器。" - } - label { - en: "Enable" - zh: "启用" - } - } - bind { - desc { - en: "Port without IP(18083) or port with specified IP(127.0.0.1:18083)." - zh: "监听地址和端口，热更新此配置时，会重启 Dashboard 服务。" - } - label { - en: "Bind" - zh: "绑定端口" - } - } - default_username { - desc { - en: "The default username of the automatically created dashboard user." - zh: "Dashboard 的默认用户名。" - } - label { - en: "Default username" - zh: "默认用户名" - } - } - default_password { - desc { - en: """The initial default password for dashboard 'admin' user. -For safety, it should be changed as soon as possible. -This value is not valid when you log in to Dashboard for the first time via the web -and change to a complex password as prompted.""" - zh: """Dashboard 的默认密码，为了安全，应该尽快修改密码。 -当通过网页首次登录 Dashboard 并按提示修改成复杂密码后，此值就会失效。""" - } - label { - en: "Default password" - zh: "默认密码" - } - } - cors { - desc { - en: """Support Cross-Origin Resource Sharing (CORS). -Allows a server to indicate any origins (domain, scheme, or port) other than -its own from which a browser should permit loading resources.""" - zh: """支持跨域资源共享(CORS)， -允许服务器指示任何来源(域名、协议或端口)，除了本服务器之外的任何浏览器应允许加载资源。""" - } - label { - en: "CORS" - zh: "跨域资源共享" - } - } - i18n_lang { - desc { - en: "Internationalization language support." - zh: "设置 Swagger 多语言的版本，可为 en 或 zh。" - } - label { - en: "I18n language" - zh: "多语言支持" - } - } - bootstrap_users_file { - desc { - en: "Deprecated, use api_key.bootstrap_file." - zh: "已废弃，请使用 api_key.bootstrap_file。" - } - label { - en: """Deprecated""" - zh: """已废弃""" - } - } + +send_timeout.desc: +"""Send timeout for the socket.""" + +send_timeout.label: +"""Send timeout""" + +token_expired_time.desc: +"""JWT token expiration time. Default is 60 minutes""" + +token_expired_time.label: +"""Token expired time""" + } diff --git a/rel/i18n/emqx_delayed_api.hocon b/rel/i18n/emqx_delayed_api.hocon index d16d61ccf..62e0fd775 100644 --- a/rel/i18n/emqx_delayed_api.hocon +++ b/rel/i18n/emqx_delayed_api.hocon @@ -1,164 +1,72 @@ emqx_delayed_api { - view_status_api { - desc { - en: "Get delayed status" - zh: "查看慢订阅状态" - } - } +bad_msgid_format.desc: +"""Bad Message ID format""" - update_api { - desc { - en: "Enable or disable delayed, set max delayed messages" - zh: "开启或者关闭功能，或者设置延迟消息数量上限" - } - } +count.desc: +"""Count of delayed messages""" - update_success { - desc { - en: "Enable or disable delayed successfully" - zh: "开启或者关闭功能操作成功" - } - } +delayed_interval.desc: +"""Delayed interval(second)""" - illegality_limit { - desc { - en: "Max limit illegality" - zh: "数量上限不合法" - } - } +delayed_remaining.desc: +"""Delayed remaining(second)""" - get_message_api { - desc { - en: "View delayed message" - zh: "查看延迟消息" - } - } +delete_api.desc: +"""Delete delayed message""" - node { - desc { - en: "The node where message from" - zh: "消息的来源节点" - } - } +expected_at.desc: +"""Expect publish time, in RFC 3339 format""" - msgid { - desc { - en: "Delayed Message ID" - zh: "延迟消息 ID" - } - } +from_clientid.desc: +"""From ClientID""" - bad_msgid_format { - desc { - en: "Bad Message ID format" - zh: "消息 ID 格式错误" - } - } +from_username.desc: +"""From Username""" - msgid_not_found { - desc { - en: "Message ID not found" - zh: "未找到对应消息" - } - } +get_message_api.desc: +"""View delayed message""" - delete_api { - desc { - en: "Delete delayed message" - zh: "删除延迟消息" - } - } +illegality_limit.desc: +"""Max limit illegality""" - list_api { - desc { - en: "List delayed messages" - zh: "查看延迟消息列表" - } - } +list_api.desc: +"""List delayed messages""" - view_page { - desc { - en: "View page" - zh: "查看的页数" - } - } +msgid.desc: +"""Delayed Message ID""" - view_limit { - desc { - en: "Page limit" - zh: "每页数量" - } - } +msgid_not_found.desc: +"""Message ID not found""" - count { - desc { - en: "Count of delayed messages" - zh: "延迟消息总数" - } - } +node.desc: +"""The node where message from""" - publish_at { - desc { - en: "Clinet publish message time, in RFC 3339 format" - zh: "客户端发送时间, RFC 3339 格式" - } - } +payload.desc: +"""Payload, base64 encoded. Payload will be set to 'PAYLOAD_TO_LARGE' if its length is larger than 2048 bytes""" - delayed_interval { - desc { - en: "Delayed interval(second)" - zh: "延迟时间(秒)" - } - } +publish_at.desc: +"""Clinet publish message time, in RFC 3339 format""" - delayed_remaining { - desc { - en: "Delayed remaining(second)" - zh: "剩余时间(秒)" - } - } +qos.desc: +"""QoS""" - expected_at { - desc { - en: "Expect publish time, in RFC 3339 format" - zh: "期望的发送时间, RFC 3339 格式" - } - } +topic.desc: +"""Topic""" - topic { - desc { - en: "Topic" - zh: "主题" - } - } +update_api.desc: +"""Enable or disable delayed, set max delayed messages""" - qos { - desc { - en: "QoS" - zh: "QoS" - } - } +update_success.desc: +"""Enable or disable delayed successfully""" - from_clientid { - desc { - en: "From ClientID" - zh: "消息的 ClientID" - } - } +view_limit.desc: +"""Page limit""" - from_username { - desc { - en: "From Username" - zh: "消息的 Username" - } - } +view_page.desc: +"""View page""" - payload { - desc { - en: "Payload, base64 encoded. Payload will be set to 'PAYLOAD_TO_LARGE' if its length is larger than 2048 bytes" - zh: "消息内容, base64 格式。如果消息的大小超过 2048 字节，则消息内容会被设置为 'PAYLOAD_TO_LARGE'" - } - } +view_status_api.desc: +"""Get delayed status""" } diff --git a/rel/i18n/emqx_ee_bridge_cassa.hocon b/rel/i18n/emqx_ee_bridge_cassa.hocon index 3bbac6658..d86c95a5f 100644 --- a/rel/i18n/emqx_ee_bridge_cassa.hocon +++ b/rel/i18n/emqx_ee_bridge_cassa.hocon @@ -1,72 +1,43 @@ emqx_ee_bridge_cassa { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to Cassandra. All MQTT 'PUBLISH' messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +cql_template.desc: +"""CQL Template""" + +cql_template.label: +"""CQL Template""" + +desc_config.desc: +"""Configuration for a Cassandra bridge.""" + +desc_config.label: +"""Cassandra Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to Cassandra. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 Cassandra。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - cql_template { - desc { - en: """CQL Template""" - zh: """CQL 模板""" - } - label { - en: "CQL Template" - zh: "CQL 模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for a Cassandra bridge.""" - zh: """Cassandra 桥接配置""" - } - label: { - en: "Cassandra Bridge Configuration" - zh: "Cassandra 桥接配置" - } - } - - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } - - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/emqx_ee_bridge_clickhouse.hocon index b54f4dc70..6735aee22 100644 --- a/rel/i18n/emqx_ee_bridge_clickhouse.hocon +++ b/rel/i18n/emqx_ee_bridge_clickhouse.hocon @@ -1,81 +1,49 @@ emqx_ee_bridge_clickhouse { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to Clickhouse. All MQTT 'PUBLISH' messages with the topic +batch_value_separator.desc: +"""The default value ',' works for the VALUES format. You can also use other separator if other format is specified. See [INSERT INTO Statement](https://clickhouse.com/docs/en/sql-reference/statements/insert-into).""" + +batch_value_separator.label: +"""Batch Value Separator""" + +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a Clickhouse bridge.""" + +desc_config.label: +"""Clickhouse Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to Clickhouse. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 Clickhouse。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - sql_template { - desc { - en: """The template string can contain ${field} placeholders for message metadata and payload field. Make sure that the inserted values are formatted and escaped correctly. [Prepared Statement](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridges.html#Prepared-Statement) is not supported.""" - zh: """可以使用 ${field} 占位符来引用消息与客户端上下文中的变量，请确保对应字段存在且数据格式符合预期。此处不支持 [SQL 预处理](https://docs.emqx.com/zh/enterprise/v5.0/data-integration/data-bridges.html#sql-预处理)。""" - } - label { - en: "SQL Template" - zh: "SQL 模板" - } - } - batch_value_separator { - desc { - en: """The default value ',' works for the VALUES format. You can also use other separator if other format is specified. See [INSERT INTO Statement](https://clickhouse.com/docs/en/sql-reference/statements/insert-into).""" - zh: """默认为逗号 ','，适用于 VALUE 格式。您也可以使用其他分隔符， 请参考 [INSERT INTO 语句](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)。""" - } - label { - en: "Batch Value Separator" - zh: "分隔符" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } - desc_config { - desc { - en: """Configuration for a Clickhouse bridge.""" - zh: """Clickhouse 桥接配置""" - } - label: { - en: "Clickhouse Bridge Configuration" - zh: "Clickhouse 桥接配置" - } - } +local_topic.label: +"""Local Topic""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +sql_template.desc: +"""The template string can contain ${field} placeholders for message metadata and payload field. Make sure that the inserted values are formatted and escaped correctly. [Prepared Statement](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridges.html#Prepared-Statement) is not supported.""" + +sql_template.label: +"""SQL Template""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_dynamo.hocon b/rel/i18n/emqx_ee_bridge_dynamo.hocon index b93b12166..7725130eb 100644 --- a/rel/i18n/emqx_ee_bridge_dynamo.hocon +++ b/rel/i18n/emqx_ee_bridge_dynamo.hocon @@ -1,72 +1,43 @@ emqx_ee_bridge_dynamo { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to DynamoDB. All MQTT `PUBLISH` messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a DynamoDB bridge.""" + +desc_config.label: +"""DynamoDB Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to DynamoDB. All MQTT `PUBLISH` messages with the topic matching the `local_topic` will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also `local_topic` is configured, then both the data got from the rule and the MQTT messages that match `local_topic` will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 DynamoDB。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - template { - desc { - en: """Template, the default value is empty. When this value is empty the whole message will be stored in the database""" - zh: """模板, 默认为空，为空时将会将整个消息存入数据库""" - } - label { - en: "Template" - zh: "模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for a DynamoDB bridge.""" - zh: """DynamoDB 桥接配置""" - } - label: { - en: "DynamoDB Bridge Configuration" - zh: "DynamoDB 桥接配置" - } - } +template.desc: +"""Template, the default value is empty. When this value is empty the whole message will be stored in the database""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +template.label: +"""Template""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon b/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon index b8fa3b43a..6f864a524 100644 --- a/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon +++ b/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon @@ -1,145 +1,80 @@ emqx_ee_bridge_gcp_pubsub { - desc_config { - desc { - en: """Configuration for a GCP PubSub bridge.""" - zh: """GCP PubSub 桥接配置""" - } - label { - en: "GCP PubSub Bridge Configuration" - zh: "GCP PubSub 桥接配置" - } - } - desc_type { - desc { - en: """The Bridge Type""" - zh: """桥接类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } +connect_timeout.desc: +"""The timeout when connecting to the HTTP server.""" - connect_timeout { - desc { - en: "The timeout when connecting to the HTTP server." - zh: "连接 HTTP 服务器的超时时间。" - } - label: { - en: "Connect Timeout" - zh: "连接超时" - } - } +connect_timeout.label: +"""Connect Timeout""" - max_retries { - desc { - en: "Max retry times if an error occurs when sending a request." - zh: "请求出错时的最大重试次数。" - } - label: { - en: "Max Retries" - zh: "最大重试次数" - } - } +desc_config.desc: +"""Configuration for a GCP PubSub bridge.""" - pool_size { - desc { - en: "The pool size." - zh: "连接池大小。" - } - label: { - en: "Pool Size" - zh: "连接池大小" - } - } +desc_config.label: +"""GCP PubSub Bridge Configuration""" - pipelining { - desc { - en: "A positive integer. Whether to send HTTP requests continuously, when set to 1, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request." - zh: "正整数，设置最大可发送的异步 HTTP 请求数量。当设置为 1 时，表示每次发送完成 HTTP 请求后都需要等待服务器返回，再继续发送下一个请求。" - } - label: { - en: "HTTP Pipelineing" - zh: "HTTP 流水线" - } - } +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" - request_timeout { - desc { - en: "Deprecated: Configure the request timeout in the buffer settings." - zh: "废弃的。在缓冲区设置中配置请求超时。" - } - label: { - en: "Request Timeout" - zh: "HTTP 请求超时" - } - } +desc_name.label: +"""Bridge Name""" - payload_template { - desc { - en: "The template for formatting the outgoing messages. If undefined, will send all the available context in JSON format." - zh: "用于格式化外发信息的模板。 如果未定义，将以JSON格式发送所有可用的上下文。" - } - label: { - en: "Payload template" - zh: "HTTP 请求消息体模板" - } - } +desc_type.desc: +"""The Bridge Type""" - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to GCP PubSub. All MQTT 'PUBLISH' messages with the topic +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to GCP PubSub. All MQTT 'PUBLISH' messages with the topic matching `local_topic` will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 GCP PubSub。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 GCP PubSub。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - pubsub_topic { - desc { - en: "The GCP PubSub topic to publish messages to." - zh: "要发布消息的GCP PubSub主题。" - } - label: { - en: "GCP PubSub Topic" - zh: "GCP PubSub 主题" - } - } +local_topic.label: +"""Local Topic""" - service_account_json { - desc { - en: "JSON containing the GCP Service Account credentials to be used with PubSub.\n" - "When a GCP Service Account is created " - "(as described in https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount), " - "you have the option of downloading the credentials in JSON form. That's the " - "file needed." - zh: "包含将与 PubSub 一起使用的 GCP 服务账户凭证的 JSON。\n" - "当创建GCP服务账户时" - "（如https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount），" - "可以选择下载 JSON 形式的凭证，然后在该配置项中使用。" - } - label: { - en: "GCP Service Account Credentials" - zh: "GCP 服务账户凭证" - } - } +max_retries.desc: +"""Max retry times if an error occurs when sending a request.""" + +max_retries.label: +"""Max Retries""" + +payload_template.desc: +"""The template for formatting the outgoing messages. If undefined, will send all the available context in JSON format.""" + +payload_template.label: +"""Payload template""" + +pipelining.desc: +"""A positive integer. Whether to send HTTP requests continuously, when set to 1, it means that after each HTTP request is sent, you need to wait for the server to return and then continue to send the next request.""" + +pipelining.label: +"""HTTP Pipelineing""" + +pool_size.desc: +"""The pool size.""" + +pool_size.label: +"""Pool Size""" + +pubsub_topic.desc: +"""The GCP PubSub topic to publish messages to.""" + +pubsub_topic.label: +"""GCP PubSub Topic""" + +request_timeout.desc: +"""Deprecated: Configure the request timeout in the buffer settings.""" + +request_timeout.label: +"""Request Timeout""" + +service_account_json.desc: +"""JSON containing the GCP Service Account credentials to be used with PubSub. +When a GCP Service Account is created (as described in https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount), you have the option of downloading the credentials in JSON form. That's the file needed.""" + +service_account_json.label: +"""GCP Service Account Credentials""" } diff --git a/rel/i18n/emqx_ee_bridge_hstreamdb.hocon b/rel/i18n/emqx_ee_bridge_hstreamdb.hocon index dce40aa85..cb43d483a 100644 --- a/rel/i18n/emqx_ee_bridge_hstreamdb.hocon +++ b/rel/i18n/emqx_ee_bridge_hstreamdb.hocon @@ -1,90 +1,55 @@ emqx_ee_bridge_hstreamdb { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to the HStreamDB. All MQTT 'PUBLISH' messages with the topic + +config_direction.desc: +"""The direction of this bridge, MUST be 'egress'""" + +config_direction.label: +"""Bridge Direction""" + +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for an HStreamDB bridge.""" + +desc_config.label: +"""HStreamDB Bridge Configuration""" + +desc_connector.desc: +"""Generic configuration for the connector.""" + +desc_connector.label: +"""Connector Generic Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to the HStreamDB. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 HStreamDB。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HStreamDB。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - payload { - desc { - en: """The payload to be forwarded to the HStreamDB. Placeholders supported.""" - zh: """要转发到 HStreamDB 的数据内容，支持占位符""" - } - label { - en: "Payload" - zh: "消息内容" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } - config_direction { - desc { - en: """The direction of this bridge, MUST be 'egress'""" - zh: """桥接的方向， 必须是 egress""" - } - label { - en: "Bridge Direction" - zh: "桥接方向" - } - } - desc_config { - desc { - en: """Configuration for an HStreamDB bridge.""" - zh: """HStreamDB 桥接配置""" - } - label: { - en: "HStreamDB Bridge Configuration" - zh: "HStreamDB 桥接配置" - } - } +local_topic.label: +"""Local Topic""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +payload.desc: +"""The payload to be forwarded to the HStreamDB. Placeholders supported.""" + +payload.label: +"""Payload""" - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } - desc_connector { - desc { - en: """Generic configuration for the connector.""" - zh: """连接器的通用配置。""" - } - label: { - en: "Connector Generic Configuration" - zh: "连接器通用配置。" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_influxdb.hocon b/rel/i18n/emqx_ee_bridge_influxdb.hocon index d73d62b14..c5cee2b66 100644 --- a/rel/i18n/emqx_ee_bridge_influxdb.hocon +++ b/rel/i18n/emqx_ee_bridge_influxdb.hocon @@ -1,22 +1,41 @@ emqx_ee_bridge_influxdb { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic + +config_enable.desc: +"""Enable or disable this bridge.""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for an InfluxDB bridge.""" + +desc_config.label: +"""InfluxDB Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type.""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to the InfluxDB. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 InfluxDB。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 InfluxDB。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - write_syntax { - desc { - en: """Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. + +local_topic.label: +"""Local Topic""" + +write_syntax.desc: +"""Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and [InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
TLDR:
@@ -24,62 +43,8 @@ TLDR:
[,=[,=]] =[,=] [] ``` Please note that a placeholder for an integer value must be annotated with a suffix `i`. For example `${payload.int_value}i`.""" - zh: """使用 InfluxDB API Line Protocol 写入 InfluxDB 的数据，支持占位符
-参考 [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) 及 -[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
-TLDR:
-``` -[,=[,=]] =[,=] [] -``` -注意，整形数值占位符后需要添加一个字符 `i` 类型标识。例如 `${payload.int_value}i`""" - } - label { - en: "Write Syntax" - zh: "写语句" - } - } - config_enable { - desc { - en: """Enable or disable this bridge.""" - zh: """启用/禁用桥接。""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } - desc_config { - desc { - en: """Configuration for an InfluxDB bridge.""" - zh: """InfluxDB 桥接配置。""" - } - label: { - en: "InfluxDB Bridge Configuration" - zh: "InfluxDB 桥接配置" - } - } - - desc_type { - desc { - en: """The Bridge Type.""" - zh: """桥接类型。""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } - - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名称。""" - } - label { - en: "Bridge Name" - zh: "桥接名称" - } - } +write_syntax.label: +"""Write Syntax""" } diff --git a/rel/i18n/emqx_ee_bridge_mongodb.hocon b/rel/i18n/emqx_ee_bridge_mongodb.hocon index 053c932f7..fab371824 100644 --- a/rel/i18n/emqx_ee_bridge_mongodb.hocon +++ b/rel/i18n/emqx_ee_bridge_mongodb.hocon @@ -1,100 +1,57 @@ emqx_ee_bridge_mongodb { - desc_config { - desc { - en: "Configuration for MongoDB Bridge" - zh: "为MongoDB桥配置" - } - label { - en: "MongoDB Bridge Configuration" - zh: "MongoDB桥配置" - } - } - enable { - desc { - en: "Enable or disable this MongoDB Bridge" - zh: "启用或停用该MongoDB桥" - } - label { - en: "Enable or disable" - zh: "启用或禁用" - } - } +collection.desc: +"""The collection where data will be stored into""" - collection { - desc { - en: "The collection where data will be stored into" - zh: "数据将被存储到的集合" - } - label { - en: "Collection to be used" - zh: "将要使用的集合（Collection）" - } - } +collection.label: +"""Collection to be used""" - mongodb_rs_conf { - desc { - en: "MongoDB (Replica Set) configuration" - zh: "MongoDB（Replica Set）配置" - } - label { - en: "MongoDB (Replica Set) Configuration" - zh: "MongoDB（Replica Set）配置" - } - } +desc_config.desc: +"""Configuration for MongoDB Bridge""" - mongodb_sharded_conf { - desc { - en: "MongoDB (Sharded) configuration" - zh: "MongoDB (Sharded)配置" - } - label { - en: "MongoDB (Sharded) Configuration" - zh: "MongoDB (Sharded)配置" - } - } +desc_config.label: +"""MongoDB Bridge Configuration""" - mongodb_single_conf { - desc { - en: "MongoDB (Standalone) configuration" - zh: "MongoDB（独立）配置" - } - label { - en: "MongoDB (Standalone) Configuration" - zh: "MongoDB（独立）配置" - } - } +desc_name.desc: +"""Bridge name.""" - desc_type { - desc { - en: """The Bridge Type.""" - zh: """桥接类型。""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +desc_name.label: +"""Bridge Name""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名称。""" - } - label { - en: "Bridge Name" - zh: "桥接名称" - } - } +desc_type.desc: +"""The Bridge Type.""" + +desc_type.label: +"""Bridge Type""" + +enable.desc: +"""Enable or disable this MongoDB Bridge""" + +enable.label: +"""Enable or disable""" + +mongodb_rs_conf.desc: +"""MongoDB (Replica Set) configuration""" + +mongodb_rs_conf.label: +"""MongoDB (Replica Set) Configuration""" + +mongodb_sharded_conf.desc: +"""MongoDB (Sharded) configuration""" + +mongodb_sharded_conf.label: +"""MongoDB (Sharded) Configuration""" + +mongodb_single_conf.desc: +"""MongoDB (Standalone) configuration""" + +mongodb_single_conf.label: +"""MongoDB (Standalone) Configuration""" + +payload_template.desc: +"""The template for formatting the outgoing messages. If undefined, rule engine will use JSON format to serialize all visible inputs, such as clientid, topic, payload etc.""" + +payload_template.label: +"""Payload template""" - payload_template { - desc { - en: "The template for formatting the outgoing messages. If undefined, rule engine will use JSON format to serialize all visible inputs, such as clientid, topic, payload etc." - zh: "用于格式化写入 MongoDB 的消息模板。 如果未定义，规则引擎会使用 JSON 格式序列化所有的可见输入，例如 clientid, topic, payload 等。" - } - label: { - en: "Payload template" - zh: "有效载荷模板" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_mysql.hocon b/rel/i18n/emqx_ee_bridge_mysql.hocon index 345fd9cba..bd627f726 100644 --- a/rel/i18n/emqx_ee_bridge_mysql.hocon +++ b/rel/i18n/emqx_ee_bridge_mysql.hocon @@ -1,72 +1,43 @@ emqx_ee_bridge_mysql { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to MySQL. All MQTT 'PUBLISH' messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for an HStreamDB bridge.""" + +desc_config.label: +"""HStreamDB Bridge Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to MySQL. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 MySQL。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - sql_template { - desc { - en: """SQL Template""" - zh: """SQL 模板""" - } - label { - en: "SQL Template" - zh: "SQL 模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for an HStreamDB bridge.""" - zh: """HStreamDB 桥接配置""" - } - label: { - en: "HStreamDB Bridge Configuration" - zh: "HStreamDB 桥接配置" - } - } +sql_template.desc: +"""SQL Template""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +sql_template.label: +"""SQL Template""" - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_pgsql.hocon b/rel/i18n/emqx_ee_bridge_pgsql.hocon index 1ce2818ca..94c263a56 100644 --- a/rel/i18n/emqx_ee_bridge_pgsql.hocon +++ b/rel/i18n/emqx_ee_bridge_pgsql.hocon @@ -1,72 +1,43 @@ emqx_ee_bridge_pgsql { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to PostgreSQL. All MQTT 'PUBLISH' messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a PostgreSQL bridge.""" + +desc_config.label: +"""PostgreSQL Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to PostgreSQL. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 PostgreSQL。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - sql_template { - desc { - en: """SQL Template""" - zh: """SQL 模板""" - } - label { - en: "SQL Template" - zh: "SQL 模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for a PostgreSQL bridge.""" - zh: """PostgreSQL 桥接配置""" - } - label: { - en: "PostgreSQL Bridge Configuration" - zh: "PostgreSQL 桥接配置" - } - } +sql_template.desc: +"""SQL Template""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +sql_template.label: +"""SQL Template""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_redis.hocon b/rel/i18n/emqx_ee_bridge_redis.hocon index 4de42b4e3..78db30196 100644 --- a/rel/i18n/emqx_ee_bridge_redis.hocon +++ b/rel/i18n/emqx_ee_bridge_redis.hocon @@ -1,74 +1,45 @@ emqx_ee_bridge_redis { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to Redis. All MQTT 'PUBLISH' messages with the topic + +command_template.desc: +"""Redis command template used to export messages. Each list element stands for a command name or its argument. +For example, to push payloads in a Redis list by key `msgs`, the elements should be the following: +`rpush`, `msgs`, `${payload}`.""" + +command_template.label: +"""Redis Command Template""" + +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a Redis bridge.""" + +desc_config.label: +"""Redis Bridge Configuration""" + +desc_name.desc: +"""Bridge name, used as a human-readable description of the bridge.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to Redis. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 Redis。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 Redis。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - command_template { - desc { - en: """Redis command template used to export messages. Each list element stands for a command name or its argument. -For example, to push payloads in a Redis list by key `msgs`, the elements should be the following: -`rpush`, `msgs`, `${payload}`.""" - zh: """用于推送数据的 Redis 命令模板。 每个列表元素代表一个命令名称或其参数。 -例如，要通过键值 `msgs` 将消息体推送到 Redis 列表中，数组元素应该是： `rpush`, `msgs`, `${payload}`。""" - } - label { - en: "Redis Command Template" - zh: "Redis Command 模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for a Redis bridge.""" - zh: """Resis 桥接配置""" - } - label: { - en: "Redis Bridge Configuration" - zh: "Redis 桥接配置" - } - } - - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } - - desc_name { - desc { - en: """Bridge name, used as a human-readable description of the bridge.""" - zh: """桥接名字，可读描述""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_rocketmq.hocon b/rel/i18n/emqx_ee_bridge_rocketmq.hocon index 2e33e6c07..a545a7fca 100644 --- a/rel/i18n/emqx_ee_bridge_rocketmq.hocon +++ b/rel/i18n/emqx_ee_bridge_rocketmq.hocon @@ -1,70 +1,41 @@ emqx_ee_bridge_rocketmq { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to RocketMQ. All MQTT `PUBLISH` messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a RocketMQ bridge.""" + +desc_config.label: +"""RocketMQ Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to RocketMQ. All MQTT `PUBLISH` messages with the topic matching the `local_topic` will be forwarded.
NOTE: if the bridge is used as a rule action, `local_topic` should be left empty otherwise the messages will be duplicated.""" - zh: """发送到 'local_topic' 的消息都会转发到 RocketMQ。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - template { - desc { - en: """Template, the default value is empty. When this value is empty the whole message will be stored in the RocketMQ""" - zh: """模板, 默认为空，为空时将会将整个消息转发给 RocketMQ""" - } - label { - en: "Template" - zh: "模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for a RocketMQ bridge.""" - zh: """RocketMQ 桥接配置""" - } - label: { - en: "RocketMQ Bridge Configuration" - zh: "RocketMQ 桥接配置" - } - } +template.desc: +"""Template, the default value is empty. When this value is empty the whole message will be stored in the RocketMQ""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +template.label: +"""Template""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_bridge_sqlserver.hocon b/rel/i18n/emqx_ee_bridge_sqlserver.hocon index a6f96f86d..396126622 100644 --- a/rel/i18n/emqx_ee_bridge_sqlserver.hocon +++ b/rel/i18n/emqx_ee_bridge_sqlserver.hocon @@ -1,85 +1,49 @@ emqx_ee_bridge_sqlserver { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to Microsoft SQL Server. All MQTT 'PUBLISH' messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a Microsoft SQL Server bridge.""" + +desc_config.label: +"""Microsoft SQL Server Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +driver.desc: +"""SQL Server Driver Name""" + +driver.label: +"""SQL Server Driver Name""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to Microsoft SQL Server. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 Microsoft SQL Server。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: """Local Topic""" - zh: """本地 Topic""" - } - } - sql_template { - desc { - en: """SQL Template""" - zh: """SQL 模板""" - } - label { - en: """SQL Template""" - zh: """SQL 模板""" - } - } +local_topic.label: +"""Local Topic""" - driver { - desc { - en: """SQL Server Driver Name""" - zh: """SQL Server Driver 名称""" - } - label { - en: """SQL Server Driver Name""" - zh: """SQL Server Driver 名称""" - } - } +sql_template.desc: +"""SQL Template""" - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: """Enable Or Disable Bridge""" - zh: """启用/禁用桥接""" - } - } - - desc_config { - desc { - en: """Configuration for a Microsoft SQL Server bridge.""" - zh: """Microsoft SQL Server 桥接配置""" - } - label: { - en: """Microsoft SQL Server Bridge Configuration""" - zh: """Microsoft SQL Server 桥接配置""" - } - } - - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: """Bridge Type""" - zh: """桥接类型""" - } - } - - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: """Bridge Name""" - zh: """桥接名字""" - } - } +sql_template.label: +"""SQL Template""" } diff --git a/rel/i18n/emqx_ee_bridge_tdengine.hocon b/rel/i18n/emqx_ee_bridge_tdengine.hocon index 12eb6f062..e6ece89c8 100644 --- a/rel/i18n/emqx_ee_bridge_tdengine.hocon +++ b/rel/i18n/emqx_ee_bridge_tdengine.hocon @@ -1,72 +1,43 @@ emqx_ee_bridge_tdengine { - local_topic { - desc { - en: """The MQTT topic filter to be forwarded to TDengine. All MQTT 'PUBLISH' messages with the topic +config_enable.desc: +"""Enable or disable this bridge""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for a TDengine bridge.""" + +desc_config.label: +"""TDengine Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to TDengine. All MQTT 'PUBLISH' messages with the topic matching the local_topic will be forwarded.
NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is configured, then both the data got from the rule and the MQTT messages that match local_topic will be forwarded.""" - zh: """发送到 'local_topic' 的消息都会转发到 TDengine。
-注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" - } - label { - en: "Local Topic" - zh: "本地 Topic" - } - } - sql_template { - desc { - en: """SQL Template""" - zh: """SQL 模板""" - } - label { - en: "SQL Template" - zh: "SQL 模板" - } - } - config_enable { - desc { - en: """Enable or disable this bridge""" - zh: """启用/禁用桥接""" - } - label { - en: "Enable Or Disable Bridge" - zh: "启用/禁用桥接" - } - } +local_topic.label: +"""Local Topic""" - desc_config { - desc { - en: """Configuration for a TDengine bridge.""" - zh: """TDengine 桥接配置""" - } - label: { - en: "TDengine Bridge Configuration" - zh: "TDengine 桥接配置" - } - } +sql_template.desc: +"""SQL Template""" - desc_type { - desc { - en: """The Bridge Type""" - zh: """Bridge 类型""" - } - label { - en: "Bridge Type" - zh: "桥接类型" - } - } +sql_template.label: +"""SQL Template""" - desc_name { - desc { - en: """Bridge name.""" - zh: """桥接名字""" - } - label { - en: "Bridge Name" - zh: "桥接名字" - } - } } diff --git a/rel/i18n/emqx_ee_connector_cassa.hocon b/rel/i18n/emqx_ee_connector_cassa.hocon index ecf004722..bd5fb544c 100644 --- a/rel/i18n/emqx_ee_connector_cassa.hocon +++ b/rel/i18n/emqx_ee_connector_cassa.hocon @@ -1,28 +1,17 @@ emqx_ee_connector_cassa { - servers { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+keyspace.desc: +"""Keyspace name to connect to.""" + +keyspace.label: +"""Keyspace""" + +servers.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port][,Host2:Port]`.
The Cassandra default port 9042 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port][,Host2:Port]`。
-如果未指定 `[:Port]`，则使用 Cassandra 默认端口 9042。""" - } - label: { - en: "Servers" - zh: "Servers" - } - } - keyspace { - desc { - en: """Keyspace name to connect to.""" - zh: """要连接到的 Keyspace 名称。""" - } - label: { - en: "Keyspace" - zh: "Keyspace" - } - } +servers.label: +"""Servers""" + } diff --git a/rel/i18n/emqx_ee_connector_clickhouse.hocon b/rel/i18n/emqx_ee_connector_clickhouse.hocon index 4d30e1715..cebba5aef 100644 --- a/rel/i18n/emqx_ee_connector_clickhouse.hocon +++ b/rel/i18n/emqx_ee_connector_clickhouse.hocon @@ -1,25 +1,15 @@ emqx_ee_connector_clickhouse { - base_url { - desc { - en: """The HTTP URL to the Clickhouse server that you want to connect to (for example http://myhostname:8123)""" - zh: """你想连接到的Clickhouse服务器的HTTP URL（例如http://myhostname:8123）。""" - } - label: { - en: "Server URL" - zh: "服务器 URL" - } - } +base_url.desc: +"""The HTTP URL to the Clickhouse server that you want to connect to (for example http://myhostname:8123)""" - connect_timeout { - desc { - en: "The timeout when connecting to the Clickhouse server." - zh: "连接HTTP服务器的超时时间。" - } - label: { - en: "Clickhouse Timeout" - zh: "连接超时" - } - } +base_url.label: +"""Server URL""" + +connect_timeout.desc: +"""The timeout when connecting to the Clickhouse server.""" + +connect_timeout.label: +"""Clickhouse Timeout""" } diff --git a/rel/i18n/emqx_ee_connector_dynamo.hocon b/rel/i18n/emqx_ee_connector_dynamo.hocon index 295929a72..939efaeec 100644 --- a/rel/i18n/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/emqx_ee_connector_dynamo.hocon @@ -1,14 +1,9 @@ emqx_ee_connector_dynamo { - url { - desc { - en: """The url of DynamoDB endpoint.""" - zh: """DynamoDB 的地址。""" - } - label: { - en: "DynamoDB Endpoint" - zh: "DynamoDB 地址" - } - } +url.desc: +"""The url of DynamoDB endpoint.""" + +url.label: +"""DynamoDB Endpoint""" } diff --git a/rel/i18n/emqx_ee_connector_hstreamdb.hocon b/rel/i18n/emqx_ee_connector_hstreamdb.hocon index 0826c8f0c..f6838297f 100644 --- a/rel/i18n/emqx_ee_connector_hstreamdb.hocon +++ b/rel/i18n/emqx_ee_connector_hstreamdb.hocon @@ -1,74 +1,45 @@ emqx_ee_connector_hstreamdb { - config { - desc { - en: "HStreamDB connection config" - zh: "HStreamDB 连接配置。" - } - label: { - en: "Connection config" - zh: "连接配置" - } - } - type { - desc { - en: "The Connector Type." - zh: "连接器类型。" - } - label: { - en: "Connector Type" - zh: "连接器类型" - } - } +config.desc: +"""HStreamDB connection config""" + +config.label: +"""Connection config""" + +name.desc: +"""Connector name, used as a human-readable description of the connector.""" + +name.label: +"""Connector Name""" + +ordering_key.desc: +"""HStreamDB Ordering Key""" + +ordering_key.label: +"""HStreamDB Ordering Key""" + +pool_size.desc: +"""HStreamDB Pool Size""" + +pool_size.label: +"""HStreamDB Pool Size""" + +stream_name.desc: +"""HStreamDB Stream Name""" + +stream_name.label: +"""HStreamDB Stream Name""" + +type.desc: +"""The Connector Type.""" + +type.label: +"""Connector Type""" + +url.desc: +"""HStreamDB Server URL""" + +url.label: +"""HStreamDB Server URL""" - name { - desc { - en: "Connector name, used as a human-readable description of the connector." - zh: "连接器名称，人类可读的连接器描述。" - } - label: { - en: "Connector Name" - zh: "连接器名称" - } - } - url { - desc { - en: """HStreamDB Server URL""" - zh: """HStreamDB 服务器 URL""" - } - label { - en: """HStreamDB Server URL""" - zh: """HStreamDB 服务器 URL""" - } - } - stream_name { - desc { - en: """HStreamDB Stream Name""" - zh: """HStreamDB 流名称""" - } - label { - en: """HStreamDB Stream Name""" - zh: """HStreamDB 流名称""" - } - } - ordering_key { - desc { - en: """HStreamDB Ordering Key""" - zh: """HStreamDB 分区键""" - } - label { - en: """HStreamDB Ordering Key""" - zh: """HStreamDB 分区键""" - } - } - pool_size { - desc { - en: """HStreamDB Pool Size""" - zh: """HStreamDB 连接池大小""" - } - label { - en: """HStreamDB Pool Size""" - zh: """HStreamDB 连接池大小""" - } - } } diff --git a/rel/i18n/emqx_ee_connector_influxdb.hocon b/rel/i18n/emqx_ee_connector_influxdb.hocon index 18ff48109..9c3b143a2 100644 --- a/rel/i18n/emqx_ee_connector_influxdb.hocon +++ b/rel/i18n/emqx_ee_connector_influxdb.hocon @@ -1,118 +1,71 @@ emqx_ee_connector_influxdb { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+bucket.desc: +"""InfluxDB bucket name.""" + +bucket.label: +"""Bucket""" + +database.desc: +"""InfluxDB database.""" + +database.label: +"""Database""" + +influxdb_api_v1.desc: +"""InfluxDB's protocol. Support InfluxDB v1.8 and before.""" + +influxdb_api_v1.label: +"""HTTP API Protocol""" + +influxdb_api_v2.desc: +"""InfluxDB's protocol. Support InfluxDB v2.0 and after.""" + +influxdb_api_v2.label: +"""HTTP API V2 Protocol""" + +org.desc: +"""Organization name of InfluxDB.""" + +org.label: +"""Organization""" + +password.desc: +"""InfluxDB password.""" + +password.label: +"""Password""" + +precision.desc: +"""InfluxDB time precision.""" + +precision.label: +"""Time Precision""" + +protocol.desc: +"""InfluxDB's protocol. HTTP API or HTTP API V2.""" + +protocol.label: +"""Protocol""" + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The InfluxDB default port 8086 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 InfluxDB 默认端口 8086。""" - } - label { - en: "Server Host" - zh: "服务器地址" - } - } - precision { - desc { - en: """InfluxDB time precision.""" - zh: """InfluxDB 时间精度。""" - } - label { - en: """Time Precision""" - zh: """时间精度""" - } - } - protocol { - desc { - en: """InfluxDB's protocol. HTTP API or HTTP API V2.""" - zh: """InfluxDB 协议。HTTP API 或 HTTP API V2。""" - } - label { - en: """Protocol""" - zh: """协议""" - } - } - influxdb_api_v1 { - desc { - en: """InfluxDB's protocol. Support InfluxDB v1.8 and before.""" - zh: """InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本。""" - } - label { - en: """HTTP API Protocol""" - zh: """HTTP API 协议""" - } - } - influxdb_api_v2 { - desc { - en: """InfluxDB's protocol. Support InfluxDB v2.0 and after.""" - zh: """InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本。""" - } - label { - en: """HTTP API V2 Protocol""" - zh: """HTTP API V2 协议""" - } - } - database { - desc { - en: """InfluxDB database.""" - zh: """InfluxDB 数据库。""" - } - label { - en: "Database" - zh: "数据库" - } - } - username { - desc { - en: "InfluxDB username." - zh: "InfluxDB 用户名。" - } - label { - en: "Username" - zh: "用户名" - } - } - password { - desc { - en: "InfluxDB password." - zh: "InfluxDB 密码。" - } - label { - en: "Password" - zh: "密码" - } - } - bucket { - desc { - en: "InfluxDB bucket name." - zh: "InfluxDB bucket 名称。" - } - label { - en: "Bucket" - zh: "Bucket" - } - } - org { - desc { - en: """Organization name of InfluxDB.""" - zh: """InfluxDB 组织名称。""" - } - label { - en: """Organization""" - zh: """组织""" - } - } - token { - desc { - en: """InfluxDB token.""" - zh: """InfluxDB token。""" - } - label { - en: """Token""" - zh: """Token""" - } - } + +server.label: +"""Server Host""" + +token.desc: +"""InfluxDB token.""" + +token.label: +"""Token""" + +username.desc: +"""InfluxDB username.""" + +username.label: +"""Username""" } diff --git a/rel/i18n/emqx_ee_connector_rocketmq.hocon b/rel/i18n/emqx_ee_connector_rocketmq.hocon index 44dda7931..672dcafce 100644 --- a/rel/i18n/emqx_ee_connector_rocketmq.hocon +++ b/rel/i18n/emqx_ee_connector_rocketmq.hocon @@ -1,62 +1,35 @@ emqx_ee_connector_rocketmq { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+refresh_interval.desc: +"""RocketMQ Topic Route Refresh Interval.""" + +refresh_interval.label: +"""Topic Route Refresh Interval""" + +security_token.desc: +"""RocketMQ Server Security Token""" + +security_token.label: +"""Security Token""" + +send_buffer.desc: +"""The socket send buffer size of the RocketMQ driver client.""" + +send_buffer.label: +"""Send Buffer Size""" + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The RocketMQ default port 9876 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } - topic { - desc { - en: """RocketMQ Topic""" - zh: """RocketMQ 主题""" - } - label: { - en: "RocketMQ Topic" - zh: "RocketMQ 主题" - } - } +server.label: +"""Server Host""" - refresh_interval { - desc { - en: """RocketMQ Topic Route Refresh Interval.""" - zh: """RocketMQ 主题路由更新间隔。""" - } - label: { - en: "Topic Route Refresh Interval" - zh: "主题路由更新间隔" - } - } +topic.desc: +"""RocketMQ Topic""" - send_buffer { - desc { - en: """The socket send buffer size of the RocketMQ driver client.""" - zh: """RocketMQ 驱动的套字节发送消息的缓冲区大小""" - } - label: { - en: "Send Buffer Size" - zh: "发送消息的缓冲区大小" - } - } - - security_token { - desc { - en: """RocketMQ Server Security Token""" - zh: """RocketMQ 服务器安全令牌""" - } - label: { - en: "Security Token" - zh: "安全令牌" - } - } +topic.label: +"""RocketMQ Topic""" } diff --git a/rel/i18n/emqx_ee_connector_sqlserver.hocon b/rel/i18n/emqx_ee_connector_sqlserver.hocon index 85280d833..ef68865fe 100644 --- a/rel/i18n/emqx_ee_connector_sqlserver.hocon +++ b/rel/i18n/emqx_ee_connector_sqlserver.hocon @@ -1,18 +1,11 @@ emqx_ee_connector_sqlserver { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The SQL Server default port 1433 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 SQL Server 默认端口 1433。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } + +server.label: +"""Server Host""" } diff --git a/rel/i18n/emqx_ee_connector_tdengine.hocon b/rel/i18n/emqx_ee_connector_tdengine.hocon index 02254124c..9a34b32ce 100644 --- a/rel/i18n/emqx_ee_connector_tdengine.hocon +++ b/rel/i18n/emqx_ee_connector_tdengine.hocon @@ -1,18 +1,11 @@ emqx_ee_connector_tdengine { - server { - desc { - en: """The IPv4 or IPv6 address or the hostname to connect to.
+server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
A host entry has the following form: `Host[:Port]`.
The TDengine default port 6041 is used if `[:Port]` is not specified.""" - zh: """将要连接的 IPv4 或 IPv6 地址，或者主机名。
-主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 TDengine 默认端口 6041。""" - } - label: { - en: "Server Host" - zh: "服务器地址" - } - } + +server.label: +"""Server Host""" } diff --git a/rel/i18n/emqx_ee_schema_registry_http_api.hocon b/rel/i18n/emqx_ee_schema_registry_http_api.hocon index 058796a66..09f268459 100644 --- a/rel/i18n/emqx_ee_schema_registry_http_api.hocon +++ b/rel/i18n/emqx_ee_schema_registry_http_api.hocon @@ -1,69 +1,39 @@ emqx_ee_schema_registry_http_api { - # apis - desc_schema_registry_api_list { - desc { - en: "List all registered schemas" - zh: "列出所有注册的模式" - } - label { - en: "List schemas" - zh: "列表模式" - } - } - desc_schema_registry_api_get { - desc { - en: "Get a schema by its name" - zh: "通过名称获取模式" - } - label { - en: "Get schema" - zh: "获取模式" - } - } +desc_param_path_schema_name.desc: +"""The schema name""" - desc_schema_registry_api_post { - desc { - en: "Register a new schema" - zh: "注册一个新的模式" - } - label { - en: "Register schema" - zh: "注册模式" - } - } +desc_param_path_schema_name.label: +"""Schema name""" - desc_schema_registry_api_put { - desc { - en: "Update an existing schema" - zh: "更新一个现有的模式" - } - label { - en: "Update schema" - zh: "更新模式" - } - } +desc_schema_registry_api_delete.desc: +"""Delete a schema""" - desc_schema_registry_api_delete { - desc { - en: "Delete a schema" - zh: "删除一个模式" - } - label { - en: "Delete schema" - zh: "删除模式" - } - } +desc_schema_registry_api_delete.label: +"""Delete schema""" + +desc_schema_registry_api_get.desc: +"""Get a schema by its name""" + +desc_schema_registry_api_get.label: +"""Get schema""" + +desc_schema_registry_api_list.desc: +"""List all registered schemas""" + +desc_schema_registry_api_list.label: +"""List schemas""" + +desc_schema_registry_api_post.desc: +"""Register a new schema""" + +desc_schema_registry_api_post.label: +"""Register schema""" + +desc_schema_registry_api_put.desc: +"""Update an existing schema""" + +desc_schema_registry_api_put.label: +"""Update schema""" - # params - desc_param_path_schema_name { - desc { - en: "The schema name" - zh: "模式名称" - } - label { - en: "Schema name" - zh: "模式名称" - } - } } diff --git a/rel/i18n/emqx_ee_schema_registry_schema.hocon b/rel/i18n/emqx_ee_schema_registry_schema.hocon index 1538fe5f9..667c4c0a4 100644 --- a/rel/i18n/emqx_ee_schema_registry_schema.hocon +++ b/rel/i18n/emqx_ee_schema_registry_schema.hocon @@ -1,78 +1,45 @@ emqx_ee_schema_registry_schema { - schema_registry_root { - desc { - en: "Schema registry configurations." - zh: "模式注册表的配置。" - } - label { - en: "Schema registry" - zh: "模式注册表" - } - } - schema_registry_schemas { - desc { - en: "Registered schemas." - zh: "注册的模式。" - } - label { - en: "Registered schemas" - zh: "注册的模式" - } - } +avro_type.desc: +"""[Apache Avro](https://avro.apache.org/) serialization format.""" - schema_name { - desc { - en: "A name for the schema that will serve as its identifier." - zh: "模式的一个名称，将作为其标识符。" - } - label { - en: "Schema name" - zh: "模式名称" - } - } +avro_type.label: +"""Apache Avro""" - schema_type { - desc { - en: "Schema type." - zh: "模式类型。" - } - label { - en: "Schema type" - zh: "模式类型" - } - } +schema_description.desc: +"""A description for this schema.""" - schema_source { - desc { - en: "Source text for the schema." - zh: "模式的源文本。" - } - label { - en: "Schema source" - zh: "模式来源" - } - } +schema_description.label: +"""Schema description""" - schema_description { - desc { - en: "A description for this schema." - zh: "对该模式的描述。" - } - label { - en: "Schema description" - zh: "模式描述" - } - } +schema_name.desc: +"""A name for the schema that will serve as its identifier.""" + +schema_name.label: +"""Schema name""" + +schema_registry_root.desc: +"""Schema registry configurations.""" + +schema_registry_root.label: +"""Schema registry""" + +schema_registry_schemas.desc: +"""Registered schemas.""" + +schema_registry_schemas.label: +"""Registered schemas""" + +schema_source.desc: +"""Source text for the schema.""" + +schema_source.label: +"""Schema source""" + +schema_type.desc: +"""Schema type.""" + +schema_type.label: +"""Schema type""" - avro_type { - desc { - en: "[Apache Avro](https://avro.apache.org/) serialization format." - zh: "[阿帕奇-阿夫罗](https://avro.apache.org/) 序列化格式。" - } - label { - en: "Apache Avro" - zh: "阿帕奇-阿夫罗" - } - } } diff --git a/rel/i18n/emqx_exhook_api.hocon b/rel/i18n/emqx_exhook_api.hocon index 3ec5367ed..9cb7177c1 100644 --- a/rel/i18n/emqx_exhook_api.hocon +++ b/rel/i18n/emqx_exhook_api.hocon @@ -1,180 +1,81 @@ emqx_exhook_api { - list_all_servers { - desc { - en: "List all servers" - zh: "查看ExHook 服务器列表" - } - } +add_server.desc: +"""Add a server""" - add_server { - desc { - en: "Add a server" - zh: "添加 ExHook 服务器" - } - } +delete_server.desc: +"""Delete the server""" - get_detail { - desc { - en: "Get the detail information of Exhook server" - zh: "查看 Exhook 服务器详细信息" - } - } +get_detail.desc: +"""Get the detail information of Exhook server""" - update_server { - desc { - en: "Update the server" - zh: "更新 Exhook 服务器" - } - } +get_hooks.desc: +"""Get the hooks information of server""" - delete_server { - desc { - en: "Delete the server" - zh: "删除 Exhook 服务器" - } - } +hook_metrics.desc: +"""Metrics information of this hook in the current node""" - get_hooks { - desc { - en: "Get the hooks information of server" - zh: "获取 Exhook 服务器的钩子信息" - } - } +hook_name.desc: +"""The hook's name""" - move_api { - desc { - en: """Move the server. -NOTE: The position should be \"front | rear | before:{name} | after:{name}""" - zh: """移动 Exhook 服务器顺序。 -注意: 移动的参数只能是：front | rear | before:{name} | after:{name}""" - } - label { - en: "Change order of execution for registered Exhook server" - zh: "改变已注册的Exhook服务器的执行顺序" - } - } +hook_params.desc: +"""The parameters used when the hook is registered""" - move_position { - desc { - en: "The target position to be moved" - zh: "移动的方向" - } - } +list_all_servers.desc: +"""List all servers""" - hook_name { - desc { - en: "The hook's name" - zh: "钩子的名称" - } - } +metric_failed.desc: +"""The number of times the hook execution failed""" - server_name { - desc { - en: "The Exhook server name" - zh: "Exhook 服务器的名称" - } - } +metric_max_rate.desc: +"""Maximum call rate of hooks""" - hook_params { - desc { - en: "The parameters used when the hook is registered" - zh: "钩子注册时使用的参数" - } - } +metric_rate.desc: +"""The call rate of hooks""" - server_metrics { - desc { - en: "Metrics information of this server in the current node" - zh: "当前节点中该服务器的指标信息" - } - } +metric_succeed.desc: +"""The number of times the hooks execution successful""" - node_metrics { - desc { - en: "Metrics information of this server in all nodes" - zh: "所有节点中该服务器的指标信息" - } - } +metrics.desc: +"""Metrics information""" - node_status { - desc { - en: "status of this server in all nodes" - zh: "所有节点中该服务器的状态信息" - } - } +move_api.desc: +"""Move the server. +NOTE: The position should be "front | rear | before:{name} | after:{name}""" - hook_metrics { - desc { - en: "Metrics information of this hook in the current node" - zh: "当前节点中该钩子的指标信息" - } - } +move_api.label: +"""Change order of execution for registered Exhook server""" - node_hook_metrics { - desc { - en: "Metrics information of this hook in all nodes" - zh: "所有节点中该钩子的指标信息" - } - } +move_position.desc: +"""The target position to be moved""" - node { - desc { - en: "Node name" - zh: "节点名称" - } - } +node.desc: +"""Node name""" - metrics { - desc { - en: "Metrics information" - zh: "指标信息" - } - } +node_hook_metrics.desc: +"""Metrics information of this hook in all nodes""" - status { - desc { - en: """The status of Exhook server.
+node_metrics.desc: +"""Metrics information of this server in all nodes""" + +node_status.desc: +"""status of this server in all nodes""" + +server_metrics.desc: +"""Metrics information of this server in the current node""" + +server_name.desc: +"""The Exhook server name""" + +status.desc: +"""The status of Exhook server.
connected: connection succeeded
connecting: connection failed, reconnecting
disconnected: failed to connect and didn't reconnect
disabled: this server is disabled
error: failed to view the status of this server""" - zh: """Exhook 服务器的状态。
-connected: 连接成功
-connecting: 连接失败，重连中
-disconnected: 连接失败，且未设置自动重连
-disabled: 该服务器未开启
-error: 查看该服务器状态时发生错误""" - } - } - metric_succeed { - desc { - en: "The number of times the hooks execution successful" - zh: "钩子执行成功的次数" - } - } - - metric_failed { - desc { - en: "The number of times the hook execution failed" - zh: "钩子执行失败的次数" - } - } - - metric_rate { - desc { - en: "The call rate of hooks" - zh: "钩子的调用速率" - } - } - - metric_max_rate { - desc { - en: "Maximum call rate of hooks" - zh: "钩子的最大调用速率" - } - } +update_server.desc: +"""Update the server""" } diff --git a/rel/i18n/emqx_exhook_schema.hocon b/rel/i18n/emqx_exhook_schema.hocon index 5b34a245a..6d262fae7 100644 --- a/rel/i18n/emqx_exhook_schema.hocon +++ b/rel/i18n/emqx_exhook_schema.hocon @@ -1,97 +1,45 @@ emqx_exhook_schema { - servers { - desc { - en: "List of exhook servers" - zh: "ExHook 服务器列表" - } - } - - name { - desc { - en: "Name of the exhook server" - zh: "ExHook 服务器名称" - } - } - - enable { - desc { - en: "Enable this Exhook server" - zh: "开启这个 Exhook 服务器" - } - } - - url { - desc { - en: "URL of the gRPC server" - zh: "gRPC 服务器地址" - } - } - - request_timeout { - desc { - en: "The timeout of request gRPC server" - zh: "gRPC 服务器请求超时时间" - } - } - - failed_action { - desc { - en: "The value that is returned when the request to the gRPC server fails for any reason" - zh: "当 gRPC 请求失败后的操作" - } - } - - auto_reconnect { - desc { - en: """Whether to automatically reconnect (initialize) the gRPC server. +auto_reconnect.desc: +"""Whether to automatically reconnect (initialize) the gRPC server. When gRPC is not available, Exhook tries to request the gRPC service at that interval and reinitialize the list of mounted hooks.""" - zh: """自动重连到 gRPC 服务器的设置。 -当 gRPC 服务器不可用时，Exhook 将会按照这里设置的间隔时间进行重连，并重新初始化注册的钩子""" - } - } - pool_size { - desc { - en: "The process pool size for gRPC client" - zh: "gRPC 客户端进程池大小" - } - } +enable.desc: +"""Enable this Exhook server""" - socket_options { - desc { - en: "Connection socket options" - zh: "连接套接字设置" - } - } +failed_action.desc: +"""The value that is returned when the request to the gRPC server fails for any reason""" - keepalive { - desc { - en: """Enables/disables periodic transmission on a connected socket when no other data is exchanged. +keepalive.desc: +"""Enables/disables periodic transmission on a connected socket when no other data is exchanged. If the other end does not respond, the connection is considered broken and an error message is sent to the controlling process.""" - zh: """当没有其他数据交换时，是否向连接的对端套接字定期的发送探测包。如果另一端没有响应，则认为连接断开，并向控制进程发送错误消息""" - } - } - nodelay { - desc { - en: """If true, option TCP_NODELAY is turned on for the socket, +name.desc: +"""Name of the exhook server""" + +nodelay.desc: +"""If true, option TCP_NODELAY is turned on for the socket, which means that also small amounts of data are sent immediately""" - zh: "如果为 true，则为套接字设置 TCP_NODELAY 选项，这意味着会立即发送数据包" - } - } - recbuf { - desc { - en: "The minimum size of receive buffer to use for the socket" - zh: "套接字的最小接收缓冲区大小" - } - } +pool_size.desc: +"""The process pool size for gRPC client""" + +recbuf.desc: +"""The minimum size of receive buffer to use for the socket""" + +request_timeout.desc: +"""The timeout of request gRPC server""" + +servers.desc: +"""List of exhook servers""" + +sndbuf.desc: +"""The minimum size of send buffer to use for the socket""" + +socket_options.desc: +"""Connection socket options""" + +url.desc: +"""URL of the gRPC server""" - sndbuf { - desc { - en: "The minimum size of send buffer to use for the socket" - zh: "套接字的最小发送缓冲区大小" - } - } } diff --git a/rel/i18n/emqx_exproto_schema.hocon b/rel/i18n/emqx_exproto_schema.hocon index 0c6fd2286..eed450208 100644 --- a/rel/i18n/emqx_exproto_schema.hocon +++ b/rel/i18n/emqx_exproto_schema.hocon @@ -1,52 +1,24 @@ emqx_exproto_schema { - exproto { - desc { - en: """The Extension Protocol configuration""" - zh: """ExProto 网关""" - } - } - exproto_server { - desc { - en: """Configurations for starting the ConnectionAdapter service""" - zh: """配置 ExProto 网关需要启动的 ConnectionAdapter 服务。 -该服务用于提供客户端的认证、发布、订阅和数据下行等功能。""" - } - } +exproto.desc: +"""The Extension Protocol configuration""" - exproto_grpc_server_bind { - desc { - en: """Listening address and port for the gRPC server.""" - zh: """服务监听地址和端口。""" - } - } +exproto_grpc_handler_address.desc: +"""gRPC server address.""" - exproto_grpc_server_ssl { - desc { - en: """SSL configuration for the gRPC server.""" - zh: """服务 SSL 配置。""" - } - } +exproto_grpc_handler_ssl.desc: +"""SSL configuration for the gRPC client.""" - exproto_handler { - desc { - en: """Configurations for request to ConnectionHandler service""" - zh: """配置 ExProto 网关需要请求的 ConnectionHandler 服务地址。 -该服务用于给 ExProto 提供客户端的 Socket 事件处理、字节解码、订阅消息接收等功能。""" - } - } +exproto_grpc_server_bind.desc: +"""Listening address and port for the gRPC server.""" - exproto_grpc_handler_address { - desc { - en: """gRPC server address.""" - zh: """对端 gRPC 服务器地址。""" - } - } +exproto_grpc_server_ssl.desc: +"""SSL configuration for the gRPC server.""" + +exproto_handler.desc: +"""Configurations for request to ConnectionHandler service""" + +exproto_server.desc: +"""Configurations for starting the ConnectionAdapter service""" - exproto_grpc_handler_ssl { - desc { - en: """SSL configuration for the gRPC client.""" - zh: """gRPC 客户端的 SSL 配置。""" - } - } } diff --git a/rel/i18n/emqx_gateway_api.hocon b/rel/i18n/emqx_gateway_api.hocon index 18ab1f242..1e0e22456 100644 --- a/rel/i18n/emqx_gateway_api.hocon +++ b/rel/i18n/emqx_gateway_api.hocon @@ -1,166 +1,73 @@ emqx_gateway_api { - list_gateway { - desc { - en: """This API returns an overview info for the specified or all gateways. -including current running status, number of connections, listener status, etc.""" - zh: """该接口会返回指定或所有网关的概览状态， -包括当前状态、连接数、监听器状态等。""" - } - } +delete_gateway.desc: +"""Unload the specified gateway""" - enable_gateway { - desc { - en: """Enable a gateway by confs.""" - zh: """使用配置启动某一网关。""" - } - } +enable_gateway.desc: +"""Enable a gateway by confs.""" - get_gateway { - desc { - en: """Get the gateway configurations""" - zh: """获取网关配置详情""" - } - } +gateway_created_at.desc: +"""The Gateway created datetime""" - delete_gateway { - desc { - en: """Unload the specified gateway""" - zh: """停用指定网关""" - } - } +gateway_current_connections.desc: +"""The Gateway current connected connections/clients""" - update_gateway { - desc { - en: """Update the gateway basic configurations and running status.
-Note: The Authentication and Listener configurations should be updated by other special APIs.""" - zh: """更新指定网关的基础配置、和启用的状态。
-注：认证、和监听器的配置更新需参考对应的 API 接口。""" - } - } +gateway_enable_in_path.desc: +"""Whether to enable this gateway""" - gateway_name { - desc { - en: """Gateway Name""" - zh: """网关名称""" - } - } +gateway_listener_id.desc: +"""Listener ID""" - gateway_name_in_qs { - desc { - en: """Gateway Name.
+gateway_listener_name.desc: +"""Listener Name""" + +gateway_listener_running.desc: +"""Listener Running status""" + +gateway_listener_type.desc: +"""Listener Type""" + +gateway_listeners.desc: +"""The Gateway listeners overview""" + +gateway_max_connections.desc: +"""The Gateway allowed maximum connections/clients""" + +gateway_name.desc: +"""Gateway Name""" + +gateway_name_in_qs.desc: +"""Gateway Name.
It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto`""" - zh: """网关名称.
-可取值为 `stomp`、`mqttsn`、`coap`、`lwm2m`、`exproto`""" - } - } - gateway_enable_in_path { - desc { - en: """Whether to enable this gateway""" +gateway_node_status.desc: +"""The status of the gateway on each node in the cluster""" - zh: """是否开启此网关""" - } - } +gateway_started_at.desc: +"""The Gateway started datetime""" - gateway_status { - desc { - en: """Gateway status""" - zh: """网关启用状态""" - } - } +gateway_status.desc: +"""Gateway status""" - gateway_status_in_qs { - desc { - en: """Filter gateways by status.
+gateway_status_in_qs.desc: +"""Filter gateways by status.
It is enum with `running`, `stopped`, `unloaded`""" - zh: """通过网关状态筛选
-可选值为 `running`、`stopped`、`unloaded`""" - } - } - gateway_created_at { - desc { - en: """The Gateway created datetime""" - zh: """网关创建时间""" - } - } +gateway_stopped_at.desc: +"""The Gateway stopped datetime""" - gateway_started_at { - desc { - en: """The Gateway started datetime""" - zh: """网关启用时间""" - } - } +get_gateway.desc: +"""Get the gateway configurations""" - gateway_stopped_at { - desc { - en: """The Gateway stopped datetime""" - zh: """网关停用时间""" - } - } +list_gateway.desc: +"""This API returns an overview info for the specified or all gateways. +including current running status, number of connections, listener status, etc.""" - gateway_max_connections { - desc { - en: """The Gateway allowed maximum connections/clients""" - zh: """最大连接数""" - } - } +node.desc: +"""Node Name""" - gateway_current_connections { - desc { - en: """The Gateway current connected connections/clients""" - zh: """当前连接数""" - } - } - - gateway_listeners { - desc { - en: """The Gateway listeners overview""" - zh: """网关监听器列表""" - } - } - - gateway_listener_id { - desc { - en: """Listener ID""" - zh: """监听器 ID""" - } - } - - gateway_listener_name { - desc { - en: """Listener Name""" - zh: """监听器名称""" - } - } - - gateway_listener_running { - desc { - en: """Listener Running status""" - zh: """监听器运行状态""" - } - } - - gateway_listener_type { - desc { - en: """Listener Type""" - zh: """监听器类型""" - } - } - - gateway_node_status { - desc { - en: """The status of the gateway on each node in the cluster""" - zh: """网关在集群中每个节点上的状态""" - } - } - - node { - desc { - en: """Node Name""" - zh: """节点名称""" - } - } +update_gateway.desc: +"""Update the gateway basic configurations and running status.
+Note: The Authentication and Listener configurations should be updated by other special APIs.""" } diff --git a/rel/i18n/emqx_gateway_api_authn.hocon b/rel/i18n/emqx_gateway_api_authn.hocon index a9ae33f0c..2d84eef54 100644 --- a/rel/i18n/emqx_gateway_api_authn.hocon +++ b/rel/i18n/emqx_gateway_api_authn.hocon @@ -1,99 +1,45 @@ emqx_gateway_api_authn { - get_authn { - desc { - en: """Gets the configuration of the specified gateway authenticator.
-Returns 404 when gateway or authentication is not enabled.""" - zh: """获取指定网关认证器的配置 -当网关或认证未启用时，返回 404。""" - } - } - - update_authn { - desc { - en: """Update the configuration of the specified gateway authenticator, or disable the authenticator.""" - zh: """更新指定网关认证器的配置，或停用认证器。""" - } - } - - add_authn { - desc { - en: """Enables the authenticator for client authentication for the specified gateway.
+add_authn.desc: +"""Enables the authenticator for client authentication for the specified gateway.
When the authenticator is not configured or turned off, all client connections are assumed to be allowed.
Note: Only one authenticator is allowed to be enabled at a time in the gateway, rather than allowing multiple authenticators to be configured to form an authentication chain as in MQTT.""" - zh: """为指定网关开启认证器实现客户端认证的功能。
-当未配置认证器或关闭认证器时，则认为允许所有客户端的连接。
-注：在网关中仅支持添加一个认证器，而不是像 MQTT 一样允许配置多个认证器构成认证链。""" - } - } - delete_authn { - desc { - en: """Delete the authenticator of the specified gateway.""" - zh: """删除指定网关的认证器。""" - } - } +add_user.desc: +"""Add user for the authenticator (only supports built_in_database).""" - list_users { - desc { - en: """Get the users for the authenticator (only supported by built_in_database).""" - zh: """获取用户列表（仅支持 built_in_database 类型的认证器）""" - } - } +delete_authn.desc: +"""Delete the authenticator of the specified gateway.""" - add_user { - desc { - en: """Add user for the authenticator (only supports built_in_database).""" - zh: """添加用户（仅支持 built_in_database 类型的认证器）""" - } - } +delete_user.desc: +"""Delete the user for the gateway authenticator (only supports built_in_database)""" - get_user { - desc { - en: """Get user info from the gateway authenticator (only supports built_in_database)""" - zh: """获取用户信息（仅支持 built_in_database 类型的认证器）""" - } - } +get_authn.desc: +"""Gets the configuration of the specified gateway authenticator.
+Returns 404 when gateway or authentication is not enabled.""" - update_user { - desc { - en: """Update the user info for the gateway authenticator (only supports built_in_database)""" - zh: """更新用户信息（仅支持 built_in_database 类型的认证器）""" - } - } +get_user.desc: +"""Get user info from the gateway authenticator (only supports built_in_database)""" - delete_user { - desc { - en: """Delete the user for the gateway authenticator (only supports built_in_database)""" - zh: """删除用户（仅支持 built_in_database 类型的认证器）""" - } - } +import_users.desc: +"""Import users into the gateway authenticator (only supports built_in_database)""" - import_users { - desc { - en: """Import users into the gateway authenticator (only supports built_in_database)""" - zh: """导入用户（仅支持 built_in_database 类型的认证器）""" - } - } +is_superuser.desc: +"""Is superuser""" - user_id { - desc { - en: """User ID""" - zh: """用户 ID""" - } - } +like_user_id.desc: +"""Fuzzy search using user ID (username or clientid), only supports search by substring.""" - like_user_id { - desc { - en: """Fuzzy search using user ID (username or clientid), only supports search by substring.""" - zh: """使用用户 ID （username 或 clientid）模糊搜索，仅支持按子串的方式进行搜索。""" - } - } +list_users.desc: +"""Get the users for the authenticator (only supported by built_in_database).""" + +update_authn.desc: +"""Update the configuration of the specified gateway authenticator, or disable the authenticator.""" + +update_user.desc: +"""Update the user info for the gateway authenticator (only supports built_in_database)""" + +user_id.desc: +"""User ID""" - is_superuser { - desc { - en: """Is superuser""" - zh: """是否是超级用户""" - } - } } diff --git a/rel/i18n/emqx_gateway_api_clients.hocon b/rel/i18n/emqx_gateway_api_clients.hocon index 1e6f575c3..4c95318ab 100644 --- a/rel/i18n/emqx_gateway_api_clients.hocon +++ b/rel/i18n/emqx_gateway_api_clients.hocon @@ -1,478 +1,207 @@ emqx_gateway_api_clients { - list_clients { - desc { - en: """Get the gateway client list""" - zh: """获取指定网关的客户端列表""" - } - } - - get_client { - desc { - en: """Get the gateway client information""" - zh: """获取客户端信息""" - } - } - - kick_client { - desc { - en: """Kick out the gateway client""" - zh: """踢出指定客户端""" - } - } - - list_subscriptions { - desc { - en: """Get the gateway client subscriptions""" - zh: """获取某客户端的主题订阅列表""" - } - } - - add_subscription { - desc { - en: """Create a subscription membership""" - zh: """为某客户端新增订阅关系""" - } - } - - delete_subscription { - desc { - en: """Delete a subscriptions membership""" - zh: """为某客户端删除某订阅关系""" - } - } - - param_node { - desc { - en: """Match the client's node name""" - zh: """匹配客户端的节点名称""" - } - } - - param_clientid { - desc { - en: """Match the client's ID""" - zh: """匹配客户端 ID""" - } - } - - param_username { - desc { - en: """Match the client's Username""" - zh: """匹配客户端 Username""" - } - } - - param_ip_address { - desc { - en: """Match the client's ip address""" - zh: """匹配客户端 IP 地址""" - } - } - - param_conn_state { - desc { - en: """Match the client's connection state""" - zh: """匹配客户端连接状态""" - } - } - - param_proto_ver { - desc { - en: """Match the client's protocol version""" - zh: """匹配客户端协议版本""" - } - } - - param_clean_start { - desc { - en: """Match the client's clean start flag""" - zh: """匹配客户端 `clean_start` 标记""" - } - } - - param_like_clientid { - desc { - en: """Use sub-string to match client's ID""" - zh: """子串匹配客户端 ID""" - } - } - - param_like_username { - desc { - en: """Use sub-string to match client's username""" - zh: """子串匹配 客户端 Username""" - } - } - - param_gte_created_at { - desc { - en: """Match the session created datetime greater than a certain value""" - zh: """匹配会话创建时间大于等于指定值的客户端""" - } - } - - param_lte_created_at { - desc { - en: """Match the session created datetime less than a certain value""" - zh: """匹配会话创建时间小于等于指定值的客户端""" - } - } - - param_gte_connected_at{ - desc { - en: """Match the client socket connected datetime greater than a certain value""" - zh: """匹配连接创建时间大于等于指定值的客户端""" - } - } - - param_lte_connected_at { - desc { - en: """Match the client socket connected datatime less than a certain value""" - zh: """匹配连接创建时间小于等于指定值的客户端""" - } - } - - param_endpoint_name { - desc { - en: """Match the lwm2m client's endpoint name""" - zh: """匹配 LwM2M 客户端 Endpoint Name""" - } - } - - param_like_endpoint_name { - desc { - en: """Use sub-string to match lwm2m client's endpoint name""" - zh: """子串匹配 LwM2M 客户端 Endpoint Name""" - } - } - - param_gte_lifetime { - desc { - en: """Match the lwm2m client registered lifetime greater than a certain value""" - zh: """匹配心跳时间大于等于指定值的 LwM2M 客户端""" - } - } - - param_lte_lifetime { - desc { - en: """Match the lwm2m client registered lifetime less than a certain value""" - zh: """匹配心跳时间小于等于指定值的 LwM2M 客户端""" - } - } - - clientid { - desc { - en: """Client ID""" - zh: """客户端 ID""" - } - } - - topic { - desc { - en: """Topic Filter/Name""" - zh: """主题过滤器或主题名称""" - } - } - - endpoint_name { - desc { - en: """The LwM2M client endpoint name""" - zh: """LwM2M 客户端 Endpoint Name""" - } - } - - lifetime { - desc { - en: """LwM2M Life time""" - zh: """LwM2M 客户端心跳周期""" - } - } - - qos { - desc { - en: """QoS level, enum: 0, 1, 2""" - zh: """QoS 等级，枚举：0，1，2""" - } - } - - nl { - desc { - en: """No Local option, enum: 0, 1""" - zh: """No Local 选项，枚举：0，1""" - } - } - - rap { - desc { - en: """Retain as Published option, enum: 0, 1""" - zh: """Retain as Published 选项，枚举：0，1""" - } - } - - rh { - desc { - en: """Retain Handling option, enum: 0, 1, 2""" - zh: """Retain Handling 选项，枚举：0，1，2""" - } - } - - sub_props { - desc { - en: """Subscription properties""" - zh: """订阅属性""" - } - } - - subid { - desc { - en: """Only stomp protocol, a unique identity for the subscription. range: 1-65535.""" - zh: """订阅ID，仅用于 Stomp 网关。用于创建订阅关系时指定订阅 ID。取值范围 1-65535。""" - } - } - - node { - desc { - en: """Name of the node to which the client is connected""" - zh: """客户端连接到的节点名称""" - } - } - - username { - desc { - en: """Username of client when connecting""" - zh: """客户端连接的用户名""" - } - } - - mountpoint { - desc { - en: """Topic mountpoint""" - zh: """主题固定前缀""" - } - } - - proto_name { - desc { - en: """Client protocol name""" - zh: """客户端使用的协议名称""" - } - } - - proto_ver { - desc { - en: """Protocol version used by the client""" - zh: """客户端使用的协议版本""" - } - } - - ip_address { - desc { - en: """Client's IP address""" - zh: """客户端 IP 地址""" - } - } - - port { - desc { - en: """Client's port""" - zh: """客户端端口""" - } - } - - is_bridge { - desc { - en: """Indicates whether the client is connected via bridge""" - zh: """标识客户端是否通过 is_bridge 标志连接""" - } - } - - connected_at { - desc { - en: """Client connection time""" - zh: """客户端连接时间""" - } - } - - disconnected_at { - desc { - en: """Client offline time, This field is only valid and returned when connected is false""" - zh: """客户端连接断开时间""" - } - } - - connected { - desc { - en: """Whether the client is connected""" - zh: """标识客户端是否已连接到网关""" - } - } - - keepalive { - desc { - en: """Keepalive time, with the unit of second""" - zh: """Keepalive 时间，单位：秒""" - } - } - - clean_start { - desc { - en: """Indicate whether the client is using a brand new session""" - zh: """标识客户端是否以 clean_start 的标志连接到网关""" - } - } - - expiry_interval { - desc { - en: """Session expiration interval, with the unit of second""" - zh: """会话超期时间，单位：秒""" - } - } - - created_at { - desc { - en: """Session creation time""" - zh: """会话创建时间""" - } - } - - subscriptions_cnt { - desc { - en: """Number of subscriptions established by this client""" - zh: """客户端已订阅主题数""" - } - } - - subscriptions_max { - desc { - en: """Maximum number of subscriptions allowed by this client""" - zh: """客户端允许订阅的最大主题数""" - } - } - - inflight_cnt { - desc { - en: """Current length of inflight""" - zh: """客户端当前飞行窗口大小""" - } - } - - inflight_max { - desc { - en: """Maximum length of inflight""" - zh: """客户端允许的飞行窗口最大值""" - } - } - - mqueue_len { - desc { - en: """Current length of message queue""" - zh: """客户端当前消息队列长度""" - } - } - - mqueue_max { - desc { - en: """Maximum length of message queue""" - zh: """客户端允许的最大消息队列长度""" - } - } - - mqueue_dropped { - desc { - en: """Number of messages dropped by the message queue due to exceeding the length""" - zh: """由于消息队列过程，客户端消息队列丢弃消息条数""" - } - } - - awaiting_rel_cnt { - desc { - en: """Number of awaiting acknowledge packet""" - zh: """客户端当前等待 PUBREL 确认的 PUBREC 消息的条数""" - } - } - - awaiting_rel_max { - desc { - en: """Maximum allowed number of awaiting PUBREC packet""" - zh: """客户端允许的最大 PUBREC 等待队列长度""" - } - } - - recv_oct { - desc { - en: """Number of bytes received""" - zh: """已接收的字节数""" - } - } - - recv_cnt { - desc { - en: """Number of socket packets received""" - zh: """已接收 Socket 报文次数""" - } - } - - recv_pkt { - desc { - en: """Number of protocol packets received""" - zh: """已接收应用层协议控制报文数""" - } - } - - recv_msg { - desc { - en: """Number of message packets received""" - zh: """已接收上行的消息条数""" - } - } - - send_oct { - desc { - en: """Number of bytes sent""" - zh: """已发送字节数""" - } - } - - send_cnt { - desc { - en: """Number of socket packets sent""" - zh: """已发送 Socket 报文次数""" - } - } - - send_pkt { - desc { - en: """Number of protocol packets sent""" - zh: """已发送应用层协议控制报文数""" - } - } - - send_msg { - desc { - en: """Number of message packets sent""" - zh: """已发送下行消息数条数""" - } - } - - mailbox_len { - desc { - en: """Process mailbox size""" - zh: """进程邮箱大小""" - } - } - - heap_size { - desc { - en: """Process heap size with the unit of byte""" - zh: """进程堆内存大小，单位：字节""" - } - } - - reductions { - desc { - en: """Erlang reduction""" - zh: """进程已消耗 Reduction 数""" - } - } +disconnected_at.desc: +"""Client offline time, This field is only valid and returned when connected is false""" + +heap_size.desc: +"""Process heap size with the unit of byte""" + +send_oct.desc: +"""Number of bytes sent""" + +get_client.desc: +"""Get the gateway client information""" + +param_gte_created_at.desc: +"""Match the session created datetime greater than a certain value""" + +param_conn_state.desc: +"""Match the client's connection state""" + +send_pkt.desc: +"""Number of protocol packets sent""" + +clean_start.desc: +"""Indicate whether the client is using a brand new session""" + +inflight_cnt.desc: +"""Current length of inflight""" + +delete_subscription.desc: +"""Delete a subscriptions membership""" + +param_lte_connected_at.desc: +"""Match the client socket connected datatime less than a certain value""" + +node.desc: +"""Name of the node to which the client is connected""" + +awaiting_rel_cnt.desc: +"""Number of awaiting acknowledge packet""" + +rap.desc: +"""Retain as Published option, enum: 0, 1""" + +inflight_max.desc: +"""Maximum length of inflight""" + +param_username.desc: +"""Match the client's Username""" + +param_like_endpoint_name.desc: +"""Use sub-string to match lwm2m client's endpoint name""" + +created_at.desc: +"""Session creation time""" + +sub_props.desc: +"""Subscription properties""" + +list_clients.desc: +"""Get the gateway client list""" + +subscriptions_cnt.desc: +"""Number of subscriptions established by this client""" + +mailbox_len.desc: +"""Process mailbox size""" + +send_cnt.desc: +"""Number of socket packets sent""" + +rh.desc: +"""Retain Handling option, enum: 0, 1, 2""" + +connected.desc: +"""Whether the client is connected""" + +qos.desc: +"""QoS level, enum: 0, 1, 2""" + +ip_address.desc: +"""Client's IP address""" + +param_gte_connected_at.desc: +"""Match the client socket connected datetime greater than a certain value""" + +awaiting_rel_max.desc: +"""Maximum allowed number of awaiting PUBREC packet""" + +param_like_username.desc: +"""Use sub-string to match client's username""" + +nl.desc: +"""No Local option, enum: 0, 1""" + +param_like_clientid.desc: +"""Use sub-string to match client's ID""" + +param_lte_created_at.desc: +"""Match the session created datetime less than a certain value""" + +topic.desc: +"""Topic Filter/Name""" + +proto_ver.desc: +"""Protocol version used by the client""" + +mountpoint.desc: +"""Topic mountpoint""" + +proto_name.desc: +"""Client protocol name""" + +param_lte_lifetime.desc: +"""Match the lwm2m client registered lifetime less than a certain value""" + +port.desc: +"""Client's port""" + +connected_at.desc: +"""Client connection time""" + +expiry_interval.desc: +"""Session expiration interval, with the unit of second""" + +username.desc: +"""Username of client when connecting""" + +param_clean_start.desc: +"""Match the client's clean start flag""" + +recv_msg.desc: +"""Number of message packets received""" + +list_subscriptions.desc: +"""Get the gateway client subscriptions""" + +recv_oct.desc: +"""Number of bytes received""" + +keepalive.desc: +"""Keepalive time, with the unit of second""" + +param_clientid.desc: +"""Match the client's ID""" + +subscriptions_max.desc: +"""Maximum number of subscriptions allowed by this client""" + +param_ip_address.desc: +"""Match the client's ip address""" + +mqueue_max.desc: +"""Maximum length of message queue""" + +mqueue_dropped.desc: +"""Number of messages dropped by the message queue due to exceeding the length""" + +subid.desc: +"""Only stomp protocol, a unique identity for the subscription. range: 1-65535.""" + +clientid.desc: +"""Client ID""" + +kick_client.desc: +"""Kick out the gateway client""" + +is_bridge.desc: +"""Indicates whether the client is connected via bridge""" + +lifetime.desc: +"""LwM2M Life time""" + +send_msg.desc: +"""Number of message packets sent""" + +add_subscription.desc: +"""Create a subscription membership""" + +param_endpoint_name.desc: +"""Match the lwm2m client's endpoint name""" + +param_node.desc: +"""Match the client's node name""" + +recv_cnt.desc: +"""Number of socket packets received""" + +recv_pkt.desc: +"""Number of protocol packets received""" + +endpoint_name.desc: +"""The LwM2M client endpoint name""" + +param_proto_ver.desc: +"""Match the client's protocol version""" + +reductions.desc: +"""Erlang reduction""" + +param_gte_lifetime.desc: +"""Match the lwm2m client registered lifetime greater than a certain value""" + +mqueue_len.desc: +"""Current length of message queue""" + } diff --git a/rel/i18n/emqx_gateway_api_listeners.hocon b/rel/i18n/emqx_gateway_api_listeners.hocon index 9319bb3e5..3b5bb65a1 100644 --- a/rel/i18n/emqx_gateway_api_listeners.hocon +++ b/rel/i18n/emqx_gateway_api_listeners.hocon @@ -1,146 +1,65 @@ emqx_gateway_api_listeners { - list_listeners { - desc { - en: """Gets a list of gateway listeners. This interface returns all the configs of the listener (including the authenticator on that listener), as well as the status of that listener running in the cluster.""" - zh: """获取网关监听器列表。该接口会返回监听器所有的配置（包括该监听器上的认证器），同时也会返回该监听器在集群中运行的状态。""" - } - } - - add_listener { - desc { - en: """Create the gateway listener.
+add_listener.desc: +"""Create the gateway listener.
Note: For listener types not supported by a gateway, this API returns `400: BAD_REQUEST`.""" - zh: """为指定网关添加监听器。
-注：对于某网关不支持的监听器类型，该接口会返回 `400: BAD_REQUEST`。""" - } - } - get_listener { - desc { - en: """Get the gateway listener configs""" - zh: """获取指定网关监听器的配置。""" - } - } - - delete_listener { - desc { - en: """Delete the gateway listener. All connected clients under the deleted listener will be disconnected.""" - zh: """删除指定监听器。被删除的监听器下所有已连接的客户端都会离线。""" - } - } - - update_listener { - desc { - en: """Update the gateway listener. The listener being updated performs a restart and all clients connected to that listener will be disconnected.""" - zh: """更新某网关监听器的配置。被更新的监听器会执行重启，所有已连接到该监听器上的客户端都会被断开。""" - } - } - - get_listener_authn { - desc { - en: """Get the listener's authenticator configs.""" - zh: """获取监听器的认证器配置。""" - } - } - - add_listener_authn { - desc { - en: """Enable authenticator for specified listener for client authentication.
+add_listener_authn.desc: +"""Enable authenticator for specified listener for client authentication.
When authenticator is enabled for a listener, all clients connecting to that listener will use that authenticator for authentication.""" - zh: """为指定监听器开启认证器以实现客户端认证的能力。
-当某一监听器开启认证后，所有连接到该监听器的客户端会使用该认证器进行认证。""" - } - } - update_listener_authn { - desc { - en: """Update authenticator configs for the listener, or disable/enable it.""" - zh: """更新指定监听器的认证器配置，或停用/启用该认证器。""" - } - } +add_user.desc: +"""Add user for the authenticator (only supports built_in_database)""" - delete_listener_authn { - desc { - en: """Remove authenticator for the listener.""" - zh: """移除指定监听器的认证器。""" - } - } +current_connections.desc: +"""Current Connections""" - list_users { - desc { - en: """Get the users for the authenticator (only supported by built_in_database)""" - zh: """获取用户列表（仅支持 built_in_database 类型的认证器）""" - } - } +delete_listener.desc: +"""Delete the gateway listener. All connected clients under the deleted listener will be disconnected.""" - add_user { - desc { - en: """Add user for the authenticator (only supports built_in_database)""" - zh: """添加用户（仅支持 built_in_database 类型的认证器）""" - } - } +delete_listener_authn.desc: +"""Remove authenticator for the listener.""" - get_user { - desc { - en: """Get user info from the gateway authenticator (only supports built_in_database)""" - zh: """获取用户信息（仅支持 built_in_database 类型的认证器）""" - } - } +delete_user.desc: +"""Delete the user for the gateway authenticator (only supports built_in_database)""" - update_user { - desc { - en: """Update the user info for the gateway authenticator (only supports built_in_database)""" - zh: """更新用户信息（仅支持 built_in_database 类型的认证器）""" - } - } +get_listener.desc: +"""Get the gateway listener configs""" - delete_user { - desc { - en: """Delete the user for the gateway authenticator (only supports built_in_database)""" - zh: """删除用户（仅支持 built_in_database 类型的认证器）""" - } - } +get_listener_authn.desc: +"""Get the listener's authenticator configs.""" - import_users { - desc { - en: """Import users into the gateway authenticator (only supports built_in_database)""" - zh: """导入用户（仅支持 built_in_database 类型的认证器）""" - } - } +get_user.desc: +"""Get user info from the gateway authenticator (only supports built_in_database)""" - listener_id { - desc { - en: """Listener ID""" - zh: """监听器 ID""" - } - } +import_users.desc: +"""Import users into the gateway authenticator (only supports built_in_database)""" - listener_status { - desc { - en: """listener status""" - zh: """监听器状态""" - } - } +list_listeners.desc: +"""Gets a list of gateway listeners. This interface returns all the configs of the listener (including the authenticator on that listener), as well as the status of that listener running in the cluster.""" - listener_node_status { - desc { - en: """listener status of each node in the cluster""" - zh: """监听器在集群中每个节点上的状态""" - } - } +list_users.desc: +"""Get the users for the authenticator (only supported by built_in_database)""" - node { - desc { - en: """Node Name""" - zh: """节点名称""" - } - } +listener_id.desc: +"""Listener ID""" + +listener_node_status.desc: +"""listener status of each node in the cluster""" + +listener_status.desc: +"""listener status""" + +node.desc: +"""Node Name""" + +update_listener.desc: +"""Update the gateway listener. The listener being updated performs a restart and all clients connected to that listener will be disconnected.""" + +update_listener_authn.desc: +"""Update authenticator configs for the listener, or disable/enable it.""" + +update_user.desc: +"""Update the user info for the gateway authenticator (only supports built_in_database)""" - current_connections { - desc { - en: """Current Connections""" - zh: """当前连接数""" - } - } } diff --git a/rel/i18n/emqx_gateway_schema.hocon b/rel/i18n/emqx_gateway_schema.hocon index fc34ef0a8..5f7d71913 100644 --- a/rel/i18n/emqx_gateway_schema.hocon +++ b/rel/i18n/emqx_gateway_schema.hocon @@ -1,255 +1,117 @@ emqx_gateway_schema { - gateway_common_enable { - desc { - en: """Whether to enable this gateway""" - zh: """是否启用该网关""" - } - } +dtls_listener_acceptors.desc: +"""Size of the acceptor pool.""" - gateway_common_enable_stats { - desc { - en: """Whether to enable client process statistic""" - zh: """是否开启客户端统计""" - } - } +dtls_listener_dtls_opts.desc: +"""DTLS socket options""" - gateway_common_idle_timeout { - desc { - en: """The idle time of the client connection process. It has two purposes: +gateway_common_authentication.desc: +"""Default authentication configs for all the gateway listeners. For per-listener overrides see authentication + in listener configs""" + +gateway_common_clientinfo_override.desc: +"""ClientInfo override.""" + +gateway_common_clientinfo_override_clientid.desc: +"""Template for overriding clientid.""" + +gateway_common_clientinfo_override_password.desc: +"""Template for overriding password.""" + +gateway_common_clientinfo_override_username.desc: +"""Template for overriding username.""" + +gateway_common_enable.desc: +"""Whether to enable this gateway""" + +gateway_common_enable_stats.desc: +"""Whether to enable client process statistic""" + +gateway_common_idle_timeout.desc: +"""The idle time of the client connection process. It has two purposes: 1. A newly created client process that does not receive any client requests after that time will be closed directly. 2. A running client process that does not receive any client requests after this time will go into hibernation to save resources.""" - zh: """客户端连接过程的空闲时间。该配置用于： - 1. 一个新创建的客户端进程如果在该时间间隔内没有收到任何客户端请求，将被直接关闭。 - 2. 一个正在运行的客户进程如果在这段时间后没有收到任何客户请求，将进入休眠状态以节省资源。""" - } - } - gateway_common_clientinfo_override { - desc { - en: """ClientInfo override.""" - zh: """ClientInfo 重写。""" - } - } +gateway_common_listener_access_rules.desc: +"""The access control rules for this listener. +See: https://github.com/emqtt/esockd#allowdeny""" - gateway_common_clientinfo_override_username { - desc { - en: """Template for overriding username.""" - zh: """username 重写模板""" - } - } - gateway_common_clientinfo_override_password { - desc { - en: """Template for overriding password.""" - zh: """password 重写模板""" - } - } - gateway_common_clientinfo_override_clientid { - desc { - en: """Template for overriding clientid.""" - zh: """clientid 重写模板""" - } - } +gateway_common_listener_bind.desc: +"""The IP address and port that the listener will bind.""" - gateway_common_authentication { - desc { - en: """Default authentication configs for all the gateway listeners. For per-listener overrides see authentication\n in listener configs""" - zh: """网关的认证器配置，对该网关下所以的监听器生效。如果每个监听器需要配置不同的认证器，需要配置监听器下的 authentication 字段。""" - } - } +gateway_common_listener_enable.desc: +"""Enable the listener.""" - tcp_udp_listeners { - desc { - en: """Settings for the listeners.""" - zh: """监听器配置。""" - } - } - - tcp_listeners { - desc { - en: """Settings for the TCP listeners.""" - zh: """配置 TCP 类型的监听器。""" - } - } - - udp_listeners { - desc { - en: """Settings for the UDP listeners.""" - zh: """配置 UDP 类型的监听器。""" - } - } - - listener_name_to_settings_map{ - desc { - en: """A map from listener names to listener settings.""" - zh: """从监听器名称到配置参数的映射。""" - } - } - - tcp_listener_acceptors { - desc { - en: """Size of the acceptor pool.""" - zh: """Acceptor 进程池大小。""" - } - } - - tcp_listener_tcp_opts{ - desc { - en: """Setting the TCP socket options.""" - zh: """TCP Socket 配置。""" - } - } - - tcp_listener_proxy_protocol { - desc { - en: """Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx. -See: https://www.haproxy.com/blog/haproxy/proxy-protocol/""" - zh: """是否开启 Proxy Protocol V1/2。当 EMQX 集群部署在 HAProxy 或 Nginx 后需要获取客户端真实 IP 时常用到该选项。参考：https://www.haproxy.com/blog/haproxy/proxy-protocol/""" - } - } - - tcp_listener_proxy_protocol_timeout { - desc { - en: """Timeout for proxy protocol. -EMQX will close the TCP connection if proxy protocol packet is not received within the timeout.""" - zh: """接收 Proxy Protocol 报文头的超时时间。如果在超时内没有收到 Proxy Protocol 包，EMQX 将关闭 TCP 连接。""" - } - } - - ssl_listener_options { - desc { - en: """SSL Socket options.""" - zh: """SSL Socket 配置。""" - } - } - - udp_listener_udp_opts { - desc { - en: """Settings for the UDP sockets.""" - zh: """UDP Socket 配置。""" - } - } - - udp_listener_active_n { - desc { - en: """Specify the {active, N} option for the socket. -See: https://erlang.org/doc/man/inet.html#setopts-2""" - zh: """为 Socket 指定 {active, N} 选项。 -参见：https://erlang.org/doc/man/inet.html#setopts-2""" - } - } - - udp_listener_recbuf { - desc { - en: """Size of the kernel-space receive buffer for the socket.""" - zh: """Socket 在内核空间接收缓冲区的大小。""" - } - } - - udp_listener_sndbuf { - desc { - en: """Size of the kernel-space send buffer for the socket.""" - zh: """Socket 在内核空间发送缓冲区的大小。""" - } - } - - udp_listener_buffer { - desc { - en: """Size of the user-space buffer for the socket.""" - zh: """Socket 在用户空间的缓冲区大小。""" - } - } - - udp_listener_reuseaddr { - desc { - en: """Allow local reuse of port numbers.""" - zh: """允许重用本地处于 TIME_WAIT 的端口号。""" - } - } - - dtls_listener_acceptors { - desc { - en: """Size of the acceptor pool.""" - zh: """Acceptor 进程池大小。""" - } - } - - dtls_listener_dtls_opts { - desc { - en: """DTLS socket options""" - zh: """DTLS Socket 配置""" - } - - } - - gateway_common_listener_enable { - desc { - en: """Enable the listener.""" - zh: """是否启用该监听器。""" - } - } - - gateway_common_listener_bind { - desc { - en: """The IP address and port that the listener will bind.""" - zh: """监听器绑定的 IP 地址或端口。""" - } - } - - gateway_common_listener_max_connections { - desc { - en: """Maximum number of concurrent connections.""" - zh: """监听器支持的最大连接数。""" - } - } - - gateway_common_listener_max_conn_rate { - desc { - en: """Maximum connections per second.""" - zh: """监听器支持的最大连接速率。""" - } - } - - gateway_common_listener_enable_authn { - desc { - en: """Set true (default) to enable client authentication on this listener. +gateway_common_listener_enable_authn.desc: +"""Set true (default) to enable client authentication on this listener. When set to false clients will be allowed to connect without authentication.""" - zh: """配置 true （默认值）启用客户端进行身份认证。 -配置 false 时，将不对客户端做任何认证。""" - } - } - gateway_mountpoint { - desc { - en: """When publishing or subscribing, prefix all topics with a mountpoint string. +gateway_common_listener_max_conn_rate.desc: +"""Maximum connections per second.""" + +gateway_common_listener_max_connections.desc: +"""Maximum number of concurrent connections.""" + +gateway_mountpoint.desc: +"""When publishing or subscribing, prefix all topics with a mountpoint string. The prefixed string will be removed from the topic name when the message is delivered to the subscriber. The mountpoint is a way that users can use to implement isolation of message routing between different listeners. For example if a client A subscribes to `t` with `listeners.tcp.\.mountpoint` set to `some_tenant`, then the client actually subscribes to the topic `some_tenant/t`. Similarly, if another client B (connected to the same listener as the client A) sends a message to topic `t`, the message is routed to all the clients subscribed `some_tenant/t`, -so client A will receive the message, with topic name `t`. Set to `\"\"` to disable the feature. +so client A will receive the message, with topic name `t`. Set to `""` to disable the feature. Variables in mountpoint string:
- ${clientid}: clientid
- ${username}: username""" - zh: """发布或订阅时，在所有主题前增加前缀字符串。 -当消息投递给订阅者时，前缀字符串将从主题名称中删除。挂载点是用户可以用来实现不同监听器之间的消息路由隔离的一种方式。 -例如，如果客户端 A 在 `listeners.tcp.\.mountpoint` 设置为 `some_tenant` 的情况下订阅 `t`， -则客户端实际上订阅了 `some_tenant/t` 主题。 -类似地，如果另一个客户端 B（连接到与客户端 A 相同的侦听器）向主题 `t` 发送消息， -则该消息被路由到所有订阅了 `some_tenant/t` 的客户端，因此客户端 A 将收到该消息，带有 主题名称`t`。 设置为 `\"\"` 以禁用该功能。 -挂载点字符串中可用的变量：
- - ${clientid}：clientid
- - ${username}：用户名""" - } - } - gateway_common_listener_access_rules { - desc { - en: """The access control rules for this listener. -See: https://github.com/emqtt/esockd#allowdeny""" - zh: """配置监听器的访问控制规则。 -见：https://github.com/emqtt/esockd#allowdeny""" - } - } +listener_name_to_settings_map.desc: +"""A map from listener names to listener settings.""" + +ssl_listener_options.desc: +"""SSL Socket options.""" + +tcp_listener_acceptors.desc: +"""Size of the acceptor pool.""" + +tcp_listener_proxy_protocol.desc: +"""Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx. +See: https://www.haproxy.com/blog/haproxy/proxy-protocol/""" + +tcp_listener_proxy_protocol_timeout.desc: +"""Timeout for proxy protocol. +EMQX will close the TCP connection if proxy protocol packet is not received within the timeout.""" + +tcp_listener_tcp_opts.desc: +"""Setting the TCP socket options.""" + +tcp_listeners.desc: +"""Settings for the TCP listeners.""" + +tcp_udp_listeners.desc: +"""Settings for the listeners.""" + +udp_listener_active_n.desc: +"""Specify the {active, N} option for the socket. +See: https://erlang.org/doc/man/inet.html#setopts-2""" + +udp_listener_buffer.desc: +"""Size of the user-space buffer for the socket.""" + +udp_listener_recbuf.desc: +"""Size of the kernel-space receive buffer for the socket.""" + +udp_listener_reuseaddr.desc: +"""Allow local reuse of port numbers.""" + +udp_listener_sndbuf.desc: +"""Size of the kernel-space send buffer for the socket.""" + +udp_listener_udp_opts.desc: +"""Settings for the UDP sockets.""" + +udp_listeners.desc: +"""Settings for the UDP listeners.""" + } diff --git a/rel/i18n/emqx_license_http_api.hocon b/rel/i18n/emqx_license_http_api.hocon index 40a18bbf3..895041c18 100644 --- a/rel/i18n/emqx_license_http_api.hocon +++ b/rel/i18n/emqx_license_http_api.hocon @@ -1,23 +1,15 @@ emqx_license_http_api { - desc_license_info_api { - desc { - en: "Get license info" - zh: "获取许可证信息" - } - label: { - en: "License info" - zh: "许可证信息" - } - } - desc_license_key_api { - desc { - en: "Update a license key" - zh: "更新一个许可证密钥" - } - label: { - en: "Update license" - zh: "更新许可证" - } - } +desc_license_info_api.desc: +"""Get license info""" + +desc_license_info_api.label: +"""License info""" + +desc_license_key_api.desc: +"""Update a license key""" + +desc_license_key_api.label: +"""Update license""" + } diff --git a/rel/i18n/emqx_license_schema.hocon b/rel/i18n/emqx_license_schema.hocon index c330f1cb2..3e4e37bff 100644 --- a/rel/i18n/emqx_license_schema.hocon +++ b/rel/i18n/emqx_license_schema.hocon @@ -1,55 +1,33 @@ emqx_license_schema { - license_root { - desc { - en: "Defines the EMQX Enterprise license. \n\n" - "\n" - "The default license has 100 connections limit, it is " - "issued on 2023-01-09 and valid for 5 years (1825 days).\n" - "\n" - "EMQX comes with a default trial license. For production use, please \n" - "visit https://www.emqx.com/apply-licenses/emqx to apply." - zh: "EMQX企业许可证。\n" - "EMQX 自带一个默认的试用许可证，" - "默认试用许可允许最多接入 100 个连接，签发时间是 2023年1月9日，有效期是 5 年（1825 天）。" - "若需要在生产环境部署，\n" - "请访问 https://www.emqx.com/apply-licenses/emqx 来申请。" - } - label { - en: "License" - zh: "许可证" - } - } - key_field { - desc { - en: "License string" - zh: "许可证字符串" - } - label { - en: "License string" - zh: "许可证字符串" - } - } +connection_high_watermark_field.desc: +"""High watermark limit above which license connection quota usage alarms are activated""" - connection_low_watermark_field { - desc { - en: "Low watermark limit below which license connection quota usage alarms are deactivated" - zh: "低水位限制，低于此水位线时系统会清除连接配额使用告警" - } - label { - en: "Connection low watermark" - zh: "连接低水位线" - } - } +connection_high_watermark_field.label: +"""Connection high watermark""" + +connection_low_watermark_field.desc: +"""Low watermark limit below which license connection quota usage alarms are deactivated""" + +connection_low_watermark_field.label: +"""Connection low watermark""" + +key_field.desc: +"""License string""" + +key_field.label: +"""License string""" + +license_root.desc: +"""Defines the EMQX Enterprise license. + + +The default license has 100 connections limit, it is issued on 2023-01-09 and valid for 5 years (1825 days). + +EMQX comes with a default trial license. For production use, please +visit https://www.emqx.com/apply-licenses/emqx to apply.""" + +license_root.label: +"""License""" - connection_high_watermark_field { - desc { - en: "High watermark limit above which license connection quota usage alarms are activated" - zh: "高水位线，连接数超过这个水位线时，系统会触发许可证连接配额使用告警" - } - label { - en: "Connection high watermark" - zh: "连接高水位" - } - } } diff --git a/rel/i18n/emqx_limiter_schema.hocon b/rel/i18n/emqx_limiter_schema.hocon index 37eb4ee1e..c99840375 100644 --- a/rel/i18n/emqx_limiter_schema.hocon +++ b/rel/i18n/emqx_limiter_schema.hocon @@ -1,168 +1,94 @@ emqx_limiter_schema { - failure_strategy { - desc { - en: """The strategy when all the retries failed.""" - zh: """当所有的重试都失败后的处理策略""" - } - label: { - en: """Failure Strategy""" - zh: """失败策略""" - } - } +bucket_cfg.desc: +"""Bucket Configs""" - max_retry_time { - desc { - en: """The maximum retry time when acquire failed.""" - zh: """申请失败后，尝试重新申请的时长最大值""" - } - label: { - en: """Max Retry Time""" - zh: """最大重试时间""" - } - } +bucket_cfg.label: +"""Buckets""" - divisible { - desc { - en: """Is it possible to split the number of requested tokens?""" - zh: """申请的令牌数是否可以被分割""" - } - label: { - en: """Divisible""" - zh: """是否可分割""" - } - } - - low_watermark { - desc { - en: """If the remaining tokens are lower than this value, -the check/consume will succeed, but it will be forced to wait for a short period of time.""" - zh: """当桶中剩余的令牌数低于这个值，即使令牌申请成功了，也会被强制暂停一会儿""" - } - label: { - en: """Low Watermark""" - zh: """低水位线""" - } - } - - initial { - desc { - en: """The initial number of tokens for this bucket.""" - zh: """桶中的初始令牌数""" - } - label: { - en: """Initial""" - zh: """初始令牌数""" - } - } - - rate { - desc { - en: """Rate for this bucket.""" - zh: """桶的令牌生成速率""" - } - label: { - en: """Rate""" - zh: """速率""" - } - } - - client { - desc { - en: """The rate limit for each user of the bucket""" - zh: """对桶的每个使用者的速率控制设置""" - } - label: { - en: """Per Client""" - zh: """每个使用者的限制""" - } - } - - bucket_cfg { - desc { - en: """Bucket Configs""" - zh: """桶的配置""" - } - label: { - en: """Buckets""" - zh: """桶的配置""" - } - } - - burst { - desc { - en: """The burst, This value is based on rate.
+burst.desc: +"""The burst, This value is based on rate.
This value + rate = the maximum limit that can be achieved when limiter burst.""" - zh: """突发速率。 -突发速率允许短时间内速率超过设置的速率值，突发速率 + 速率 = 当前桶能达到的最大速率值""" - } - label: { - en: """Burst""" - zh: """突发速率""" - } - } - message_routing { - desc { - en: """The message routing limiter. -This is used to limit the forwarding rate for this EMQX node. -Once the limit is reached, new publish will be refused""" - zh: """消息派发速率控制器。 -这个用来控制当前节点内的消息派发速率，当达到最大速率后，新的推送将会被拒绝""" - } - label: { - en: """Message Routing""" - zh: """消息派发""" - } - } +burst.label: +"""Burst""" - connection { - desc { - en: """The connection limiter. -This is used to limit the connection rate for this EMQX node. -Once the limit is reached, new connections will be refused""" - zh: """连接速率控制器。 -这个用来控制当前节点上的连接速率，当达到最大速率后，新的连接将会被拒绝""" - } - label: { - en: """Connection""" - zh: """连接速率""" - } - } - - messages { - desc { - en: """The `messages` limiter. -This is used to limit the inbound message numbers for this EMQX node -Once the limit is reached, the restricted client will be slow down even be hung for a while.""" - zh: """流入速率控制器。 -这个用来控制当前节点上的消息流入速率，当达到最大速率后，会话将会被限速甚至被强制挂起一小段时间""" - } - label: { - en: """Messages""" - zh: """消息流入速率""" - } - } - - bytes { - desc { - en: """The `bytes` limiter. +bytes.desc: +"""The `bytes` limiter. This is used to limit the inbound bytes rate for this EMQX node. Once the limit is reached, the restricted client will be slow down even be hung for a while.""" - zh: """流入字节率控制器。 -这个是用来控制当前节点上的数据流入的字节率，每条消息将会消耗和其二进制大小等量的令牌，当达到最大速率后，会话将会被限速甚至被强制挂起一小段时间""" - } - label: { - en: """Bytes""" - zh: """流入字节率""" - } - } - internal { - desc { - en: """Limiter for EMQX internal app.""" - zh: """EMQX 内部功能所用限制器。""" +bytes.label: +"""Bytes""" + +client.desc: +"""The rate limit for each user of the bucket""" + +client.label: +"""Per Client""" + +connection.desc: +"""The connection limiter. +This is used to limit the connection rate for this EMQX node. +Once the limit is reached, new connections will be refused""" + +connection.label: +"""Connection""" + +divisible.desc: +"""Is it possible to split the number of requested tokens?""" + +divisible.label: +"""Divisible""" + +failure_strategy.desc: +"""The strategy when all the retries failed.""" + +failure_strategy.label: +"""Failure Strategy""" + +initial.desc: +"""The initial number of tokens for this bucket.""" + +initial.label: +"""Initial""" + +internal.desc: +"""Limiter for EMQX internal app.""" + +low_watermark.desc: +"""If the remaining tokens are lower than this value, +the check/consume will succeed, but it will be forced to wait for a short period of time.""" + +low_watermark.label: +"""Low Watermark""" + +max_retry_time.desc: +"""The maximum retry time when acquire failed.""" + +max_retry_time.label: +"""Max Retry Time""" + +message_routing.desc: +"""The message routing limiter. +This is used to limit the forwarding rate for this EMQX node. +Once the limit is reached, new publish will be refused""" + +message_routing.label: +"""Message Routing""" + +messages.desc: +"""The `messages` limiter. +This is used to limit the inbound message numbers for this EMQX node +Once the limit is reached, the restricted client will be slow down even be hung for a while.""" + +messages.label: +"""Messages""" + +rate.desc: +"""Rate for this bucket.""" + +rate.label: +"""Rate""" - } - } } diff --git a/rel/i18n/emqx_lwm2m_api.hocon b/rel/i18n/emqx_lwm2m_api.hocon index 9cd7e27c0..7ff1fdcce 100644 --- a/rel/i18n/emqx_lwm2m_api.hocon +++ b/rel/i18n/emqx_lwm2m_api.hocon @@ -1,58 +1,27 @@ emqx_lwm2m_api { - lookup_resource { - desc { - en: """Look up a resource""" - zh: """查看指定资源状态""" - } - } +dataType.desc: +"""Data Type""" - observe_resource { - desc { - en: """Observe or Cancel observe a resource""" - zh: """Observe/Un-Observe 指定资源""" - } - } +lookup_resource.desc: +"""Look up a resource""" - read_resource { - desc { - en: """Send a read command to a resource""" - zh: """发送读指令到某资源""" - } - } +name.desc: +"""Resource Name""" - write_resource { - desc { - en: """Send a write command to a resource""" - zh: """发送写指令到某资源""" - } - } +observe_resource.desc: +"""Observe or Cancel observe a resource""" - operations { - desc { - en: """Resource Operations""" - zh: """资源可用操作列表""" - } - } +operations.desc: +"""Resource Operations""" - dataType { - desc { - en: """Data Type""" - zh: """数据类型""" - } - } +path.desc: +"""Resource Path""" - path { - desc { - en: """Resource Path""" - zh: """资源路径""" - } - } +read_resource.desc: +"""Send a read command to a resource""" + +write_resource.desc: +"""Send a write command to a resource""" - name { - desc { - en: """Resource Name""" - zh: """资源名称""" - } - } } diff --git a/rel/i18n/emqx_lwm2m_schema.hocon b/rel/i18n/emqx_lwm2m_schema.hocon index 822570f1d..0193ce88f 100644 --- a/rel/i18n/emqx_lwm2m_schema.hocon +++ b/rel/i18n/emqx_lwm2m_schema.hocon @@ -1,127 +1,56 @@ emqx_lwm2m_schema { - lwm2m { - desc { - en: """The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" - zh: """LwM2M 网关配置。仅支持 v1.0.1 协议。""" - } - } +lwm2m.desc: +"""The LwM2M Gateway configuration. This gateway only supports the v1.0.1 protocol.""" - lwm2m_xml_dir { - desc { - en: """The Directory for LwM2M Resource definition.""" - zh: """LwM2M Resource 定义的 XML 文件目录路径。""" - } - } +lwm2m_auto_observe.desc: +"""Automatically observe the object list of REGISTER packet.""" - lwm2m_lifetime_min { - desc { - en: """Minimum value of lifetime allowed to be set by the LwM2M client.""" - zh: """允许 LwM2M 客户端允许设置的心跳最小值。""" - } - } +lwm2m_lifetime_max.desc: +"""Maximum value of lifetime allowed to be set by the LwM2M client.""" - lwm2m_lifetime_max { - desc { - en: """Maximum value of lifetime allowed to be set by the LwM2M client.""" - zh: """允许 LwM2M 客户端允许设置的心跳最大值。""" - } - } +lwm2m_lifetime_min.desc: +"""Minimum value of lifetime allowed to be set by the LwM2M client.""" - lwm2m_qmode_time_window { - desc { - en: """The value of the time window during which the network link is considered valid by the LwM2M Gateway in QMode mode. +lwm2m_qmode_time_window.desc: +"""The value of the time window during which the network link is considered valid by the LwM2M Gateway in QMode mode. For example, after receiving an update message from a client, any messages within this time window are sent directly to the LwM2M client, and all messages beyond this time window are temporarily stored in memory.""" - zh: """在QMode模式下，LwM2M网关认为网络链接有效的时间窗口的值。 -例如，在收到客户端的更新信息后，在这个时间窗口内的任何信息都会直接发送到LwM2M客户端，而超过这个时间窗口的所有信息都会暂时储存在内存中。""" - } - } +lwm2m_translators.desc: +"""Topic configuration for LwM2M's gateway publishing and subscription.""" - lwm2m_auto_observe { - desc { - en: """Automatically observe the object list of REGISTER packet.""" - zh: """自动 Observe REGISTER 数据包的 Object 列表。""" - } - } - - lwm2m_update_msg_publish_condition { - desc { - en: """Policy for publishing UPDATE event message.
- - always: send update events as long as the UPDATE request is received.
- - contains_object_list: send update events only if the UPDATE request carries any Object List""" - zh: """发布UPDATE事件消息的策略。
- - always: 只要收到 UPDATE 请求，就发送更新事件。
- - contains_object_list: 仅当 UPDATE 请求携带 Object 列表时才发送更新事件。""" - } - } - - lwm2m_translators { - desc { - en: """Topic configuration for LwM2M's gateway publishing and subscription.""" - zh: """LwM2M 网关订阅/发布消息的主题映射配置。""" - } - } - - lwm2m_translators_command { - desc { - en: """The topic for receiving downstream commands. +lwm2m_translators_command.desc: +"""The topic for receiving downstream commands. For each new LwM2M client that succeeds in going online, the gateway creates a subscription relationship to receive downstream commands and send it to the LwM2M client""" - zh: """下行命令主题。 -对于每个成功上线的新 LwM2M 客户端，网关会创建一个订阅关系来接收下行消息并将其发送给客户端。""" - } - } - - lwm2m_translators_response { - desc { - en: """The topic for gateway to publish the acknowledge events from LwM2M client""" - zh: """用于网关发布来自 LwM2M 客户端的确认事件的主题。""" - } - } - - lwm2m_translators_notify { - desc { - en: """The topic for gateway to publish the notify events from LwM2M client. +lwm2m_translators_notify.desc: +"""The topic for gateway to publish the notify events from LwM2M client. After succeed observe a resource of LwM2M client, Gateway will send the notify events via this topic, if the client reports any resource changes""" - zh: """用于发布来自 LwM2M 客户端的通知事件的主题。 -在成功 Observe 到 LwM2M 客户端的资源后，如果客户端报告任何资源状态的变化，网关将通过该主题发送通知事件。""" - } - } +lwm2m_translators_register.desc: +"""The topic for gateway to publish the register events from LwM2M client.""" - lwm2m_translators_register { - desc { - en: """The topic for gateway to publish the register events from LwM2M client.""" - zh: """用于发布来自 LwM2M 客户端的注册事件的主题。""" - } - } +lwm2m_translators_response.desc: +"""The topic for gateway to publish the acknowledge events from LwM2M client""" - lwm2m_translators_update { - desc { - en: """The topic for gateway to publish the update events from LwM2M client""" - zh: """用于发布来自LwM2M客户端的更新事件的主题。""" - } - } +lwm2m_translators_update.desc: +"""The topic for gateway to publish the update events from LwM2M client""" - translator { - desc { - en: """MQTT topic that corresponds to a particular type of event.""" - zh: """配置某网关客户端对于发布消息或订阅的主题和 QoS 等级。""" - } - } +lwm2m_update_msg_publish_condition.desc: +"""Policy for publishing UPDATE event message.
+ - always: send update events as long as the UPDATE request is received.
+ - contains_object_list: send update events only if the UPDATE request carries any Object List""" - translator_topic { - desc { - en: """Topic Name""" - zh: """主题名称""" - } - } +lwm2m_xml_dir.desc: +"""The Directory for LwM2M Resource definition.""" + +translator.desc: +"""MQTT topic that corresponds to a particular type of event.""" + +translator_qos.desc: +"""QoS Level""" + +translator_topic.desc: +"""Topic Name""" - translator_qos { - desc { - en: """QoS Level""" - zh: """QoS 等级""" - } - } } diff --git a/rel/i18n/emqx_mgmt_api_alarms.hocon b/rel/i18n/emqx_mgmt_api_alarms.hocon index ca8bf0769..0327fffcd 100644 --- a/rel/i18n/emqx_mgmt_api_alarms.hocon +++ b/rel/i18n/emqx_mgmt_api_alarms.hocon @@ -1,84 +1,39 @@ emqx_mgmt_api_alarms { - list_alarms_api { - desc { - en: """List currently activated alarms or historical alarms, determined by query parameters.""" - zh: """列出当前激活的告警或历史告警，由查询参数决定。""" - } - } +activate_at.desc: +"""Alarm start time, using rfc3339 standard time format.""" - delete_alarms_api { - desc { - en: """Remove all historical alarms.""" - zh: """删除所有历史告警。""" - } - } +deactivate_at.desc: +"""Alarm end time, using rfc3339 standard time format.""" - delete_alarms_api_response204 { - desc { - en: """Historical alarms have been cleared successfully.""" - zh: """历史告警已成功清除。""" - } - } +delete_alarms_api.desc: +"""Remove all historical alarms.""" - get_alarms_qs_activated { - desc { - en: """It is used to specify the alarm type of the query. +delete_alarms_api_response204.desc: +"""Historical alarms have been cleared successfully.""" + +details.desc: +"""Alarm details, provides more alarm information, mainly for program processing.""" + +duration.desc: +"""Indicates how long the alarm has been active in milliseconds.""" + +get_alarms_qs_activated.desc: +"""It is used to specify the alarm type of the query. When true, it returns the currently activated alarm, and when it is false, it returns the historical alarm. The default is false.""" - zh: """用于指定查询的告警类型， -为 true 时返回当前激活的告警，为 false 时返回历史告警，默认为 false。""" - } - } - node { - desc { - en: """The name of the node that triggered this alarm.""" - zh: """触发此告警的节点名称。""" - } - } +list_alarms_api.desc: +"""List currently activated alarms or historical alarms, determined by query parameters.""" - name { - desc { - en: """Alarm name, used to distinguish different alarms.""" - zh: """告警名称，用于区分不同的告警。""" - } - } +message.desc: +"""Alarm message, which describes the alarm content in a human-readable format.""" - message { - desc { - en: """Alarm message, which describes the alarm content in a human-readable format.""" - zh: """告警消息，以人类可读的方式描述告警内容。""" - } - } +name.desc: +"""Alarm name, used to distinguish different alarms.""" - details { - desc { - en: """Alarm details, provides more alarm information, mainly for program processing.""" - zh: """告警详情，提供了更多的告警信息，主要提供给程序处理。""" - } - } - - duration { - desc { - en: """Indicates how long the alarm has been active in milliseconds.""" - zh: """表明告警已经持续了多久，单位：毫秒。""" - } - } - - activate_at { - desc { - en: """Alarm start time, using rfc3339 standard time format.""" - zh: """告警开始时间，使用 rfc3339 标准时间格式。""" - } - } - - deactivate_at { - desc { - en: """Alarm end time, using rfc3339 standard time format.""" - zh: """告警结束时间，使用 rfc3339 标准时间格式。""" - } - } +node.desc: +"""The name of the node that triggered this alarm.""" } diff --git a/rel/i18n/emqx_mgmt_api_banned.hocon b/rel/i18n/emqx_mgmt_api_banned.hocon index b45a40ba6..1a9700641 100644 --- a/rel/i18n/emqx_mgmt_api_banned.hocon +++ b/rel/i18n/emqx_mgmt_api_banned.hocon @@ -1,98 +1,54 @@ emqx_mgmt_api_banned { - list_banned_api { - desc { - en: """List all currently banned client IDs, usernames and IP addresses.""" - zh: """列出目前所有被封禁的客户端 ID、用户名和 IP 地址。""" - } - } +as.desc: +"""Ban method, which can be client ID, username or IP address.""" - create_banned_api { - desc { - en: """Add a client ID, username or IP address to the blacklist.""" - zh: """添加一个客户端 ID、用户名或者 IP 地址到黑名单。""" - } - } +as.label: +"""Ban Method""" - create_banned_api_response400 { - desc { - en: """Bad request, possibly due to wrong parameters or the existence of a banned object.""" - zh: """错误的请求，可能是参数错误或封禁对象已存在等原因。""" - } - } +at.desc: +"""The start time of the ban, the format is rfc3339, the default is the time when the operation was initiated.""" - delete_banned_api { - desc { - en: """Remove a client ID, username or IP address from the blacklist.""" - zh: """将一个客户端 ID、用户名或者 IP 地址从黑名单中删除。""" - } - } +at.label: +"""Ban Time""" - delete_banned_api_response404 { - desc { - en: """The banned object was not found in the blacklist.""" - zh: """未在黑名单中找到该封禁对象。""" - } - } +by.desc: +"""Initiator of the ban.""" + +by.label: +"""Ban Initiator""" + +create_banned_api.desc: +"""Add a client ID, username or IP address to the blacklist.""" + +create_banned_api_response400.desc: +"""Bad request, possibly due to wrong parameters or the existence of a banned object.""" + +delete_banned_api.desc: +"""Remove a client ID, username or IP address from the blacklist.""" + +delete_banned_api_response404.desc: +"""The banned object was not found in the blacklist.""" + +list_banned_api.desc: +"""List all currently banned client IDs, usernames and IP addresses.""" + +reason.desc: +"""Ban reason, record the reason why the current object was banned.""" + +reason.label: +"""Ban Reason""" + +until.desc: +"""The end time of the ban, the format is rfc3339, the default is the time when the operation was initiated + 1 year.""" + +until.label: +"""Ban End Time""" + +who.desc: +"""Ban object, specific client ID, username or IP address.""" + +who.label: +"""Ban Object""" - as { - desc { - en: """Ban method, which can be client ID, username or IP address.""" - zh: """封禁方式，可以通过客户端 ID、用户名或者 IP 地址等方式进行封禁。""" - } - label { - en: """Ban Method""" - zh: """封禁方式""" - } - } - who { - desc { - en: """Ban object, specific client ID, username or IP address.""" - zh: """封禁对象，具体的客户端 ID、用户名或者 IP 地址。""" - } - label { - en: """Ban Object""" - zh: """封禁对象""" - } - } - by { - desc { - en: """Initiator of the ban.""" - zh: """封禁的发起者。""" - } - label { - en: """Ban Initiator""" - zh: """封禁发起者""" - } - } - reason { - desc { - en: """Ban reason, record the reason why the current object was banned.""" - zh: """封禁原因，记录当前对象被封禁的原因。""" - } - label { - en: """Ban Reason""" - zh: """封禁原因""" - } - } - at { - desc { - en: """The start time of the ban, the format is rfc3339, the default is the time when the operation was initiated.""" - zh: """封禁的起始时间，格式为 rfc3339，默认为发起操作的时间。""" - } - label { - en: """Ban Time""" - zh: """封禁时间""" - } - } - until { - desc { - en: """The end time of the ban, the format is rfc3339, the default is the time when the operation was initiated + 1 year.""" - zh: """封禁的结束时间，格式为 rfc3339，默认值为发起操作的时间 + 1 年。""" - } - label { - en: """Ban End Time""" - zh: """封禁结束时间""" - } - } } diff --git a/rel/i18n/emqx_mgmt_api_key_schema.hocon b/rel/i18n/emqx_mgmt_api_key_schema.hocon index f96c5c11b..0dc11c7ac 100644 --- a/rel/i18n/emqx_mgmt_api_key_schema.hocon +++ b/rel/i18n/emqx_mgmt_api_key_schema.hocon @@ -1,32 +1,20 @@ emqx_mgmt_api_key_schema { - api_key { - desc { - en: """API Key, can be used to request API other than the management API key and the Dashboard user management API""" - zh: """API 密钥， 可用于请求除管理 API 密钥及 Dashboard 用户管理 API 的其它接口""" - } - label { - en: "API Key" - zh: "API 密钥" - } - } - bootstrap_file { - desc { - en: """Bootstrap file is used to add an api_key when emqx is launched, + +api_key.desc: +"""API Key, can be used to request API other than the management API key and the Dashboard user management API""" + +api_key.label: +"""API Key""" + +bootstrap_file.desc: +"""Bootstrap file is used to add an api_key when emqx is launched, the format is: ``` 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL ```""" - zh: """用于在启动 emqx 时，添加 API 密钥，其格式为： - ``` - 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK - ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL - ```""" - } - label { - en: "Initialize api_key file." - zh: "API 密钥初始化文件" - } - } +bootstrap_file.label: +"""Initialize api_key file.""" + } diff --git a/rel/i18n/emqx_mgmt_api_publish.hocon b/rel/i18n/emqx_mgmt_api_publish.hocon index a09732cfc..50589f8d7 100644 --- a/rel/i18n/emqx_mgmt_api_publish.hocon +++ b/rel/i18n/emqx_mgmt_api_publish.hocon @@ -1,27 +1,51 @@ emqx_mgmt_api_publish { - publish_api { - desc { - en: """Possible HTTP status response codes are:
+ +error_message.desc: +"""Describes the failure reason in detail.""" + +message_id.desc: +"""A globally unique message ID for correlation/tracing.""" + +message_properties.desc: +"""The Properties of the PUBLISH message.""" + +msg_content_type.desc: +"""The Content Type MUST be a UTF-8 Encoded String.""" + +msg_correlation_data.desc: +"""Identifier of the Correlation Data. The Server MUST send the Correlation Data unaltered to all subscribers receiving the Application Message.""" + +msg_message_expiry_interval.desc: +"""Identifier of the Message Expiry Interval. If the Message Expiry Interval has passed and the Server has not managed to start onward delivery to a matching subscriber, then it MUST delete the copy of the message for that subscriber.""" + +msg_payload_format_indicator.desc: +"""0 (0x00) Byte Indicates that the Payload is unspecified bytes, which is equivalent to not sending a Payload Format Indicator. +1 (0x01) Byte Indicates that the Payload is UTF-8 Encoded Character Data. The UTF-8 data in the Payload MUST be well-formed UTF-8 as defined by the Unicode specification and restated in RFC 3629.""" + +msg_response_topic.desc: +"""Identifier of the Response Topic.The Response Topic MUST be a UTF-8 Encoded, It MUST NOT contain wildcard characters.""" + +msg_user_properties.desc: +"""The User-Property key-value pairs. Note: in case there are duplicated keys, only the last one will be used.""" + +payload.desc: +"""The MQTT message payload.""" + +payload_encoding.desc: +"""MQTT Payload Encoding, base64 or plain. When set to base64, the message is decoded before it is published.""" + +publish_api.desc: +"""Possible HTTP status response codes are:
200: The message is delivered to at least one subscriber;
202: No matched subscribers;
400: Message is invalid. for example bad topic name, or QoS is out of range;
503: Failed to deliver the message to subscriber(s)""" - zh: """发布一个消息。
-可能的 HTTP 状态码如下：
-200: 消息被成功发送到至少一个订阅。
-202: 没有匹配到任何订阅。
-400: 消息编码错误，如非法主题，或 QoS 超出范围等。
-503: 服务重启等过程中导致转发失败。""" - } - label { - en: "Publish a message" - zh: "发布一条信息" - } - } - publish_bulk_api { - desc { - en: """Possible HTTP response status code are:
+publish_api.label: +"""Publish a message""" + +publish_bulk_api.desc: +"""Possible HTTP response status code are:
200: All messages are delivered to at least one subscriber;
202: At least one message was not delivered to any subscriber;
400: At least one message is invalid. For example bad topic name, or QoS is out of range;
@@ -31,62 +55,15 @@ In case there is at lest one invalid message in the batch, the HTTP response bod is the same as for /publish API.
Otherwise the HTTP response body is an array of JSON objects indicating the publish result of each individual message in the batch.""" - zh: """批量发布一组消息。
-可能的 HTTP 状态码如下：
-200: 所有的消息都被成功发送到至少一个订阅。
-202: 至少有一个消息没有匹配到任何订阅。
-400: 至少有一个消息编码错误，如非法主题，或 QoS 超出范围等。
-503: 至少有一个小因为服务重启的原因导致转发失败。
-请求的 Body 或者 Body 中包含的某个消息无法通过 API 规范的类型检查时，HTTP 响应的消息与发布单个消息的 API - /publish 是一样的。 -如果所有的消息都是合法的，那么 HTTP 返回的内容是一个 JSON 数组，每个元素代表了该消息转发的状态。""" - } - label { - en: "Publish a batch of messages" - zh: "发布一批信息" - } - } +publish_bulk_api.label: +"""Publish a batch of messages""" - topic_name { - desc { - en: "Topic Name" - zh: "主题名称" - } - } - qos { - desc { - en: "MQTT message QoS" - zh: "MQTT 消息的 QoS" - } - } - payload { - desc { - en: "The MQTT message payload." - zh: "MQTT 消息体。" - } - } - retain { - desc { - en: "A boolean field to indicate if this message should be retained." - zh: "布尔型字段，用于表示该消息是否保留消息。" - } - } - payload_encoding { - desc { - en: "MQTT Payload Encoding, base64 or plain. When set to base64, the message is decoded before it is published." - zh: "MQTT 消息体的编码方式，可以是 base64 或 plain。当设置为 base64 时，消息在发布前会先被解码。" - } - } - message_id { - desc { - en: "A globally unique message ID for correlation/tracing." - zh: "全局唯一的一个消息 ID，方便用于关联和追踪。" - } - } - reason_code { - desc { - en: """The MQTT reason code, as the same ones used in PUBACK packet.
+qos.desc: +"""MQTT message QoS""" + +reason_code.desc: +"""The MQTT reason code, as the same ones used in PUBACK packet.
Currently supported codes are:
16(0x10): No matching subscribers;
@@ -95,63 +72,11 @@ Currently supported codes are:
151(0x97): Publish rate limited, or message size exceeded limit. The global size limit can be configured with mqtt.max_packet_size
NOTE: The message size is estimated with the received topic and payload size, meaning the actual size of serialized bytes (when sent to MQTT subscriber) might be slightly over the limit.""" - zh: """MQTT 消息发布的错误码，这些错误码也是 MQTT 规范中 PUBACK 消息可能携带的错误码。
-当前支持如下错误码：
-16(0x10)：没能匹配到任何订阅；
-131(0x81)：消息转发时发生错误，例如 EMQX 服务重启；
-144(0x90)：主题名称非法；
-151(0x97)：受到了速率限制，或者消息尺寸过大。全局消息大小限制可以通过配置项 mqtt.max_packet_size 来进行修改。
-注意：消息尺寸的是通过主题和消息体的字节数进行估算的。具体发布时所占用的字节数可能会稍大于这个估算的值。""" - } - } - error_message { - desc { - en: "Describes the failure reason in detail." - zh: "失败的详细原因。" - } - } - message_properties { - desc { - en: "The Properties of the PUBLISH message." - zh: "PUBLISH 消息里的 Property 字段。" - } - } - msg_payload_format_indicator { - desc { - en: """0 (0x00) Byte Indicates that the Payload is unspecified bytes, which is equivalent to not sending a Payload Format Indicator. -1 (0x01) Byte Indicates that the Payload is UTF-8 Encoded Character Data. The UTF-8 data in the Payload MUST be well-formed UTF-8 as defined by the Unicode specification and restated in RFC 3629.""" - zh: "载荷格式指示标识符，0 表示载荷是未指定格式的数据，相当于没有发送载荷格式指示；1 表示载荷是 UTF-8 编码的字符数据，载荷中的 UTF-8 数据必须是按照 Unicode 的规范和 RFC 3629 的标准要求进行编码的。" - } - } - msg_message_expiry_interval { - desc { - en: "Identifier of the Message Expiry Interval. If the Message Expiry Interval has passed and the Server has not managed to start onward delivery to a matching subscriber, then it MUST delete the copy of the message for that subscriber." - zh: "消息过期间隔标识符，以秒为单位。当消失已经过期时，如果服务端还没有开始向匹配的订阅者投递该消息，则服务端会删除该订阅者的消息副本。如果不设置，则消息永远不会过期" - } - } - msg_response_topic { - desc { - en: "Identifier of the Response Topic.The Response Topic MUST be a UTF-8 Encoded, It MUST NOT contain wildcard characters." - zh: "响应主题标识符， UTF-8 编码的字符串，用作响应消息的主题名。响应主题不能包含通配符，也不能包含多个主题，否则将造成协议错误。当存在响应主题时，消息将被视作请求报文。服务端在收到应用消息时必须将响应主题原封不动的发送给所有的订阅者。" - } - } - msg_correlation_data { - desc { - en: "Identifier of the Correlation Data. The Server MUST send the Correlation Data unaltered to all subscribers receiving the Application Message." - zh: "对比数据标识符，服务端在收到应用消息时必须原封不动的把对比数据发送给所有的订阅者。对比数据只对请求消息(Request Message)的发送端和响应消息(Response Message)的接收端有意义。" - } - } - msg_user_properties { - desc { - en: "The User-Property key-value pairs. Note: in case there are duplicated keys, only the last one will be used." - zh: "指定 MQTT 消息的 User Property 键值对。注意，如果出现重复的键，只有最后一个会保留。" - } - } - msg_content_type { - desc { - en: "The Content Type MUST be a UTF-8 Encoded String." - zh: "内容类型标识符，以 UTF-8 格式编码的字符串，用来描述应用消息的内容，服务端必须把收到的应用消息中的内容类型原封不动的发送给所有的订阅者。" - } - } +retain.desc: +"""A boolean field to indicate if this message should be retained.""" + +topic_name.desc: +"""Topic Name""" + } diff --git a/rel/i18n/emqx_mgmt_api_status.hocon b/rel/i18n/emqx_mgmt_api_status.hocon index d72fd0998..28278b747 100644 --- a/rel/i18n/emqx_mgmt_api_status.hocon +++ b/rel/i18n/emqx_mgmt_api_status.hocon @@ -1,48 +1,21 @@ emqx_mgmt_api_status { - get_status_api { - desc { - en: "Serves as a health check for the node. Returns a plain text response" - " describing the status of the node. This endpoint requires no" - " authentication.\n" - "\n" - "Returns status code 200 if the EMQX application is up and running, " - "503 otherwise." - "\n" - "This API was introduced in v5.0.10." - "\n" - "The GET `/status` endpoint (without the `/api/...` prefix) is also an alias" - " to this endpoint and works in the same way. This alias has been available since" - " v5.0.0." - zh: "作为节点的健康检查。 返回一个纯文本的响应，描述节点的状态。\n" - "\n" - "如果 EMQX 应用程序已经启动并运行，返回状态代码 200，否则返回 503。\n" - "\n" - "这个API是在v5.0.10中引入的。" - "\n" - "GET `/status`端点（没有`/api/...`前缀）也是这个端点的一个别名，工作方式相同。" - " 这个别名从v5.0.0开始就有了。" - } - label { - en: "Service health check" - zh: "服务健康检查" - } - } - get_status_response200 { - desc { - en: "Node emqx@127.0.0.1 is started\n" - "emqx is running" - zh: "Node emqx@127.0.0.1 is started\n" - "emqx is running" - } - } +get_status_api.desc: +"""Serves as a health check for the node. Returns a plain text response describing the status of the node. This endpoint requires no authentication. + +Returns status code 200 if the EMQX application is up and running, 503 otherwise. +This API was introduced in v5.0.10. +The GET `/status` endpoint (without the `/api/...` prefix) is also an alias to this endpoint and works in the same way. This alias has been available since v5.0.0.""" + +get_status_api.label: +"""Service health check""" + +get_status_response200.desc: +"""Node emqx@127.0.0.1 is started +emqx is running""" + +get_status_response503.desc: +"""Node emqx@127.0.0.1 is stopped +emqx is not_running""" - get_status_response503 { - desc { - en: "Node emqx@127.0.0.1 is stopped\n" - "emqx is not_running" - zh: "Node emqx@127.0.0.1 is stopped\n" - "emqx is not_running" - } - } } diff --git a/rel/i18n/emqx_modules_schema.hocon b/rel/i18n/emqx_modules_schema.hocon index 248f85341..25588b4e2 100644 --- a/rel/i18n/emqx_modules_schema.hocon +++ b/rel/i18n/emqx_modules_schema.hocon @@ -1,8 +1,13 @@ emqx_modules_schema { - rewrite { - desc { - en: """The topic rewriting function of EMQX supports rewriting topic A to topic B when the client subscribes to topics, publishes messages, and cancels subscriptions according to user-configured rules. +enable.desc: +"""Enable this feature""" + +max_delayed_messages.desc: +"""Maximum number of delayed messages (0 is no limit).""" + +rewrite.desc: +"""The topic rewriting function of EMQX supports rewriting topic A to topic B when the client subscribes to topics, publishes messages, and cancels subscriptions according to user-configured rules. Each rewrite rule consists of three parts: subject filter, regular expression, and target expression. Under the premise that the subject rewriting function is enabled, when EMQX receives a subject-based MQTT message such as a `PUBLISH` message, it will use the subject of the message to sequentially match the subject filter part of the rule in the configuration file. If the match is successful, @@ -13,78 +18,32 @@ It should be noted that EMQX uses reverse order to read the rewrite rules in the When a topic can match the topic filter of multiple topic rewrite rules at the same time, EMQX will only use the first rule it matches. Rewrite. If the regular expression in this rule does not match the subject of the MQTT message, the rewriting will fail, and no other rules will be attempted for rewriting. Therefore, users need to carefully design MQTT message topics and topic rewriting rules when using them.""" - zh: """EMQX 的主题重写功能支持根据用户配置的规则在客户端订阅主题、发布消息、取消订阅的时候将 A 主题重写为 B 主题。 -重写规则分为 Pub 规则和 Sub 规则，Pub 规则匹配 PUSHLISH 报文携带的主题，Sub 规则匹配 SUBSCRIBE、UNSUBSCRIBE 报文携带的主题。 -每条重写规则都由主题过滤器、正则表达式、目标表达式三部分组成。 -在主题重写功能开启的前提下，EMQX 在收到诸如 PUBLISH 报文等带有主题的 MQTT 报文时，将使用报文中的主题去依次匹配配置文件中规则的主题过滤器部分，一旦成功匹配，则使用正则表达式提取主题中的信息，然后替换至目标表达式以构成新的主题。 -目标表达式中可以使用 `$N` 这种格式的变量匹配正则表达中提取出来的元素，`$N` 的值为正则表达式中提取出来的第 N 个元素，比如 `$1` 即为正则表达式提取的第一个元素。 -需要注意的是，EMQX 使用倒序读取配置文件中的重写规则，当一条主题可以同时匹配多条主题重写规则的主题过滤器时，EMQX 仅会使用它匹配到的第一条规则进行重写，如果该条规则中的正则表达式与 MQTT 报文主题不匹配，则重写失败，不会再尝试使用其他的规则进行重写。 -因此用户在使用时需要谨慎的设计 MQTT 报文主题以及主题重写规则。""" - } - label { - en: """Topic Rewrite""" - zh: """主题重写""" - } - } - tr_source_topic { - desc { - en: """Source topic, specified by the client.""" - zh: """源主题，客户端业务指定的主题""" - } - label { - en: """Source Topic""" - zh: """源主题""" - } - } +rewrite.label: +"""Topic Rewrite""" - tr_action { - desc { - en: """Topic rewriting takes effect on the type of operation: +tr_action.desc: +"""Topic rewriting takes effect on the type of operation: - `subscribe`: Rewrite topic when client do subscribe. - `publish`: Rewrite topic when client do publish. - `all`: Both""" - zh: """主题重写在哪种操作上生效： - - `subscribe`：订阅时重写主题； - - `publish`：发布时重写主题； - -`all`：全部重写主题""" - } - label { - en: """Action""" - zh: """Action""" - } - } +tr_action.label: +"""Action""" - tr_re { - desc { - en: """Regular expressions""" - zh: """正则表达式""" - } - } +tr_dest_topic.desc: +"""Destination topic.""" - tr_dest_topic { - desc { - en: """Destination topic.""" - zh: """目标主题。""" - } - label { - en: """Destination Topic""" - zh: """目标主题""" - } - } +tr_dest_topic.label: +"""Destination Topic""" - enable { - desc { - en: "Enable this feature" - zh: "是否开启该功能" - } - } +tr_re.desc: +"""Regular expressions""" + +tr_source_topic.desc: +"""Source topic, specified by the client.""" + +tr_source_topic.label: +"""Source Topic""" - max_delayed_messages { - desc { - en: "Maximum number of delayed messages (0 is no limit)." - zh: "延迟消息的数量上限(0 代表无限)" - } - } } diff --git a/rel/i18n/emqx_mqttsn_schema.hocon b/rel/i18n/emqx_mqttsn_schema.hocon index 20c160b11..1541a8c78 100644 --- a/rel/i18n/emqx_mqttsn_schema.hocon +++ b/rel/i18n/emqx_mqttsn_schema.hocon @@ -1,64 +1,31 @@ emqx_mqttsn_schema { - mqttsn { - desc { - en: """The MQTT-SN Gateway configuration. + +mqttsn.desc: +"""The MQTT-SN Gateway configuration. This gateway only supports the v1.2 protocol""" - zh: """MQTT-SN 网关配置。当前实现仅支持 v1.2 版本""" - } - } - mqttsn_gateway_id { - desc { - en: """MQTT-SN Gateway ID. -When the broadcast option is enabled, the gateway will broadcast ADVERTISE message with this value""" - zh: """MQTT-SN 网关 ID。 -当 broadcast 打开时，MQTT-SN 网关会使用该 ID 来广播 ADVERTISE 消息""" - } - } +mqttsn_broadcast.desc: +"""Whether to periodically broadcast ADVERTISE messages""" - mqttsn_broadcast { - desc { - en: """Whether to periodically broadcast ADVERTISE messages""" - zh: """是否周期性广播 ADVERTISE 消息""" - } - } - - mqttsn_enable_qos3 { - desc { - en: """Allows connectionless clients to publish messages with a Qos of -1. +mqttsn_enable_qos3.desc: +"""Allows connectionless clients to publish messages with a Qos of -1. This feature is defined for very simple client implementations which do not support any other features except this one. There is no connection setup nor tear down, no registration nor subscription. The client just sends its 'PUBLISH' messages to a GW""" - zh: """是否允许无连接的客户端发送 QoS 等于 -1 的消息。 -该功能主要用于支持轻量的 MQTT-SN 客户端实现，它不会向网关建立连接，注册主题，也不会发起订阅；它只使用 QoS 为 -1 来发布消息""" - } - } - mqttsn_subs_resume { - desc { - en: """Whether to initiate all subscribed topic name registration messages to the client after the Session has been taken over by a new channel""" - zh: """在会话被重用后，网关是否主动向客户端注册对已订阅主题名称""" - } - } +mqttsn_gateway_id.desc: +"""MQTT-SN Gateway ID. +When the broadcast option is enabled, the gateway will broadcast ADVERTISE message with this value""" - mqttsn_predefined { - desc { - en: """The pre-defined topic IDs and topic names. +mqttsn_predefined.desc: +"""The pre-defined topic IDs and topic names. A 'pre-defined' topic ID is a topic ID whose mapping to a topic name is known in advance by both the client's application and the gateway""" - zh: """预定义主题列表。 -预定义的主题列表，是一组 主题 ID 和 主题名称 的映射关系。使用预先定义的主题列表，可以减少 MQTT-SN 客户端和网关对于固定主题的注册请求""" - } - } - mqttsn_predefined_id { - desc { - en: """Topic ID. Range: 1-65535""" - zh: """主题 ID。范围：1-65535""" - } - } +mqttsn_predefined_id.desc: +"""Topic ID. Range: 1-65535""" + +mqttsn_predefined_topic.desc: +"""Topic Name""" + +mqttsn_subs_resume.desc: +"""Whether to initiate all subscribed topic name registration messages to the client after the Session has been taken over by a new channel""" - mqttsn_predefined_topic { - desc { - en: """Topic Name""" - zh: """主题名称。注：不支持通配符""" - } - } } diff --git a/rel/i18n/emqx_plugins_schema.hocon b/rel/i18n/emqx_plugins_schema.hocon index c3fed1884..b72c87054 100644 --- a/rel/i18n/emqx_plugins_schema.hocon +++ b/rel/i18n/emqx_plugins_schema.hocon @@ -1,85 +1,55 @@ emqx_plugins_schema { - plugins { - desc { - en: """Manage EMQX plugins.
-Plugins can be pre-built as a part of EMQX package, -or installed as a standalone package in a location specified by -install_dir config key
-The standalone-installed plugins are referred to as 'external' plugins.""" - zh: """管理EMQX插件。
-插件可以是EMQX安装包中的一部分，也可以是一个独立的安装包。
-独立安装的插件称为“外部插件”。""" - } - label { - en: "Plugins" - zh: "插件" - } - } - state { - desc { - en: "A per-plugin config to describe the desired state of the plugin." - zh: "描述插件的状态" - } - label { - en: "State" - zh: "插件状态" - } - } - name_vsn { - desc { - en: """The {name}-{version} of the plugin.
-It should match the plugin application name-version as the for the plugin release package name
-For example: my_plugin-0.1.0.""" - zh: """插件的名称{name}-{version}。
-它应该与插件的发布包名称一致，如my_plugin-0.1.0。""" - } - label { - en: "Name-Version" - zh: "名称-版本" - } - } - enable { - desc { - en: "Set to 'true' to enable this plugin" - zh: "设置为“true”以启用此插件" - } - label { - en: "Enable" - zh: "启用" - } - } - states { - desc { - en: """An array of plugins in the desired states.
-The plugins are started in the defined order""" - zh: """一组插件的状态。插件将按照定义的顺序启动""" - } - label { - en: "States" - zh: "插件启动顺序及状态" - } - } - install_dir { - desc { - en: """The installation directory for the external plugins. + +check_interval.desc: +"""Check interval: check if the status of the plugins in the cluster is consistent,
+if the results of 3 consecutive checks are not consistent, then alarm.""" + +enable.desc: +"""Set to 'true' to enable this plugin""" + +enable.label: +"""Enable""" + +install_dir.desc: +"""The installation directory for the external plugins. The plugin beam files and configuration files should reside in the subdirectory named as emqx_foo_bar-0.1.0.
NOTE: For security reasons, this directory should **NOT** be writable by anyone except emqx (or any user which runs EMQX).""" - zh: "插件安装包的目录，出于安全考虑，该目录应该值允许 emqx，或用于运行 EMQX 服务的用户拥有写入权限。" - } - label { - en: "Install Directory" - zh: "安装目录" - } - } - check_interval { - desc { - en: """Check interval: check if the status of the plugins in the cluster is consistent,
-if the results of 3 consecutive checks are not consistent, then alarm.""" - zh: """检查间隔：检查集群中插件的状态是否一致，
-如果连续3次检查结果不一致，则报警。""" - } - } + +install_dir.label: +"""Install Directory""" + +name_vsn.desc: +"""The {name}-{version} of the plugin.
+It should match the plugin application name-version as the for the plugin release package name
+For example: my_plugin-0.1.0.""" + +name_vsn.label: +"""Name-Version""" + +plugins.desc: +"""Manage EMQX plugins.
+Plugins can be pre-built as a part of EMQX package, +or installed as a standalone package in a location specified by +install_dir config key
+The standalone-installed plugins are referred to as 'external' plugins.""" + +plugins.label: +"""Plugins""" + +state.desc: +"""A per-plugin config to describe the desired state of the plugin.""" + +state.label: +"""State""" + +states.desc: +"""An array of plugins in the desired states.
+The plugins are started in the defined order""" + +states.label: +"""States""" + } diff --git a/rel/i18n/emqx_prometheus_schema.hocon b/rel/i18n/emqx_prometheus_schema.hocon index e9a0fc11e..d79685a4d 100644 --- a/rel/i18n/emqx_prometheus_schema.hocon +++ b/rel/i18n/emqx_prometheus_schema.hocon @@ -1,95 +1,47 @@ emqx_prometheus_schema { - prometheus { - desc { - en: """Settings for reporting metrics to Prometheus""" - zh: """Prometheus 监控数据推送""" - } - label { - en: """Prometheus""" - zh: """Prometheus""" - } - } +enable.desc: +"""Turn Prometheus data pushing on or off""" - push_gateway_server { - desc { - en: """URL of Prometheus server""" - zh: """Prometheus 服务器地址""" - } - } - - interval { - desc { - en: """Data reporting interval""" - zh: """数据推送间隔""" - } - } - - headers { - desc { - en: """A list of HTTP Headers when pushing to Push Gateway.
+headers.desc: +"""A list of HTTP Headers when pushing to Push Gateway.
For example, { Authorization = "some-authz-tokens"}""" - zh: """推送到 Push Gateway 的 HTTP Headers 列表。
-例如， { Authorization = "some-authz-tokens"}""" - } - } - job_name { - desc { - en: """Job Name that is pushed to the Push Gateway. Available variables:
+interval.desc: +"""Data reporting interval""" + +job_name.desc: +"""Job Name that is pushed to the Push Gateway. Available variables:
- ${name}: Name of EMQX node.
- ${host}: Host name of EMQX node.
For example, when the EMQX node name is emqx@127.0.0.1 then the name variable takes value emqx and the host variable takes value 127.0.0.1.
Default value is: ${name}/instance/${name}~${host}""" - zh: """推送到 Push Gateway 的 Job 名称。可用变量为：
-- ${name}: EMQX 节点的名称。 -- ${host}: EMQX 节点主机名。 -例如，当 EMQX 节点名为 emqx@127.0.0.1 则 name 变量的值为 emqx，host 变量的值为 127.0.0.1。
-默认值为: ${name}/instance/${name}~${host}""" - } - } - enable { - desc { - en: """Turn Prometheus data pushing on or off""" - zh: """开启或关闭 Prometheus 数据推送""" - } - } - vm_dist_collector { - desc { - en: """Enable or disable VM distribution collector, collects information about the sockets and processes involved in the Erlang distribution mechanism.""" - zh: """开启或关闭 VM 分布采集器，收集 Erlang 分布机制中涉及的套接字和进程的信息。""" - } - } - mnesia_collector { - desc { - en: """Enable or disable Mnesia collector, collects Mnesia metrics mainly using mnesia:system_info/1 .""" - zh: """开启或关闭 Mnesia 采集器, 使用 mnesia:system_info/1 收集 Mnesia 相关指标""" - } - } - vm_statistics_collector { - desc { - en: """Enable or disable VM statistics collector, collects Erlang VM metrics using erlang:statistics/1 .""" - zh: """开启或关闭 VM 统计采集器, 使用 erlang:statistics/1 收集 Erlang VM 相关指标""" - } - } +mnesia_collector.desc: +"""Enable or disable Mnesia collector, collects Mnesia metrics mainly using mnesia:system_info/1 .""" + +prometheus.desc: +"""Settings for reporting metrics to Prometheus""" + +prometheus.label: +"""Prometheus""" + +push_gateway_server.desc: +"""URL of Prometheus server""" + +vm_dist_collector.desc: +"""Enable or disable VM distribution collector, collects information about the sockets and processes involved in the Erlang distribution mechanism.""" + +vm_memory_collector.desc: +"""Enable or disable VM memory collector, collects information about memory dynamically allocated by the Erlang emulator using erlang:memory/0 , also provides basic (D)ETS statistics .""" + +vm_msacc_collector.desc: +"""Enable or disable VM msacc collector, collects microstate accounting metrics using erlang:statistics(microstate_accounting) .""" + +vm_statistics_collector.desc: +"""Enable or disable VM statistics collector, collects Erlang VM metrics using erlang:statistics/1 .""" + +vm_system_info_collector.desc: +"""Enable or disable VM system info collector, collects Erlang VM metrics using erlang:system_info/1 .""" - vm_system_info_collector { - desc { - en: """Enable or disable VM system info collector, collects Erlang VM metrics using erlang:system_info/1 .""" - zh: """开启或关闭 VM 系统信息采集器, 使用 erlang:system_info/1 收集 Erlang VM 相关指标""" - } - } - vm_memory_collector { - desc { - en: """Enable or disable VM memory collector, collects information about memory dynamically allocated by the Erlang emulator using erlang:memory/0 , also provides basic (D)ETS statistics .""" - zh: """开启或关闭 VM 内存采集器, 使用 erlang:memory/0 收集 Erlang 虚拟机动态分配的内存信息，同时提供基本的 (D)ETS 统计信息""" - } - } - vm_msacc_collector { - desc { - en: """Enable or disable VM msacc collector, collects microstate accounting metrics using erlang:statistics(microstate_accounting) .""" - zh: """开启或关闭 VM msacc 采集器, 使用 erlang:statistics(microstate_accounting) 收集微状态计数指标""" - } - } } diff --git a/rel/i18n/emqx_psk_schema.hocon b/rel/i18n/emqx_psk_schema.hocon index 60d45977a..1a99e1b19 100644 --- a/rel/i18n/emqx_psk_schema.hocon +++ b/rel/i18n/emqx_psk_schema.hocon @@ -1,8 +1,18 @@ emqx_psk_schema { - psk_authentication { - desc { - en: """PSK stands for 'Pre-Shared Keys'. +chunk_size.desc: +"""The size of each chunk used to import to the built-in database from PSK file""" + +enable.desc: +"""Whether to enable TLS PSK support""" + +init_file.desc: +"""If init_file is specified, EMQX will import PSKs from the file into the built-in database at startup for use by the runtime. +The file has to be structured line-by-line, each line must be in the format of PSKIdentity:SharedSecret. +For example: mydevice1:c2VjcmV0""" + +psk_authentication.desc: +"""PSK stands for 'Pre-Shared Keys'. This config to enable TLS-PSK authentication. Important! Make sure the SSL listener with only tlsv1.2 enabled, and also PSK cipher suites @@ -11,49 +21,8 @@ configured, such as RSA-PSK-AES256-GCM-SHA384. See listener SSL options config for more details. The IDs and secrets can be provided from a file which is configurable by the init_file field.""" - zh: """此配置用于启用 TLS-PSK 身份验证。 -PSK 是 “Pre-Shared-Keys” 的缩写。 - -注意: 确保 SSL 监听器仅启用了 'tlsv1.2'，并且配置了PSK 密码套件，例如 'RSA-PSK-AES256-GCM-SHA384'。 - -可以通过查看监听器中的 SSL 选项，了解更多详细信息。 - -可以通过配置 'init_file' 来设置初始化的 ID 和 密钥""" - } - } - - enable { - desc { - en: "Whether to enable TLS PSK support" - zh: "是否开启 TLS PSK 支持" - } - } - - init_file { - desc { - en: """If init_file is specified, EMQX will import PSKs from the file into the built-in database at startup for use by the runtime. -The file has to be structured line-by-line, each line must be in the format of PSKIdentity:SharedSecret. -For example: mydevice1:c2VjcmV0""" - zh: """如果设置了初始化文件，EMQX 将在启动时从初始化文件中导入 PSK 信息到内建数据库中。 -这个文件需要按行进行组织，每一行必须遵守如下格式: PSKIdentity:SharedSecret -例如: mydevice1:c2VjcmV0""" - } - } - - separator { - desc { - en: "The separator between PSKIdentity and SharedSecret in the PSK file" - - zh: "PSK 文件中 PSKIdentity 和 SharedSecret 之间的分隔符" - } - } - - chunk_size { - desc { - en: "The size of each chunk used to import to the built-in database from PSK file" - zh: "将 PSK 文件导入到内建数据时每个块的大小" - } - } +separator.desc: +"""The separator between PSKIdentity and SharedSecret in the PSK file""" } diff --git a/rel/i18n/emqx_resource_schema.hocon b/rel/i18n/emqx_resource_schema.hocon index f4a9982bc..8fc781794 100644 --- a/rel/i18n/emqx_resource_schema.hocon +++ b/rel/i18n/emqx_resource_schema.hocon @@ -1,214 +1,116 @@ emqx_resource_schema { - resource_opts { - desc { - en: """Resource options.""" - zh: """资源相关的选项。""" - } - label { - en: """Resource Options""" - zh: """资源选项""" - } - } +auto_restart_interval.desc: +"""The auto restart interval after the resource is disconnected.""" - creation_opts { - desc { - en: """Creation options.""" - zh: """资源启动相关的选项。""" - } - label { - en: """Creation Options""" - zh: """资源启动选项""" - } - } +auto_restart_interval.label: +"""Auto Restart Interval""" - worker_pool_size { - desc { - en: """The number of buffer workers. Only applicable for egress type bridges. -For bridges only have ingress direction data flow, it can be set to 0 otherwise must be greater than 0.""" - zh: """缓存队列 worker 数量。仅对 egress 类型的桥接有意义。当桥接仅有 ingress 方向时，可设置为 0，否则必须大于 0。""" - } - label { - en: """Buffer Pool Size""" - zh: """缓存池大小""" - } - } +batch_size.desc: +"""Maximum batch count. If equal to 1, there's effectively no batching.""" - health_check_interval { - desc { - en: """Health check interval.""" - zh: """健康检查间隔。""" - } - label { - en: """Health Check Interval""" - zh: """健康检查间隔""" - } - } +batch_size.label: +"""Max batch size""" - resume_interval { - desc { - en: """The interval at which the buffer worker attempts to resend failed requests in the inflight window.""" - zh: """在发送失败后尝试重传飞行窗口中的请求的时间间隔。""" - } - label { - en: """Resume Interval""" - zh: """重试时间间隔""" - } - } +batch_time.desc: +"""Maximum waiting interval when accumulating a batch at a low message rates for more efficient resource usage.""" - start_after_created { - desc { - en: """Whether start the resource right after created.""" - zh: """是否在创建资源后立即启动资源。""" - } - label { - en: """Start After Created""" - zh: """创建后立即启动""" - } - } +batch_time.label: +"""Max batch wait time""" - start_timeout { - desc { - en: """Time interval to wait for an auto-started resource to become healthy before responding resource creation requests.""" - zh: """在回复资源创建请求前等待资源进入健康状态的时间。""" - } - label { - en: """Start Timeout""" - zh: """启动超时时间""" - } - } +buffer_mode.desc: +"""Buffer operation mode. +memory_only: Buffer all messages in memory.volatile_offload: Buffer message in memory first, when up to certain limit (see buffer_seg_bytes config for more information), then start offloading messages to disk""" - auto_restart_interval { - desc { - en: """The auto restart interval after the resource is disconnected.""" - zh: """资源断开以后，自动重连的时间间隔。""" - } - label { - en: """Auto Restart Interval""" - zh: """自动重连间隔""" - } - } +buffer_mode.label: +"""Buffer Mode""" - query_mode { - desc { - en: """Query mode. Optional 'sync/async', default 'async'.""" - zh: """请求模式。可选 '同步/异步'，默认为'异步'模式。""" - } - label { - en: """Query mode""" - zh: """请求模式""" - } - } +buffer_seg_bytes.desc: +"""Applicable when buffer mode is set to volatile_offload. +This value is to specify the size of each on-disk buffer file.""" - request_timeout { - desc { - en: """Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired.""" - zh: """从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。""" - } - label { - en: """Request Expiry""" - zh: """请求超期""" - } - } +buffer_seg_bytes.label: +"""Segment File Bytes""" - enable_batch { - desc { - en: """Batch mode enabled.""" - zh: """启用批量模式。""" - } - label { - en: """Enable batch""" - zh: """启用批量模式""" - } - } +creation_opts.desc: +"""Creation options.""" - enable_queue { - desc { - en: """Enable disk buffer queue (only applicable for egress bridges). +creation_opts.label: +"""Creation Options""" + +enable_batch.desc: +"""Batch mode enabled.""" + +enable_batch.label: +"""Enable batch""" + +enable_queue.desc: +"""Enable disk buffer queue (only applicable for egress bridges). When Enabled, messages will be buffered on disk when the bridge connection is down. When disabled the messages are buffered in RAM only.""" - zh: """启用磁盘缓存队列（仅对 egress 方向桥接有用）。""" - } - label { - en: """Enable disk buffer queue""" - zh: """启用磁盘缓存队列""" - } - } - inflight_window { - desc { - en: """Query inflight window. When query_mode is set to async, this config has to be set to 1 if messages from the same MQTT client have to be strictly ordered.""" - zh: """请求飞行队列窗口大小。当请求模式为异步时，如果需要严格保证来自同一 MQTT 客户端的消息有序，则必须将此值设为 1。""""" - } - label { - en: """Inflight window""" - zh: """请求飞行队列窗口""" - } - } +enable_queue.label: +"""Enable disk buffer queue""" - batch_size { - desc { - en: """Maximum batch count. If equal to 1, there's effectively no batching.""" - zh: """最大批量请求大小。如果设为1，则无批处理。""" - } - label { - en: """Max batch size""" - zh: """最大批量请求大小""" - } - } +health_check_interval.desc: +"""Health check interval.""" - batch_time { - desc { - en: """Maximum waiting interval when accumulating a batch at a low message rates for more efficient resource usage.""" - zh: """在较低消息率情况下尝试累积批量输出时的最大等待间隔，以提高资源的利用率。""" - } - label { - en: """Max batch wait time""" - zh: """批量等待最大间隔""" - } - } +health_check_interval.label: +"""Health Check Interval""" - max_buffer_bytes { - desc { - en: """Maximum number of bytes to buffer for each buffer worker.""" - zh: """每个缓存 worker 允许使用的最大字节数。""" - } - label { - en: """Max buffer queue size""" - zh: """缓存队列最大长度""" - } - } +inflight_window.desc: +"""Query inflight window. When query_mode is set to async, this config has to be set to 1 if messages from the same MQTT client have to be strictly ordered.""" - buffer_seg_bytes { - desc { - en: "Applicable when buffer mode is set to volatile_offload.\n" - "This value is to specify the size of each on-disk buffer file." - zh: "当缓存模式是 volatile_offload 时适用。" - "该配置用于指定缓存到磁盘上的文件的大小。" - } - label { - en: "Segment File Bytes" - zh: "缓存文件大小" - } - } +inflight_window.label: +"""Inflight window""" - buffer_mode { - desc { - en: "Buffer operation mode.\n" - "memory_only: Buffer all messages in memory." - "volatile_offload: Buffer message in memory first, when up to certain limit" - " (see buffer_seg_bytes config for more information), then start offloading messages to disk" - zh: "队列操作模式。\n" - "memory_only: 所有的消息都缓存在内存里。" - "volatile_offload: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制" - "（配置项 buffer_seg_bytes 指定该限制）后，" - " 消息会开始缓存到磁盘上。" - } - label { - en: "Buffer Mode" - zh: "缓存模式" - } - } +max_buffer_bytes.desc: +"""Maximum number of bytes to buffer for each buffer worker.""" +max_buffer_bytes.label: +"""Max buffer queue size""" + +query_mode.desc: +"""Query mode. Optional 'sync/async', default 'async'.""" + +query_mode.label: +"""Query mode""" + +request_timeout.desc: +"""Starting from the moment when the request enters the buffer, if the request remains in the buffer for the specified time or is sent but does not receive a response or acknowledgement in time, the request is considered expired.""" + +request_timeout.label: +"""Request Expiry""" + +resource_opts.desc: +"""Resource options.""" + +resource_opts.label: +"""Resource Options""" + +resume_interval.desc: +"""The interval at which the buffer worker attempts to resend failed requests in the inflight window.""" + +resume_interval.label: +"""Resume Interval""" + +start_after_created.desc: +"""Whether start the resource right after created.""" + +start_after_created.label: +"""Start After Created""" + +start_timeout.desc: +"""Time interval to wait for an auto-started resource to become healthy before responding resource creation requests.""" + +start_timeout.label: +"""Start Timeout""" + +worker_pool_size.desc: +"""The number of buffer workers. Only applicable for egress type bridges. +For bridges only have ingress direction data flow, it can be set to 0 otherwise must be greater than 0.""" + +worker_pool_size.label: +"""Buffer Pool Size""" } diff --git a/rel/i18n/emqx_retainer_api.hocon b/rel/i18n/emqx_retainer_api.hocon index aced87076..5c7084778 100644 --- a/rel/i18n/emqx_retainer_api.hocon +++ b/rel/i18n/emqx_retainer_api.hocon @@ -1,143 +1,63 @@ emqx_retainer_api { - get_config_api { - desc { - en: "View config" - zh: "查看配置内容" - } - } +config_content.desc: +"""The config content""" - config_content { - desc { - en: "The config content" - zh: "配置内容" - } - } +config_not_found.desc: +"""Config not found.""" - config_not_found { - desc { - en: "Config not found." - zh: "配置不存在" - } - } +delete_matching_api.desc: +"""Delete matching messages.""" - update_retainer_api { - desc { - en: "Update retainer config." - zh: "更新配置" - } - } +from_clientid.desc: +"""The clientid of publisher.""" - update_config_success { - desc { - en: "Update configs successfully." - zh: "配置更新成功" - } - } +from_username.desc: +"""The username of publisher.""" - update_config_failed { - desc { - en: "Update config failed" - zh: "配置更新失败" - } - } +get_config_api.desc: +"""View config""" - list_retained_api { - desc { - en: "List retained messages." - zh: "查看保留消息列表" - } - } +list_retained_api.desc: +"""List retained messages.""" - retained_list { - desc { - en: "Retained messages list." - zh: "保留消息列表" - } - } +lookup_api.desc: +"""Lookup a message by a topic without wildcards.""" - unsupported_backend { - desc { - en: "Unsupported backend." - zh: "不支持的后端" - } - } +message_detail.desc: +"""Details of the message.""" - lookup_api { - desc { - en: "Lookup a message by a topic without wildcards." - zh: "通过不带通配符的主题查看对应的保留消息" - } - } +message_not_exist.desc: +"""Viewed message doesn't exist.""" - message_detail { - desc { - en: "Details of the message." - zh: "消息详情" - } - } +msgid.desc: +"""Message ID.""" - message_not_exist { - desc { - en: "Viewed message doesn't exist." - zh: "消息不存在" - } - } +payload.desc: +"""Payload.""" - delete_matching_api { - desc { - en: "Delete matching messages." - zh: "删除对应的消息" - } - } +publish_at.desc: +"""Message publish time, RFC 3339 format.""" - topic { - desc { - en: "Topic." - zh: "主题" - } - } +qos.desc: +"""QoS.""" - msgid { - desc { - en: "Message ID." - zh: "消息 ID" - } - } +retained_list.desc: +"""Retained messages list.""" - qos { - desc { - en: "QoS." - zh: "QoS" - } - } +topic.desc: +"""Topic.""" - publish_at { - desc { - en: "Message publish time, RFC 3339 format." - zh: "消息发送时间, RFC 3339 格式" - } - } +unsupported_backend.desc: +"""Unsupported backend.""" - from_clientid { - desc { - en: "The clientid of publisher." - zh: "发布者的 ClientID" - } - } +update_config_failed.desc: +"""Update config failed""" - from_username { - desc { - en: "The username of publisher." - zh: "发布者的用户名" - } - } +update_config_success.desc: +"""Update configs successfully.""" - payload { - desc { - en: "Payload." - zh: "消息内容" - } - } +update_retainer_api.desc: +"""Update retainer config.""" } diff --git a/rel/i18n/emqx_retainer_schema.hocon b/rel/i18n/emqx_retainer_schema.hocon index 274c260d4..9b2905da1 100644 --- a/rel/i18n/emqx_retainer_schema.hocon +++ b/rel/i18n/emqx_retainer_schema.hocon @@ -1,105 +1,49 @@ emqx_retainer_schema { - enable { - desc { - en: "Enable retainer feature" - zh: "是否开启消息保留功能" - } - } +backend.desc: +"""Settings for the database storing the retained messages.""" - msg_expiry_interval { - desc { - en: "Message retention time. 0 means message will never be expired." - zh: "消息保留时间。0 代表永久保留" - } - } - - flow_control { - desc { - en: "Flow control." - zh: "流控设置" - } - } - - msg_clear_interval { - desc { - en: """Periodic interval for cleaning up expired messages. -Never clear if the value is 0.""" - zh: "消息清理间隔。0 代表不进行清理" - } - } - - max_payload_size { - desc { - en: "Maximum retained message size." - zh: "消息大小最大值" - } - } - - stop_publish_clear_msg { - desc { - en: """When the retained flag of the `PUBLISH` message is set and Payload is empty, -whether to continue to publish the message. -See: -http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718038""" - zh: """是否不发送保留消息的清理消息，在 MQTT 5.0 中如果一条保留消息的消息体为空，则会清除掉之前存储 -的对应的保留消息，通过这个值控制是否停止发送清理消息""" - } - } - - backend { - desc { - en: "Settings for the database storing the retained messages." - zh: "保留消息的存储后端" - } - } - - mnesia_config_type { - desc { - en: "Backend type." - zh: "后端类型" - } - } - - mnesia_config_storage_type { - desc { - en: "Specifies whether the messages are stored in RAM or persisted on disc." - zh: "选择消息是存放在磁盘还是内存中" - } - } - - max_retained_messages { - desc { - en: "Maximum number of retained messages. 0 means no limit." - zh: "消息保留的数量上限。0 表示无限" - } - } - - batch_read_number { - desc { - en: "Size of the batch when reading messages from storage. 0 means no limit." - zh: "从存储后端批量加载时的每批数量上限，0 代表一次性读取" - } - } - - batch_deliver_number { - desc { - en: "The number of retained messages can be delivered per batch." - zh: "批量派发时每批的数量。0 代表一次性全部派发" - } - } - - batch_deliver_limiter { - desc { - en: """The rate limiter name for retained messages' delivery. +batch_deliver_limiter.desc: +"""The rate limiter name for retained messages' delivery. Limiter helps to avoid delivering too many messages to the client at once, which may cause the client to block or crash, or drop messages due to exceeding the size of the message queue. The names of the available rate limiters are taken from the existing rate limiters under `limiter.batch`. If this field is empty, limiter is not used.""" - zh: """批量发送的限流器的名称。 -限流器可以用来防止短时间内向客户端发送太多的消息，从而避免过多的消息导致客户端队列堵塞甚至崩溃。 -这个名称需要是指向 `limiter.batch` 下的一个真实存在的限流器。 -如果这个字段为空，则不使用限流器。""" - } - } + +batch_deliver_number.desc: +"""The number of retained messages can be delivered per batch.""" + +batch_read_number.desc: +"""Size of the batch when reading messages from storage. 0 means no limit.""" + +enable.desc: +"""Enable retainer feature""" + +flow_control.desc: +"""Flow control.""" + +max_payload_size.desc: +"""Maximum retained message size.""" + +max_retained_messages.desc: +"""Maximum number of retained messages. 0 means no limit.""" + +mnesia_config_storage_type.desc: +"""Specifies whether the messages are stored in RAM or persisted on disc.""" + +mnesia_config_type.desc: +"""Backend type.""" + +msg_clear_interval.desc: +"""Periodic interval for cleaning up expired messages. +Never clear if the value is 0.""" + +msg_expiry_interval.desc: +"""Message retention time. 0 means message will never be expired.""" + +stop_publish_clear_msg.desc: +"""When the retained flag of the `PUBLISH` message is set and Payload is empty, +whether to continue to publish the message. +See: +http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718038""" } diff --git a/rel/i18n/emqx_rewrite_api.hocon b/rel/i18n/emqx_rewrite_api.hocon index 91f15cb8c..7e090e590 100644 --- a/rel/i18n/emqx_rewrite_api.hocon +++ b/rel/i18n/emqx_rewrite_api.hocon @@ -1,25 +1,12 @@ emqx_rewrite_api { - list_topic_rewrite_api { - desc { - en: """List all rewrite rules""" - zh: """列出全部主题重写规则""" - } - } +list_topic_rewrite_api.desc: +"""List all rewrite rules""" - update_topic_rewrite_api { - desc { - en: """Update all rewrite rules""" - zh: """更新全部主题重写规则""" - } - } +update_topic_rewrite_api.desc: +"""Update all rewrite rules""" - - update_topic_rewrite_api_response413 { - desc { - en: """Rules count exceed max limit""" - zh: """超出主题重写规则数量上限""" - } - } +update_topic_rewrite_api_response413.desc: +"""Rules count exceed max limit""" } diff --git a/rel/i18n/emqx_rule_api_schema.hocon b/rel/i18n/emqx_rule_api_schema.hocon index 0d8253223..29ecaa18e 100644 --- a/rel/i18n/emqx_rule_api_schema.hocon +++ b/rel/i18n/emqx_rule_api_schema.hocon @@ -1,696 +1,381 @@ emqx_rule_api_schema { - event_event_type { - desc { - en: "Event Type" - zh: "事件类型" - } - label: { - en: "Event Type" - zh: "事件类型" - } - } +event_action.desc: +"""Publish or Subscribe""" - event_id { - desc { - en: "Message ID" - zh: "消息 ID" - } - label: { - en: "Message ID" - zh: "消息 ID" - } - } +event_action.label: +"""Publish or Subscribe""" - event_clientid { - desc { - en: "The Client ID" - zh: "客户端 ID" - } - label: { - en: "Client ID" - zh: "客户端 ID" - } - } +event_payload.desc: +"""The Message Payload""" - event_username { - desc { - en: "Username" - zh: "用户名" - } - label: { - en: "Username" - zh: "用户名" - } - } +event_payload.label: +"""Message Payload""" - event_payload { - desc { - en: "The Message Payload" - zh: "消息负载" - } - label: { - en: "Message Payload" - zh: "消息负载" - } - } +metrics_actions_failed_out_of_service.desc: +"""How much times the rule failed to call actions due to the action is out of service. For example, a bridge is disabled or stopped.""" - event_peerhost { - desc { - en: "The IP Address of the Peer Client" - zh: "对等客户端的 IP 地址" - } - label: { - en: "Peer IP Address" - zh: "对等客户端的 IP" - } - } +metrics_actions_failed_out_of_service.label: +"""Fail Action""" - event_topic { - desc { - en: "Message Topic" - zh: "消息主题" - } - label: { - en: "Message Topic" - zh: "消息主题" - } - } +metrics_actions_failed_unknown.desc: +"""How much times the rule failed to call actions due to to an unknown error.""" - event_publish_received_at { - desc { - en: "The Time that this Message is Received" - zh: "消息被接受的时间" - } - label: { - en: "Message Received Time" - zh: "消息被接受的时间" - } - } +metrics_actions_failed_unknown.label: +"""Fail Action""" - event_qos { - desc { - en: "The Message QoS" - zh: "消息的 QoS" - } - label: { - en: "Message QoS" - zh: "消息 QoS" - } - } +event_server.desc: +"""The IP address (or hostname) and port of the MQTT broker, in IP:Port format""" - event_from_clientid { - desc { - en: "The Client ID" - zh: "事件来源客户端的 ID" - } - label: { - en: "Client ID" - zh: "客户端 ID" - } - } +event_server.label: +"""Server IP And Port""" - event_from_username { - desc { - en: "The User Name" - zh: "事件来源客户端的用户名" - } - label: { - en: "Username" - zh: "用户名" - } - } +metrics_actions_total.desc: +"""How much times the actions are called by the rule. This value may several times of 'matched', depending on the number of the actions of the rule.""" - event_mountpoint { - desc { - en: "The Mountpoint" - zh: "挂载点" - } - label: { - en: "Mountpoint" - zh: "挂载点" - } - } +metrics_actions_total.label: +"""Action Total""" - event_peername { - desc { - en: "The IP Address and Port of the Peer Client" - zh: "对等客户端的 IP 地址和端口" - } - label: { - en: "IP Address And Port" - zh: "IP 地址和端口" - } - } +event_ctx_disconnected_da.desc: +"""The Time that this Client is Disconnected""" - event_sockname { - desc { - en: "The IP Address and Port of the Local Listener" - zh: "本地监听的 IP 地址和端口" - } - label: { - en: "IP Address And Port" - zh: "IP 地址和端口" - } - } +event_ctx_disconnected_da.label: +"""Disconnected Time""" - event_proto_name { - desc { - en: "Protocol Name" - zh: "协议名称" - } - label: { - en: "Protocol Name" - zh: "协议名称" - } - } +event_topic.desc: +"""Message Topic""" - event_proto_ver { - desc { - en: "Protocol Version" - zh: "协议版本" - } - label: { - en: "Protocol Version" - zh: "协议版本" - } - } +event_topic.label: +"""Message Topic""" - event_keepalive { - desc { - en: "KeepAlive" - zh: "保持连接" - } - label: { - en: "KeepAlive" - zh: "保持连接" - } - } +event_peername.desc: +"""The IP Address and Port of the Peer Client""" - event_clean_start { - desc { - en: "Clean Start" - zh: "清除会话" - } - label: { - en: "Clean Start" - zh: "清除会话" - } - } +event_peername.label: +"""IP Address And Port""" - event_expiry_interval { - desc { - en: "Expiry Interval" - zh: "到期间隔" - } - label: { - en: "Expiry Interval" - zh: "到期间隔" - } - } +metrics_sql_passed.desc: +"""How much times the SQL is passed""" - event_is_bridge { - desc { - en: "Is Bridge" - zh: "是否桥接" - } - label: { - en: "Is Bridge" - zh: "是否桥接" - } - } +metrics_sql_passed.label: +"""SQL Passed""" - event_connected_at { - desc { - en: "The Time that this Client is Connected" - zh: "客户端连接完成时的时刻" - } - label: { - en: "Connected Time" - zh: "连接完成时的时刻" - } - } +test_context.desc: +"""The context of the event for testing""" - event_action { - desc { - en: "Publish or Subscribe" - zh: "订阅或发布" - } - label: { - en: "Publish or Subscribe" - zh: "订阅或发布" - } - } +test_context.label: +"""Event Conetxt""" - event_authz_source { - desc { - en: "Cache, Plugs or Default" - zh: "缓存，插件或者默认值" - } - label: { - en: "Auth Source" - zh: "认证源" - } - } +node_node.desc: +"""The node name""" - event_result { - desc { - en: "Allow or Deny" - zh: "允许或禁止" - } - label: { - en: "Auth Result" - zh: "认证结果" - } - } +node_node.label: +"""Node Name""" - event_server { - desc { - en: "The IP address (or hostname) and port of the MQTT broker, in IP:Port format" - zh: "MQTT broker的 IP 地址（或主机名）和端口，采用 IP:Port 格式" - } - label: { - en: "Server IP And Port" - zh: "服务器 IP 地址和端口" - } - } +event_from_clientid.desc: +"""The Client ID""" - event_dup { - desc { - en: "The DUP flag of the MQTT message" - zh: "MQTT 消息的 DUP 标志" - } - label: { - en: "DUP Flag" - zh: "DUP 标志" - } - } +event_from_clientid.label: +"""Client ID""" - event_retain { - desc { - en: "If is a retain message" - zh: "是否是保留消息" - } - label: { - en: "Retain Message" - zh: "保留消息" - } - } +event_keepalive.desc: +"""KeepAlive""" - event_ctx_dropped { - desc { - en: "The Reason for Dropping" - zh: "消息被丢弃的原因" - } - label: { - en: "Dropped Reason" - zh: "丢弃原因" - } - } +event_keepalive.label: +"""KeepAlive""" - event_ctx_disconnected_reason { - desc { - en: "The Reason for Disconnect" - zh: "断开连接的原因" - } - label: { - en: "Disconnect Reason" - zh: "断开连接原因" - } - } +event_connected_at.desc: +"""The Time that this Client is Connected""" - event_ctx_disconnected_da { - desc { - en: "The Time that this Client is Disconnected" - zh: "客户端断开连接的时刻" - } - label: { - en: "Disconnected Time" - zh: "客户端断开连接时刻" - } - } +event_connected_at.label: +"""Connected Time""" - event_ctx_connack_reason_code { - desc { - en: "The reason code" - zh: "错误码" - } - label: { - en: "Reason Code" - zh: "错误码" - } - } +metrics_sql_failed_exception.desc: +"""How much times the SQL is failed due to exceptions. This may because of a crash when calling a SQL function, or trying to do arithmetic operation on undefined variables""" - rule_id { - desc { - en: "The ID of the rule" - zh: "规则的 ID" - } - label: { - en: "Rule ID" - zh: "规则 ID" - } - } +metrics_sql_failed_exception.label: +"""SQL Exception""" - node_node { - desc { - en: "The node name" - zh: "节点名字" - } - label: { - en: "Node Name" - zh: "节点名字" - } - } +event_from_username.desc: +"""The User Name""" - metrics_sql_matched { - desc { - en: "How much times the FROM clause of the SQL is matched." - zh: "SQL 的 FROM 子句匹配的次数。" - } - label: { - en: "Matched" - zh: "命中数" - } - } +event_from_username.label: +"""Username""" - metrics_sql_matched_rate { - desc { - en: "The rate of matched, times/second" - zh: "命中速率，次/秒" - } - label: { - en: "命中速率" - zh: "Matched Rate" - } - } +event_ctx_connack_reason_code.desc: +"""The reason code""" - metrics_sql_matched_rate_max { - desc { - en: "The max rate of matched, times/second" - zh: "最大命中速率，次/秒" - } - label: { - en: "Max Matched Rate" - zh: "最大命中速率" - } - } +event_ctx_connack_reason_code.label: +"""Reason Code""" - metrics_sql_matched_rate_last5m { - desc { - en: "The average rate of matched in last 5 minutes, times/second" - zh: "5分钟平均命中速率，次/秒" - } - label: { - en: "Average Matched Rate" - zh: "平均命中速率" - } - } +rs_description.desc: +"""The description""" - metrics_sql_passed { - desc { - en: "How much times the SQL is passed" - zh: "SQL 通过的次数" - } - label: { - en: "SQL Passed" - zh: "SQL 通过" - } - } +rs_description.label: +"""Description""" - metrics_sql_failed { - desc { - en: "How much times the SQL is failed" - zh: "SQL 失败的次数" - } - label: { - en: "SQL Failed" - zh: "SQL 失败" - } - } +rule_id.desc: +"""The ID of the rule""" - metrics_sql_failed_exception { - desc { - en: "How much times the SQL is failed due to exceptions. This may because of a crash when calling a SQL function, or trying to do arithmetic operation on undefined variables" - zh: "SQL 由于执行异常而失败的次数。 这可能是因为调用 SQL 函数时崩溃，或者试图对未定义的变量进行算术运算" - } - label: { - en: "SQL Exception" - zh: "SQL 执行异常" - } - } +rule_id.label: +"""Rule ID""" - metrics_sql_failed_unknown { - desc { - en: "How much times the SQL is failed due to an unknown error." - zh: "由于未知错误导致 SQL 失败的次数。" - } - label: { - en: "SQL Unknown Error" - zh: "SQL 未知错误" - } - } +rs_event.desc: +"""The event topics""" - metrics_actions_total { - desc { - en: "How much times the actions are called by the rule. This value may several times of 'matched', depending on the number of the actions of the rule." - zh: "规则调用输出的次数。 该值可能是“sql.matched”的几倍，具体取决于规则输出的数量。" - } - label: { - en: "Action Total" - zh: "调用输出次数" - } - } +rs_event.label: +"""Event Topics""" - metrics_actions_success { - desc { - en: "How much times the rule success to call the actions." - zh: "规则成功调用输出的次数。" - } - label: { - en: "Success Action" - zh: "成功调用输出次数" - } - } +root_rule_engine.desc: +"""Rule engine configurations. This API can be used to change EMQX rule engine settings. But not for the rules. To list, create, or update rules, call the '/rules' API instead.""" - metrics_actions_failed { - desc { - en: "How much times the rule failed to call the actions." - zh: "规则调用输出失败的次数。" - } - label: { - en: "Failed Action" - zh: "调用输出失败次数" - } - } +root_rule_engine.label: +"""Rule engine configuration""" - metrics_actions_failed_out_of_service { - desc { - en: "How much times the rule failed to call actions due to the action is out of service. For example, a bridge is disabled or stopped." - zh: "由于输出停止服务而导致规则调用输出失败的次数。 例如，桥接被禁用或停止。" - } - label: { - en: "Fail Action" - zh: "调用输出失败次数" - } - } +event_sockname.desc: +"""The IP Address and Port of the Local Listener""" - metrics_actions_failed_unknown { - desc { - en: "How much times the rule failed to call actions due to to an unknown error." - zh: "由于未知错误，规则调用输出失败的次数。" - } - label: { - en: "Fail Action" - zh: "调用输出失败次数" - } - } +event_sockname.label: +"""IP Address And Port""" - test_context { - desc { - en: "The context of the event for testing" - zh: "测试事件的上下文" - } - label: { - en: "Event Conetxt" - zh: "事件上下文" - } - } +event_qos.desc: +"""The Message QoS""" - test_sql { - desc { - en: "The SQL of the rule for testing" - zh: "测试的 SQL" - } - label: { - en: "Test SQL" - zh: "测试 SQL" - } - } +event_qos.label: +"""Message QoS""" - rs_event { - desc { - en: "The event topics" - zh: "事件主题" - } - label: { - en: "Event Topics" - zh: "事件主题" - } - } +event_mountpoint.desc: +"""The Mountpoint""" - rs_title { - desc { - en: "The title" - zh: "标题" - } - label: { - en: "Title" - zh: "标题" - } - } +event_mountpoint.label: +"""Mountpoint""" - rs_description { - desc { - en: "The description" - zh: "描述" - } - label: { - en: "Description" - zh: "描述" - } - } +rs_title.desc: +"""The title""" - rs_columns { - desc { - en: "The columns" - zh: "列" - } - label: { - en: "Column" - zh: "列" - } - } +rs_title.label: +"""Title""" - rs_test_columns { - desc { - en: "The test columns" - zh: "测试列" - } - label: { - en: "Test Columns" - zh: "测试列" - } - } +ri_metrics.desc: +"""The metrics of the rule""" - rs_sql_example { - desc { - en: "The sql_example" - zh: "SQL 例子" - } - label: { - en: "SQL Example" - zh: "SQL 例子" - } - } +ri_metrics.label: +"""Rule Metrics""" - ri_metrics { - desc { - en: "The metrics of the rule" - zh: "规则的计数器" - } - label: { - en: "Rule Metrics" - zh: "规则计数器" - } - } +event_retain.desc: +"""If is a retain message""" - ri_node_metrics { - desc { - en: "The metrics of the rule for each node" - zh: "每个节点的规则计数器" - } - label: { - en: "Each Node Rule Metrics" - zh: "每个节点规则计数器" - } - } +event_retain.label: +"""Retain Message""" - ri_from { - desc { - en: "The topics of the rule" - zh: "规则指定的主题" - } - label: { - en: "Topics of Rule" - zh: "规则指定的主题" - } - } +event_event_type.desc: +"""Event Type""" - ri_created_at { - desc { - en: "The created time of the rule" - zh: "规则创建时间" - } - label: { - en: "Rule Create Time" - zh: "规则创建时间" - } - } +event_event_type.label: +"""Event Type""" - root_rule_engine { - desc { - en: "Rule engine configurations. This API can be used to change EMQX rule engine settings. But not for the rules. To list, create, or update rules, call the '/rules' API instead." - zh: "规则引擎配置。该 API 可用于查看和修改规则引擎相关的一些设置。但不可用于规则，如需查看或修改规则，请调用 '/rules' API 进行操作。" - } - label: { - en: "Rule engine configuration" - zh: "规则引擎配置" - } - } +event_expiry_interval.desc: +"""Expiry Interval""" - root_rule_creation { - desc { - en: "Schema for creating rules" - zh: "用于创建规则的 Schema" - } - label: { - en: "Create Schema" - zh: "用于创建规则的 Schema" - } - } +event_expiry_interval.label: +"""Expiry Interval""" - root_rule_info { - desc { - en: "Schema for rule info" - zh: "用于规则信息的 Schema" - } - label: { - en: "Info Schema" - zh: "用于规则信息的 Schema" - } - } +metrics_sql_matched.desc: +"""How much times the FROM clause of the SQL is matched.""" - root_rule_events { - desc { - en: "Schema for rule events" - zh: "用于事件的 Schema" - } - label: { - en: "Rule Events Schema" - zh: "用于规则事件的 Schema" - } - } +metrics_sql_matched.label: +"""Matched""" - root_rule_test { - desc { - en: "Schema for testing rules" - zh: "用于规则测试的 Schema" - } - label: { - en: "Rule Test Schema" - zh: "用于规则测试的 Schema" - } - } +event_clientid.desc: +"""The Client ID""" + +event_clientid.label: +"""Client ID""" + +metrics_actions_success.desc: +"""How much times the rule success to call the actions.""" + +metrics_actions_success.label: +"""Success Action""" + +metrics_actions_failed.desc: +"""How much times the rule failed to call the actions.""" + +metrics_actions_failed.label: +"""Failed Action""" + +metrics_sql_matched_rate.desc: +"""The rate of matched, times/second""" + +metrics_sql_matched_rate.label: +"""命中速率""" + +event_proto_ver.desc: +"""Protocol Version""" + +event_proto_ver.label: +"""Protocol Version""" + +event_publish_received_at.desc: +"""The Time that this Message is Received""" + +event_publish_received_at.label: +"""Message Received Time""" + +metrics_sql_matched_rate_last5m.desc: +"""The average rate of matched in last 5 minutes, times/second""" + +metrics_sql_matched_rate_last5m.label: +"""Average Matched Rate""" + +event_is_bridge.desc: +"""Is Bridge""" + +event_is_bridge.label: +"""Is Bridge""" + +event_authz_source.desc: +"""Cache, Plugs or Default""" + +event_authz_source.label: +"""Auth Source""" + +metrics_sql_failed_unknown.desc: +"""How much times the SQL is failed due to an unknown error.""" + +metrics_sql_failed_unknown.label: +"""SQL Unknown Error""" + +metrics_sql_failed.desc: +"""How much times the SQL is failed""" + +metrics_sql_failed.label: +"""SQL Failed""" + +event_ctx_dropped.desc: +"""The Reason for Dropping""" + +event_ctx_dropped.label: +"""Dropped Reason""" + +root_rule_test.desc: +"""Schema for testing rules""" + +root_rule_test.label: +"""Rule Test Schema""" + +rs_test_columns.desc: +"""The test columns""" + +rs_test_columns.label: +"""Test Columns""" + +event_peerhost.desc: +"""The IP Address of the Peer Client""" + +event_peerhost.label: +"""Peer IP Address""" + +event_proto_name.desc: +"""Protocol Name""" + +event_proto_name.label: +"""Protocol Name""" + +root_rule_events.desc: +"""Schema for rule events""" + +root_rule_events.label: +"""Rule Events Schema""" + +rs_sql_example.desc: +"""The sql_example""" + +rs_sql_example.label: +"""SQL Example""" + +metrics_sql_matched_rate_max.desc: +"""The max rate of matched, times/second""" + +metrics_sql_matched_rate_max.label: +"""Max Matched Rate""" + +event_clean_start.desc: +"""Clean Start""" + +event_clean_start.label: +"""Clean Start""" + +ri_created_at.desc: +"""The created time of the rule""" + +ri_created_at.label: +"""Rule Create Time""" + +event_dup.desc: +"""The DUP flag of the MQTT message""" + +event_dup.label: +"""DUP Flag""" + +ri_from.desc: +"""The topics of the rule""" + +ri_from.label: +"""Topics of Rule""" + +ri_node_metrics.desc: +"""The metrics of the rule for each node""" + +ri_node_metrics.label: +"""Each Node Rule Metrics""" + +root_rule_creation.desc: +"""Schema for creating rules""" + +root_rule_creation.label: +"""Create Schema""" + +event_result.desc: +"""Allow or Deny""" + +event_result.label: +"""Auth Result""" + +event_id.desc: +"""Message ID""" + +event_id.label: +"""Message ID""" + +event_username.desc: +"""Username""" + +event_username.label: +"""Username""" + +root_rule_info.desc: +"""Schema for rule info""" + +root_rule_info.label: +"""Info Schema""" + +rs_columns.desc: +"""The columns""" + +rs_columns.label: +"""Column""" + +test_sql.desc: +"""The SQL of the rule for testing""" + +test_sql.label: +"""Test SQL""" + +event_ctx_disconnected_reason.desc: +"""The Reason for Disconnect""" + +event_ctx_disconnected_reason.label: +"""Disconnect Reason""" } diff --git a/rel/i18n/emqx_rule_engine_api.hocon b/rel/i18n/emqx_rule_engine_api.hocon index 8a57f8e31..385b71ddc 100644 --- a/rel/i18n/emqx_rule_engine_api.hocon +++ b/rel/i18n/emqx_rule_engine_api.hocon @@ -1,168 +1,93 @@ emqx_rule_engine_api { - api1 { - desc { - en: "List all rules" - zh: "列出所有规则" - } - label: { - en: "List All Rules" - zh: "列出所有规则" - } - } - api1_enable { - desc { - en: "Filter enable/disable rules" - zh: "根据规则是否开启条件过滤" - } - } +api1.desc: +"""List all rules""" - api1_from { - desc { - en: "Filter rules by from(topic), exact match" - zh: "根据规则来源 Topic 过滤, 需要完全匹配" - } - } +api1.label: +"""List All Rules""" - api1_like_id { - desc { - en: "Filter rules by id, Substring matching" - zh: "根据规则 id 过滤, 使用子串模糊匹配" - } - } +api10.desc: +"""Update rule engine configuration.""" - api1_like_from { - desc { - en: "Filter rules by from(topic), Substring matching" - zh: "根据规则来源 Topic 过滤, 使用子串模糊匹配" - } - } +api10.label: +"""Update configuration""" - api1_like_description { - desc { - en: "Filter rules by description, Substring matching" - zh: "根据规则描述过滤, 使用子串模糊匹配" - } - } - api1_match_from { - desc { - en: "Filter rules by from(topic), Mqtt topic matching" - zh: "根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配" - } - } - api1_resp { - desc { - en: "List of rules" - zh: "规则列表" - } - label: { - en: "List Rules" - zh: "列出所有规则" - } - } - api2 { - desc { - en: "Create a new rule using given Id" - zh: "通过指定 ID 创建规则" - } - label: { - en: "Create Rule By ID" - zh: "通过指定 ID 创建规则" - } - } +api1_enable.desc: +"""Filter enable/disable rules""" - api3 { - desc { - en: "List all events can be used in rules" - zh: "列出所有能被规则使用的事件" - } - label: { - en: "List All Events Can Be Used In Rule" - zh: "列出所有能被规则使用的事件" - } - } +api1_from.desc: +"""Filter rules by from(topic), exact match""" - api4 { - desc { - en: "Get a rule by given Id" - zh: "通过 ID 查询规则" - } - label: { - en: "Get Rule" - zh: "查询规则" - } - } +api1_like_description.desc: +"""Filter rules by description, Substring matching""" - api4_1 { - desc { - en: "Get a rule's metrics by given Id" - zh: "通过给定的 Id 获得规则的指标数据" - } - label: { - en: "Get Metric" - zh: "获得指标数据" - } - } +api1_like_from.desc: +"""Filter rules by from(topic), Substring matching""" - api5 { - desc { - en: "Update a rule by given Id to all nodes in the cluster" - zh: "通过 ID 更新集群里所有节点上的规则" - } - label: { - en: "Update Cluster Rule" - zh: "更新集群规则" - } - } +api1_like_id.desc: +"""Filter rules by id, Substring matching""" + +api1_match_from.desc: +"""Filter rules by from(topic), Mqtt topic matching""" + +api1_resp.desc: +"""List of rules""" + +api1_resp.label: +"""List Rules""" + +api2.desc: +"""Create a new rule using given Id""" + +api2.label: +"""Create Rule By ID""" + +api3.desc: +"""List all events can be used in rules""" + +api3.label: +"""List All Events Can Be Used In Rule""" + +api4.desc: +"""Get a rule by given Id""" + +api4.label: +"""Get Rule""" + +api4_1.desc: +"""Get a rule's metrics by given Id""" + +api4_1.label: +"""Get Metric""" + +api5.desc: +"""Update a rule by given Id to all nodes in the cluster""" + +api5.label: +"""Update Cluster Rule""" + +api6.desc: +"""Delete a rule by given Id from all nodes in the cluster""" + +api6.label: +"""Delete Cluster Rule""" + +api7.desc: +"""Reset a rule metrics""" + +api7.label: +"""Reset Rule Metrics""" + +api8.desc: +"""Test a rule""" + +api8.label: +"""Test Rule""" + +api9.desc: +"""Get rule engine configuration.""" + +api9.label: +"""Get configuration""" - api6 { - desc { - en: "Delete a rule by given Id from all nodes in the cluster" - zh: "通过 ID 删除集群里所有节点上的规则" - } - label: { - en: "Delete Cluster Rule" - zh: "基于给定 ID 新建一条规则" - } - } - api7 { - desc { - en: "Reset a rule metrics" - zh: "重置规则计数" - } - label: { - en: "Reset Rule Metrics" - zh: "重置规则计数" - } - } - api8 { - desc { - en: "Test a rule" - zh: "测试一个规则" - } - label: { - en: "Test Rule" - zh: "测试规则" - } - } - api9 { - desc { - en: "Get rule engine configuration." - zh: "获取规则引擎配置。" - } - label { - en: "Get configuration" - zh: "获取配置" - } - } - api10 { - desc { - en: "Update rule engine configuration." - zh: "更新规则引擎配置。" - } - label { - en: "Update configuration" - zh: "更新配置" - } - } } diff --git a/rel/i18n/emqx_rule_engine_schema.hocon b/rel/i18n/emqx_rule_engine_schema.hocon index ca0a73f0f..9b1f1f802 100644 --- a/rel/i18n/emqx_rule_engine_schema.hocon +++ b/rel/i18n/emqx_rule_engine_schema.hocon @@ -1,326 +1,25 @@ emqx_rule_engine_schema { - rules_name { - desc { - en: "The name of the rule" - zh: "规则名字" - } - label: { - en: "Rule Name" - zh: "规则名字" - } - } +console_function.desc: +"""Print the actions to the console""" - rules_sql { - desc { - en: """SQL query to transform the messages. -Example: SELECT * FROM "test/topic" WHERE payload.x = 1""" - zh: """用于处理消息的 SQL 。 -示例：SELECT * FROM "test/topic" WHERE payload.x = 1""" - } - label: { - en: "Rule SQL" - zh: "规则 SQL" - } - } +console_function.label: +"""Console Function""" - rules_actions { - desc { - en: """A list of actions of the rule. -An action can be a string that refers to the channel ID of an EMQX bridge, or an object -that refers to a function. -There a some built-in functions like "republish" and "console", and we also support user -provided functions in the format: "{module}:{function}". -The actions in the list are executed sequentially. -This means that if one of the action is executing slowly, all the following actions will not -be executed until it returns. -If one of the action crashed, all other actions come after it will still be executed, in the -original order. -If there's any error when running an action, there will be an error message, and the 'failure' -counter of the function action or the bridge channel will increase.""" +desc_builtin_action_console.desc: +"""Configuration for a built-in action.""" - zh: """规则的动作列表。 -动作可以是指向 EMQX bridge 的引用，也可以是一个指向函数的对象。 -我们支持一些内置函数，如“republish”和“console”，我们还支持用户提供的函数，它的格式为：“{module}:{function}”。 -列表中的动作按顺序执行。这意味着如果其中一个动作执行缓慢，则以下所有动作都不会被执行直到它返回。 -如果其中一个动作崩溃，在它之后的所有动作仍然会被按照原始顺序执行。 -如果运行动作时出现任何错误，则会出现错误消息，并且相应的计数器会增加。""" - } - label: { - en: "Rule Action List" - zh: "动作列表" - } - } +desc_builtin_action_console.label: +"""Action Console Configuration""" - rules_enable { - desc { - en: "Enable or disable the rule" - zh: "启用或禁用规则引擎" - } - label: { - en: "Enable Or Disable Rule" - zh: "启用或禁用规则引擎" - } - } +desc_builtin_action_republish.desc: +"""Configuration for a built-in action.""" - rules_metadata { - desc { - en: "Rule metadata, do not change manually" - zh: "规则的元数据，不要手动修改" - } - label: { - en: "Rule metadata" - zh: "规则的元数据" - } - } +desc_builtin_action_republish.label: +"""Republish Configuration""" - rules_description { - desc { - en: "The description of the rule" - zh: "规则的描述" - } - label: { - en: "Rule Description" - zh: "规则描述" - } - } - - republish_function { - desc { - en: """Republish the message as a new MQTT message""" - zh: """将消息重新发布为新的 MQTT 消息""" - } - label: { - en: "Republish Function" - zh: "重新发布函数" - } - } - - console_function { - desc { - en: """Print the actions to the console""" - zh: "将输出打印到控制台" - } - label: { - en: "Console Function" - zh: "控制台函数" - } - } - - user_provided_function_function { - desc { - en: """The user provided function. Should be in the format: '{module}:{function}'. -Where {module} is the Erlang callback module and {function} is the Erlang function. - -To write your own function, checkout the function console and -republish in the source file: -apps/emqx_rule_engine/src/emqx_rule_actions.erl as an example.""" - - zh: """用户提供的函数。 格式应为：'{module}:{function}'。 -其中 {module} 是 Erlang 回调模块， {function} 是 Erlang 函数。 -要编写自己的函数，请检查源文件：apps/emqx_rule_engine/src/emqx_rule_actions.erl 中的示例函数 console 和republish 。""" - } - label: { - en: "User Provided Function" - zh: "用户提供的函数" - } - } - - user_provided_function_args { - desc { - en: """The args will be passed as the 3rd argument to module:function/3, -checkout the function console and republish in the source file: -apps/emqx_rule_engine/src/emqx_rule_actions.erl as an example.""" - zh: """用户提供的参数将作为函数 module:function/3 的第三个参数， -请检查源文件：apps/emqx_rule_engine/src/emqx_rule_actions.erl 中的示例函数 console 和republish 。""" - } - label: { - en: "User Provided Function Args" - zh: "用户提供函数的参数" - } - } - - republish_args_topic { - desc { - en: """The target topic of message to be re-published. -Template with variables is allowed, see description of the 'republish_args'.""" - zh: """重新发布消息的目标主题。 -允许使用带有变量的模板，请参阅“republish_args”的描述。""" - } - label: { - en: "Target Topic" - zh: "目标主题" - } - } - - republish_args_qos { - desc { - en: """The qos of the message to be re-published. -Template with variables is allowed, see description of the 'republish_args'. -Defaults to ${qos}. If variable ${qos} is not found from the selected result of the rule, -0 is used.""" - zh: """要重新发布的消息的 qos。允许使用带有变量的模板，请参阅“republish_args”的描述。 -默认为 ${qos}。 如果从规则的选择结果中没有找到变量 ${qos}，则使用 0。""" - } - label: { - en: "Message QoS" - zh: "消息 QoS 等级" - } - } - - republish_args_retain { - desc { - en: """The 'retain' flag of the message to be re-published. -Template with variables is allowed, see description of the 'republish_args'. -Defaults to ${retain}. If variable ${retain} is not found from the selected result -of the rule, false is used.""" - zh: """要重新发布的消息的“保留”标志。允许使用带有变量的模板，请参阅“republish_args”的描述。 -默认为 ${retain}。 如果从所选结果中未找到变量 ${retain}，则使用 false。""" - } - label: { - en: "Retain Flag" - zh: "保留消息标志" - } - } - - republish_args_payload { - desc { - en: """The payload of the message to be re-published. -Template with variables is allowed, see description of the 'republish_args'. -Defaults to ${payload}. If variable ${payload} is not found from the selected result -of the rule, then the string "undefined" is used.""" - zh: """要重新发布的消息的有效负载。允许使用带有变量的模板，请参阅“republish_args”的描述。 -默认为 ${payload}。 如果从所选结果中未找到变量 ${payload}，则使用字符串 "undefined"。""" - } - label: { - en: "Message Payload" - zh: "消息负载" - } - } - republish_args_user_properties { - desc { - en: """From which variable should the MQTT message's User-Property pairs be taken from. -The value must be a map. -You may configure it to ${pub_props.'User-Property'} or -use SELECT *,pub_props.'User-Property' as user_properties -to forward the original user properties to the republished message. -You may also call map_put function like -map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties -to inject user properties. -NOTE: MQTT spec allows duplicated user property names, but EMQX Rule-Engine does not.""" - - zh: """指定使用哪个变量来填充 MQTT 消息的 User-Property 列表。这个变量的值必须是一个 map 类型。 -可以设置成 ${pub_props.'User-Property'} 或者 -使用 SELECT *,pub_props.'User-Property' as user_properties 来把源 MQTT 消息 -的 User-Property 列表用于填充。 -也可以使用 map_put 函数来添加新的 User-Property， -map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties -注意：MQTT 协议允许一个消息中出现多次同一个 property 名，但是 EMQX 的规则引擎不允许。""" - } - } - - rule_engine_ignore_sys_message { - desc { - en: "When set to 'true' (default), rule-engine will ignore messages published to $SYS topics." - zh: "当设置为“true”（默认）时，规则引擎将忽略发布到 $SYS 主题的消息。" - } - label: { - en: "Ignore Sys Message" - zh: "忽略系统消息" - } - } - - rule_engine_rules { - desc { - en: """The rules""" - zh: "规则" - } - label: { - en: "Rules" - zh: "规则" - } - } - - rule_engine_jq_function_default_timeout { - desc { - en: "Default timeout for the `jq` rule engine function" - zh: "规则引擎内建函数 `jq` 默认时间限制" - } - label: { - en: "Rule engine jq function default timeout" - zh: "规则引擎 jq 函数时间限制" - } - } - - rule_engine_jq_implementation_module { - desc { - en: "The implementation module for the jq rule engine function. The two options are jq_nif and jq_port. With the jq_nif option an Erlang NIF library is used while with the jq_port option an implementation based on Erlang port programs is used. The jq_nif option (the default option) is the fastest implementation of the two but jq_port is safer as the jq programs will not execute in the same process as the Erlang VM." - zh: "jq 规则引擎功能的实现模块。可用的两个选项是 jq_nif 和 jq_port。jq_nif 使用 Erlang NIF 库访问 jq 库，而 jq_port 使用基于 Erlang Port 的实现。jq_nif 方式（默认选项）是这两个选项中最快的实现，但 jq_port 方式更安全，因为这种情况下 jq 程序不会在 Erlang VM 进程中执行。" - } - label: { - en: "JQ Implementation Module" - zh: "JQ 实现模块" - } - } - - desc_rule_engine { - desc { - en: """Configuration for the EMQX Rule Engine.""" - zh: """配置 EMQX 规则引擎。""" - } - label: { - en: "Rule Engine Configuration" - zh: "配置规则引擎" - } - } - - desc_rules { - desc { - en: """Configuration for a rule.""" - zh: """配置规则""" - } - label: { - en: "Rule Configuration" - zh: "配置规则" - } - } - - desc_builtin_action_republish { - desc { - en: """Configuration for a built-in action.""" - zh: """配置重新发布。""" - } - label: { - en: "Republish Configuration" - zh: "配置重新发布" - } - } - - desc_builtin_action_console { - desc { - en: """Configuration for a built-in action.""" - zh: """配置打印到控制台""" - } - label: { - en: "Action Console Configuration" - zh: "配置打印到控制台" - } - } - - desc_user_provided_function { - desc { - en: """Configuration for a built-in action.""" - zh: """配置用户函数""" - } - label: { - en: "User Provid Function Configuration" - zh: "配置用户函数" - } - } - - desc_republish_args { - desc { - en: """The arguments of the built-in 'republish' action.One can use variables in the args. +desc_republish_args.desc: +"""The arguments of the built-in 'republish' action.One can use variables in the args. The variables are selected by the rule. For example, if the rule SQL is defined as following: SELECT clientid, qos, payload FROM "t/1" @@ -337,27 +36,168 @@ Then there are 3 variables available: clientid, qos an When the rule is triggered by an MQTT message with payload = `hello`, qos = 1, clientid = `Steve`, the rule will republish a new MQTT message to topic `t/Steve`, payload = `msg: hello`, and `qos = 1`.""" - zh: """内置 'republish' 动作的参数。 -可以在参数中使用变量。 -变量是规则中选择的字段。 例如规则 SQL 定义如下： - - SELECT clientid, qos, payload FROM "t/1" - -然后有 3 个变量可用：clientid、qos 和 payload。 如果我们将参数设置为： - - { - topic = "t/${clientid}" - qos = "${qos}" - payload = "msg: ${payload}" - } - -当收到一条消息 payload = `hello`, qos = 1, clientid = `Steve` 时，将重新发布一条新的 MQTT 消息到主题 `t/Steve` -消息内容为 payload = `msg: hello`, and `qos = 1""" - } - label: { - en: "Republish Args" - zh: "重新发布参数" - } - } + +desc_republish_args.label: +"""Republish Args""" + +desc_rule_engine.desc: +"""Configuration for the EMQX Rule Engine.""" + +desc_rule_engine.label: +"""Rule Engine Configuration""" + +desc_rules.desc: +"""Configuration for a rule.""" + +desc_rules.label: +"""Rule Configuration""" + +desc_user_provided_function.desc: +"""Configuration for a built-in action.""" + +desc_user_provided_function.label: +"""User Provid Function Configuration""" + +republish_args_payload.desc: +"""The payload of the message to be re-published. +Template with variables is allowed, see description of the 'republish_args'. +Defaults to ${payload}. If variable ${payload} is not found from the selected result +of the rule, then the string "undefined" is used.""" + +republish_args_payload.label: +"""Message Payload""" + +republish_args_qos.desc: +"""The qos of the message to be re-published. +Template with variables is allowed, see description of the 'republish_args'. +Defaults to ${qos}. If variable ${qos} is not found from the selected result of the rule, +0 is used.""" + +republish_args_qos.label: +"""Message QoS""" + +republish_args_retain.desc: +"""The 'retain' flag of the message to be re-published. +Template with variables is allowed, see description of the 'republish_args'. +Defaults to ${retain}. If variable ${retain} is not found from the selected result +of the rule, false is used.""" + +republish_args_retain.label: +"""Retain Flag""" + +republish_args_topic.desc: +"""The target topic of message to be re-published. +Template with variables is allowed, see description of the 'republish_args'.""" + +republish_args_topic.label: +"""Target Topic""" + +republish_args_user_properties.desc: +"""From which variable should the MQTT message's User-Property pairs be taken from. +The value must be a map. +You may configure it to ${pub_props.'User-Property'} or +use SELECT *,pub_props.'User-Property' as user_properties +to forward the original user properties to the republished message. +You may also call map_put function like +map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties +to inject user properties. +NOTE: MQTT spec allows duplicated user property names, but EMQX Rule-Engine does not.""" + +republish_function.desc: +"""Republish the message as a new MQTT message""" + +republish_function.label: +"""Republish Function""" + +rule_engine_ignore_sys_message.desc: +"""When set to 'true' (default), rule-engine will ignore messages published to $SYS topics.""" + +rule_engine_ignore_sys_message.label: +"""Ignore Sys Message""" + +rule_engine_jq_function_default_timeout.desc: +"""Default timeout for the `jq` rule engine function""" + +rule_engine_jq_function_default_timeout.label: +"""Rule engine jq function default timeout""" + +rule_engine_jq_implementation_module.desc: +"""The implementation module for the jq rule engine function. The two options are jq_nif and jq_port. With the jq_nif option an Erlang NIF library is used while with the jq_port option an implementation based on Erlang port programs is used. The jq_nif option (the default option) is the fastest implementation of the two but jq_port is safer as the jq programs will not execute in the same process as the Erlang VM.""" + +rule_engine_jq_implementation_module.label: +"""JQ Implementation Module""" + +rule_engine_rules.desc: +"""The rules""" + +rule_engine_rules.label: +"""Rules""" + +rules_actions.desc: +"""A list of actions of the rule. +An action can be a string that refers to the channel ID of an EMQX bridge, or an object +that refers to a function. +There a some built-in functions like "republish" and "console", and we also support user +provided functions in the format: "{module}:{function}". +The actions in the list are executed sequentially. +This means that if one of the action is executing slowly, all the following actions will not +be executed until it returns. +If one of the action crashed, all other actions come after it will still be executed, in the +original order. +If there's any error when running an action, there will be an error message, and the 'failure' +counter of the function action or the bridge channel will increase.""" + +rules_actions.label: +"""Rule Action List""" + +rules_description.desc: +"""The description of the rule""" + +rules_description.label: +"""Rule Description""" + +rules_enable.desc: +"""Enable or disable the rule""" + +rules_enable.label: +"""Enable Or Disable Rule""" + +rules_metadata.desc: +"""Rule metadata, do not change manually""" + +rules_metadata.label: +"""Rule metadata""" + +rules_name.desc: +"""The name of the rule""" + +rules_name.label: +"""Rule Name""" + +rules_sql.desc: +"""SQL query to transform the messages. +Example: SELECT * FROM "test/topic" WHERE payload.x = 1""" + +rules_sql.label: +"""Rule SQL""" + +user_provided_function_args.desc: +"""The args will be passed as the 3rd argument to module:function/3, +checkout the function console and republish in the source file: +apps/emqx_rule_engine/src/emqx_rule_actions.erl as an example.""" + +user_provided_function_args.label: +"""User Provided Function Args""" + +user_provided_function_function.desc: +"""The user provided function. Should be in the format: '{module}:{function}'. +Where {module} is the Erlang callback module and {function} is the Erlang function. + +To write your own function, checkout the function console and +republish in the source file: +apps/emqx_rule_engine/src/emqx_rule_actions.erl as an example.""" + +user_provided_function_function.label: +"""User Provided Function""" } diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index d36809c3b..9c2a1530d 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -1,1582 +1,441 @@ emqx_schema { - force_shutdown_enable { - desc { - en: "Enable `force_shutdown` feature." - zh: "启用 `force_shutdown` 功能。" - } - label { - en: "Enable `force_shutdown` feature" - zh: "启用 `force_shutdown` 功能" - } - } +fields_mqtt_quic_listener_peer_unidi_stream_count.desc: +"""Number of unidirectional streams to allow the peer to open.""" - force_shutdown_max_message_queue_len { - desc { - en: "Maximum message queue length." - zh: "消息队列的最大长度。" - } - label { - en: "Maximum mailbox queue length of process." - zh: "进程邮箱消息队列的最大长度" - } - } +fields_mqtt_quic_listener_peer_unidi_stream_count.label: +"""Peer unidi stream count""" - force_shutdown_max_heap_size { - desc { - en: "Total heap size" - zh: "Heap 的总大小。" - } - label { - en: "Total heap size" - zh: "Heap 的总大小" - } - } +fields_authorization_no_match.desc: +"""Default access control action if the user or client matches no ACL rules, +or if no such user or client is found by the configurable authorization +sources such as built_in_database, an HTTP API, or a query against PostgreSQL. +Find more details in 'authorization.sources' config.""" - overload_protection_enable { - desc { - en: "React on system overload or not." - zh: "是否对系统过载做出反应。" - } - label { - en: "React on system overload or not" - zh: "是否对系统过载做出反应" - } - } +fields_authorization_no_match.label: +"""Authorization no match""" - overload_protection_backoff_delay { - desc { - en: "The maximum duration of delay for background task execution during high load conditions." - zh: "高负载时，一些不重要的任务可能会延迟执行，在这里设置允许延迟的时间。" - } - label { - en: "Delay Time" - zh: "延迟时间" - } - } +sysmon_top_db_hostname.desc: +"""Hostname of the PostgreSQL database that collects the data points""" - overload_protection_backoff_gc { - desc { - en: "When at high load, skip forceful GC." - zh: "高负载时，跳过强制 GC。" - } - label { - en: "Skip GC" - zh: "跳过GC" - } - } +sysmon_top_db_hostname.label: +"""DB Hostname""" - overload_protection_backoff_hibernation { - desc { - en: "When at high load, skip process hibernation." - zh: "高负载时，跳过进程休眠。" - } - label { - en: "Skip hibernation" - zh: "跳过休眠" - } - } - - overload_protection_backoff_new_conn { - desc { - en: "When at high load, close new incoming connections." - zh: "高负载时，拒绝新进来的客户端连接。" - } - label { - en: "Close new connections" - zh: "关闭新连接" - } - } - - conn_congestion_enable_alarm { - desc { - en: "Enable or disable connection congestion alarm." - zh: "启用或者禁用连接阻塞告警功能。" - } - label { - en: "Enable/disable congestion alarm" - zh: "启用/禁用阻塞告警" - } - } - - conn_congestion_min_alarm_sustain_duration { - desc { - en: "Minimal time before clearing the alarm.
" - "The alarm is cleared only when there's no pending data in
" - "the queue, and at least min_alarm_sustain_duration" - "milliseconds passed since the last time we considered the connection 'congested'.
" - "This is to avoid clearing and raising the alarm again too often." - zh: "清除警报前的最短时间。
" - "只有当队列中没有挂起的数据，并且连接至少被堵塞了 min_alarm_sustain_duration 毫秒时，
" - "报警才会被清除。这是为了避免太频繁地清除和再次发出警报。" - } - label { - en: "Sustain duration" - zh: "告警维持时间" - } - } - - force_gc_enable { - desc { - en: "Enable forced garbage collection." - zh: "启用强制垃圾回收。" - } - label { - en: "Enable forced garbage collection" - zh: "启用强制垃圾回收" - } - } - - force_gc_count { - desc { - en: "GC the process after this many received messages." - zh: "在进程收到多少消息之后，对此进程执行垃圾回收。" - } - label { - en: "Process GC messages num" - zh: "垃圾回收消息数" - } - } - - force_gc_bytes { - desc { - en: "GC the process after specified number of bytes have passed through." - zh: "在进程处理过多少个字节之后，对此进程执行垃圾回收。" - } - label { - en: "Process GC bytes" - zh: "垃圾回收字节数" - } - } - - sysmon_vm_process_check_interval { - desc { - en: "The time interval for the periodic process limit check." - zh: "定期进程限制检查的时间间隔。" - } - label { - en: "Process limit check interval" - zh: "进程限制检查时间" - } - } - - sysmon_vm_process_high_watermark { - desc { - en: "The threshold, as percentage of processes, for how many\n" - " processes can simultaneously exist at the local node before the corresponding\n" - " alarm is raised." - zh: "在发出相应警报之前，本地节点上可以同时存在多少进程的阈值（以进程百分比表示）。" - } - label { - en: "Process high watermark" - zh: "进程数高水位线" - } - } - - sysmon_vm_process_low_watermark { - desc { - en: "The threshold, as percentage of processes, for how many\n" - " processes can simultaneously exist at the local node before the corresponding\n" - " alarm is cleared." - zh: "在清除相应警报之前，本地节点上可以同时存在多少进程的阈值（以进程百分比表示）。" - } - label { - en: "Process low watermark" - zh: "进程数低水位线" - } - } - - sysmon_vm_long_gc { - desc { - en: """When an Erlang process spends long time to perform garbage collection, a warning level long_gc log is emitted, -and an MQTT message is published to the system topic $SYS/sysmon/long_gc.""" - zh: """当系统检测到某个 Erlang 进程垃圾回收占用过长时间，会触发一条带有 long_gc 关键字的日志。 -同时还会发布一条主题为 $SYS/sysmon/long_gc 的 MQTT 系统消息。""" - } - label { - en: "Enable Long GC monitoring." - zh: "启用长垃圾回收监控" - } - } - - sysmon_vm_long_schedule { - desc { - en: """When the Erlang VM detect a task scheduled for too long, a warning level 'long_schedule' log is emitted, -and an MQTT message is published to the system topic $SYS/sysmon/long_schedule.""" - zh: """启用后，如果 Erlang VM 调度器出现某个任务占用时间过长时，会触发一条带有 'long_schedule' 关键字的日志。 -同时还会发布一条主题为 $SYS/sysmon/long_schedule 的 MQTT 系统消息。""" - } - label { - en: "Enable Long Schedule monitoring." - zh: "启用长调度监控" - } - } - - sysmon_vm_large_heap { - desc { - en: """When an Erlang process consumed a large amount of memory for its heap space, -the system will write a warning level large_heap log, and an MQTT message is published to -the system topic $SYS/sysmon/large_heap.""" - zh: """启用后，当一个 Erlang 进程申请了大量内存，系统会触发一条带有 large_heap 关键字的 -warning 级别日志。同时还会发布一条主题为 $SYS/sysmon/busy_dist_port 的 MQTT 系统消息。""" - } - label { - en: "Enable Large Heap monitoring." - zh: "启用大 heap 监控" - } - } - - sysmon_vm_busy_dist_port { - desc { - en: """When the RPC connection used to communicate with other nodes in the cluster is overloaded, -there will be a busy_dist_port warning log, -and an MQTT message is published to system topic $SYS/sysmon/busy_dist_port.""" - zh: """启用后，当用于集群接点之间 RPC 的连接过忙时，会触发一条带有 busy_dist_port 关键字的 warning 级别日志。 -同时还会发布一条主题为 $SYS/sysmon/busy_dist_port 的 MQTT 系统消息。""" - } - label { - en: "Enable Busy Distribution Port monitoring." - zh: "启用分布式端口过忙监控" - } - } - - sysmon_vm_busy_port { - desc { - en: """When a port (e.g. TCP socket) is overloaded, there will be a busy_port warning log, -and an MQTT message is published to the system topic $SYS/sysmon/busy_port.""" - zh: """当一个系统接口（例如 TCP socket）过忙，会触发一条带有 busy_port 关键字的 warning 级别的日志。 -同时还会发布一条主题为 $SYS/sysmon/busy_port 的 MQTT 系统消息。""" - } - label { - en: "Enable Busy Port monitoring." - zh: "启用端口过忙监控" - } - } - - sysmon_os_cpu_check_interval { - desc { - en: "The time interval for the periodic CPU check." - zh: "定期 CPU 检查的时间间隔。" - } - label { - en: "The time interval for the periodic CPU check." - zh: "定期 CPU 检查的时间间隔" - } - } - - sysmon_os_cpu_high_watermark { - desc { - en: "The threshold, as percentage of system CPU load,\n" - " for how much system cpu can be used before the corresponding alarm is raised." - zh: "在发出相应警报之前可以使用多少系统 CPU 的阈值，以系统CPU负载的百分比表示。" - } - label { - en: "CPU high watermark" - zh: "CPU 高水位线" - } - } - - sysmon_os_cpu_low_watermark { - desc { - en: "The threshold, as percentage of system CPU load,\n" - " for how much system cpu can be used before the corresponding alarm is cleared." - zh: "在解除相应警报之前可以使用多少系统 CPU 的阈值，以系统CPU负载的百分比表示。" - } - label { - en: "CPU low watermark" - zh: "CPU 低水位线" - } - } - - sysmon_os_mem_check_interval { - desc { - en: "The time interval for the periodic memory check." - zh: "定期内存检查的时间间隔。" - } - label { - en: "Mem check interval" - zh: "内存检查间隔" - } - } - - sysmon_os_sysmem_high_watermark { - desc { - en: "The threshold, as percentage of system memory,\n" - " for how much system memory can be allocated before the corresponding alarm is raised." - zh: "在发出相应报警之前可以分配多少系统内存的阈值，以系统内存的百分比表示。" - } - label { - en: "SysMem high wartermark" - zh: "系统内存高水位线" - } - } - - sysmon_os_procmem_high_watermark { - desc { - en: "The threshold, as percentage of system memory,\n" - " for how much system memory can be allocated by one Erlang process before\n" - " the corresponding alarm is raised." - zh: "在发出相应警报之前，一个Erlang进程可以分配多少系统内存的阈值，以系统内存的百分比表示。" - } - label { - en: "ProcMem high wartermark" - zh: "进程内存高水位线" - } - } - - sysmon_top_num_items { - desc { - en: "The number of top processes per monitoring group" - zh: "每个监视组的顶级进程数。" - } - label { - en: "Top num items" - zh: "顶级进程数" - } - } - - sysmon_top_sample_interval { - desc { - en: "Specifies how often process top should be collected" - zh: "指定应收集进程顶部的频率。" - } - label { - en: "Top sample interval" - zh: "取样时间" - } - } - - sysmon_top_max_procs { - desc { - en: "Stop collecting data when the number of processes\n" - "in the VM exceeds this value" - zh: "当 VM 中的进程数超过此值时，停止收集数据。" - } - label { - en: "Max procs" - zh: "最大进程数" - } - } - - sysmon_top_db_hostname { - desc { - en: "Hostname of the PostgreSQL database that collects the data points" - zh: "收集数据点的 PostgreSQL 数据库的主机名。" - } - label { - en: "DB Hostname" - zh: "数据库主机名" - } - } - - sysmon_top_db_port { - desc { - en: "Port of the PostgreSQL database that collects the data points." - zh: "收集数据点的 PostgreSQL 数据库的端口。" - } - label { - en: "DB Port" - zh: "数据库端口" - } - } - - sysmon_top_db_username { - desc { - en: "Username of the PostgreSQL database" - zh: "PostgreSQL 数据库的用户名" - } - label { - en: "DB Username" - zh: "数据库用户名" - } - } - - sysmon_top_db_password { - desc { - en: "EMQX user password in the PostgreSQL database" - zh: "PostgreSQL 数据库的密码" - } - label { - en: "DB Password" - zh: "数据库密码" - } - } - - sysmon_top_db_name { - desc { - en: "PostgreSQL database name" - zh: "PostgreSQL 数据库的数据库名" - } - label { - en: "DB Name" - zh: "数据库名" - } - } - - alarm_actions { - desc { - en: "The actions triggered when the alarm is activated.
" - "Currently, the following actions are supported: log and " - "publish.\n" - "log is to write the alarm to log (console or file).\n" - "publish is to publish the alarm as an MQTT message to " - "the system topics:\n" - "$SYS/brokers/emqx@xx.xx.xx.x/alarms/activate and\n" - "$SYS/brokers/emqx@xx.xx.xx.x/alarms/deactivate" - zh: "警报激活时触发的动作。
" - "目前，支持以下操作：log 和 " - "publish.\n" - "log 将告警写入日志 (控制台或者文件).\n" - "publish 将告警作为 MQTT 消息发布到系统主题:\n" - "$SYS/brokers/emqx@xx.xx.xx.x/alarms/activate and\n" - "$SYS/brokers/emqx@xx.xx.xx.x/alarms/deactivate" - } - label: { - en: "Alarm Actions" - zh: "告警动作" - } - } - - alarm_size_limit { - desc { - en: "The maximum total number of deactivated alarms to keep as history.
" - "When this limit is exceeded, the oldest deactivated alarms are " - "deleted to cap the total number." - zh: "要保留为历史记录的已停用报警的最大总数。当超过此限制时，将删除最旧的停用报警，以限制总数。" - } - label: { - en: "Alarm size limit" - zh: "告警总数限制" - } - } - - alarm_validity_period { - desc { - en: "Retention time of deactivated alarms. Alarms are not deleted immediately\n" - "when deactivated, but after the retention time." - zh: "停用报警的保留时间。报警在停用时不会立即删除，而是在保留时间之后删除。" - } - label: { - en: "Alarm validity period" - zh: "告警保留时间" - } - } - - flapping_detect_enable { - desc { - en: "Enable flapping connection detection feature." - zh: "启用抖动检测功能。" - } - label: { - en: "Enable flapping detection" - zh: "启用抖动检测" - } - } - - flapping_detect_max_count { - desc { - en: "The maximum number of disconnects allowed for a MQTT Client in `window_time`" - zh: "MQTT 客户端在“窗口”时间内允许的最大断开次数。" - } - label: { - en: "Max count" - zh: "最大断开次数" - } - } - - flapping_detect_window_time { - desc { - en: "The time window for flapping detection." - zh: "抖动检测的时间窗口。" - } - label: { - en: "Window time" - zh: "时间窗口" - } - } - - flapping_detect_ban_time { - desc { - en: "How long the flapping clientid will be banned." - zh: "抖动的客户端将会被禁止登录多长时间。" - } - label: { - en: "Ban time" - zh: "禁止登录时长" - } - } - - persistent_session_store_enabled { - desc { - en: "Use the database to store information about persistent sessions.\n" - "This makes it possible to migrate a client connection to another\n" - "cluster node if a node is stopped." - zh: "使用数据库存储有关持久会话的信息。\n" - "这使得在节点停止时，可以将客户端连接迁移到另一个群集节点。" - } - label: { - en: "Enable persistent session store" - zh: "启用持久会话保存" - } - } - - persistent_session_store_backend { - desc { - en: "Database management system used to store information about persistent sessions and messages.\n" - "- `builtin`: Use the embedded database (mria)" - zh: "用于存储持久性会话和信息的数据库管理后端\n" - "- `builtin`: 使用内置的数据库（mria）" - } - label: { - en: "Backend" - zh: "后端类型" - } - } - - persistent_store_on_disc { - desc { - en: "Save information about the persistent sessions on disc.\n" - "If this option is enabled, persistent sessions will survive full restart of the cluster.\n" - "Otherwise, all the data will be stored in RAM, and it will be lost when all the nodes in the cluster are stopped." - zh: "将持久会话数据保存在磁盘上。如果为 false 则存储在内存中。\n" - "如开启， 持久会话数据可在集群重启后恢复。\n" - "如关闭， 数据仅存储在内存中， 则在整个集群停止后丢失。" - } - label: { - en: "Persist on disc" - zh: "持久化在磁盘上" - } - } - - persistent_store_ram_cache { - desc { - en: "Maintain a copy of the data in RAM for faster access." - zh: "在内存中保持一份数据的副本，以便更快地访问。" - } - label: { - en: "RAM cache" - zh: "内存缓存" - } - } - - persistent_session_store_max_retain_undelivered { - desc { - en: "The time messages that was not delivered to a persistent session\n" - "is stored before being garbage collected if the node the previous\n" - "session was handled on restarts of is stopped." - zh: "如果重新启动时处理上一个会话的节点已停止，则未传递到持久会话的消息在垃圾收集之前会被存储。" - } - label: { - en: "Max retain undelivered" - zh: "未投递的消息保留条数" - } - } - - persistent_session_store_message_gc_interval { - desc { - en: "The starting interval for garbage collection of undelivered messages to\n" - "a persistent session. This affects how often the \"max_retain_undelivered\"\n" - "is checked for removal." - zh: "将未送达的消息垃圾收集到持久会话的开始间隔。\n" - "这会影响检查 \"max_retain_undelivered\"（最大保留未送达）的删除频率。" - } - label: { - en: "Message GC interval" - zh: "消息清理间隔" - } - } - - persistent_session_store_session_message_gc_interval { - desc { - en: "The starting interval for garbage collection of transient data for\n" - "persistent session messages. This does not affect the lifetime length\n" - "of persistent session messages." - zh: "持久会话消息的临时数据垃圾收集的开始间隔。\n" - "这不会影响持久会话消息的生命周期长度。" - } - label: { - en: "Session message GC interval" - zh: "会话消息清理间隔" - } - } - - persistent_session_builtin_session_table { - desc { - en: "Performance tuning options for built-in session table." - zh: "用于内建会话表的性能调优参数。" - } - label: { - en: "Persistent session" - zh: "持久会话" - } - } - - persistent_session_builtin_sess_msg_table { - desc { - en: "Performance tuning options for built-in session messages table." - zh: "优化内置的会话消息表的配置。" - } - label: { - en: "Persistent session messages" - zh: "用于内建会话管理表的性能调优参数" - } - } - - persistent_session_builtin_messages_table { - desc { - en: "Performance tuning options for built-in messages table." - zh: "用于内建消息表的性能调优参数。" - } - label: { - en: "Persistent messages" - zh: "持久化消息" - } - } - - stats_enable { - desc { - en: "Enable/disable statistic data collection." - zh: "启用/禁用统计数据收集功能。" - } - label: { - en: "Enable/disable statistic data collection." - zh: "启用/禁用统计数据收集功能" - } - } - - zones { - desc { - en: """A zone is a set of configs grouped by the zone name. +zones.desc: +"""A zone is a set of configs grouped by the zone name. For flexible configuration mapping, the name can be set to a listener's zone config. NOTE: A built-in zone named default is auto created and can not be deleted.""" - zh: """zone 是按name 分组的一组配置。 -对于灵活的配置映射，可以将 name 设置为侦听器的 zone 配置。 -注：名为 default 的内置区域是自动创建的，无法删除。""" - } - } - mqtt { - desc { - en: """Global MQTT configuration. -The configs here work as default values which can be overridden in zone configs""" - zh: """全局的 MQTT 配置项。 -mqtt 下所有的配置作为全局的默认值存在，它可以被 zone 中的配置覆盖。""" - } - } +fields_mqtt_quic_listener_certfile.desc: +"""Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead.""" - mqtt_idle_timeout { - desc { - en: """Configure the duration of time that a connection can remain idle (i.e., without any data transfer) before being: - - Automatically disconnected if no CONNECT package is received from the client yet. - - Put into hibernation mode to save resources if some CONNECT packages are already received. -Note: Please set the parameter with caution as long idle time will lead to resource waste.""" - zh: """设置连接被断开或进入休眠状态前的等待时间，空闲超时后， - - 如暂未收到客户端的 CONNECT 报文，连接将断开； - - 如已收到客户端的 CONNECT 报文，连接将进入休眠模式以节省系统资源。 +fields_mqtt_quic_listener_certfile.label: +"""Certificate file""" -注意：请合理设置该参数值，如等待时间设置过长，可能造成系统资源的浪费。""" - } - label: { - en: """Idle Timeout""" - zh: """空闲超时""" - } - } +fields_rate_limit_conn_bytes_in.desc: +"""Limit the rate of receiving packets for a MQTT connection. +The rate is counted by bytes of packets per second.""" - mqtt_max_packet_size { - desc { - en: """Maximum MQTT packet size allowed.""" - zh: """允许的最大 MQTT 报文大小。""" - } - label: { - en: """Max Packet Size""" - zh: """最大报文大小""" - } - } +fields_rate_limit_conn_bytes_in.label: +"""Connection bytes in""" - mqtt_max_clientid_len { - desc { - en: """Maximum allowed length of MQTT Client ID.""" - zh: """允许的最大 MQTT Client ID 长度。""" - } - label: { - en: """Max Client ID Length""" - zh: """最大 Client ID 长度""" - } - } +crl_cache_capacity.desc: +"""The maximum number of CRL URLs that can be held in cache. If the cache is at full capacity and a new URL must be fetched, then it'll evict the oldest inserted URL in the cache.""" - mqtt_max_topic_levels { - desc { - en: """Maximum topic levels allowed.""" - zh: """允许的最大主题层级。""" - } - label: { - en: """Max Topic Levels""" - zh: """最大主题层级""" - } - } +crl_cache_capacity.label: +"""CRL Cache Capacity""" - mqtt_max_qos_allowed { - desc { - en: """Maximum QoS allowed.""" - zh: """允许的最大 QoS 等级。""" - } - label: { - en: """Max QoS""" - zh: """最大 QoS""" - } - } +alarm_actions.desc: +"""The actions triggered when the alarm is activated.
Currently, the following actions are supported: log and publish. +log is to write the alarm to log (console or file). +publish is to publish the alarm as an MQTT message to the system topics: +$SYS/brokers/emqx@xx.xx.xx.x/alarms/activate and +$SYS/brokers/emqx@xx.xx.xx.x/alarms/deactivate""" - mqtt_max_topic_alias { - desc { - en: """Maximum topic alias, 0 means no topic alias supported.""" - zh: """允许的最大主题别名数，0 表示不支持主题别名。""" - } - label: { - en: """Max Topic Alias""" - zh: """最大主题别名""" - } - } +alarm_actions.label: +"""Alarm Actions""" - mqtt_retain_available { - desc { - en: """Whether to enable support for MQTT retained message.""" - zh: """是否启用对 MQTT 保留消息的支持。""" - } - label: { - en: """Retain Available""" - zh: """保留消息可用""" - } - } +base_listener_max_connections.desc: +"""The maximum number of concurrent connections allowed by the listener.""" - mqtt_wildcard_subscription { - desc { - en: """Whether to enable support for MQTT wildcard subscription.""" - zh: """是否启用对 MQTT 通配符订阅的支持。""" - } - label: { - en: """Wildcard Subscription Available""" - zh: """通配符订阅可用""" - } - } +base_listener_max_connections.label: +"""Max connections""" - mqtt_shared_subscription { - desc { - en: """Whether to enable support for MQTT shared subscription.""" - zh: """是否启用对 MQTT 共享订阅的支持。""" - } - label: { - en: """Shared Subscription Available""" - zh: """共享订阅可用""" - } - } - - mqtt_exclusive_subscription { - desc { - en: """Whether to enable support for MQTT exclusive subscription.""" - zh: """是否启用对 MQTT 排它订阅的支持。""" - } - label: { - en: """Exclusive Subscription""" - zh: """排它订阅""" - } - } - - mqtt_ignore_loop_deliver { - desc { - en: """Whether the messages sent by the MQTT v3.1.1/v3.1.0 client will be looped back to the publisher itself, similar to No Local in MQTT 5.0.""" - zh: """设置由 MQTT v3.1.1/v3.1.0 客户端发布的消息是否将转发给其本身；类似 MQTT 5.0 协议中的 No Local 选项。""" - } - label: { - en: """Ignore Loop Deliver""" - zh: """忽略循环投递""" - } - } - - mqtt_strict_mode { - desc { - en: """Whether to parse MQTT messages in strict mode. -In strict mode, invalid utf8 strings in for example client ID, topic name, etc. will cause the client to be disconnected.""" - zh: """是否以严格模式解析 MQTT 消息。 -严格模式下，如客户端 ID、主题名称等中包含无效 utf8 字符串，连接将被断开。""" - } - label: { - en: """Strict Mode""" - zh: """严格模式""" - } - } - - mqtt_response_information { - desc { - en: """UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communicate using the topic prefix reqrsp/. -To disable this feature, input \"\" in the text box below. Only applicable to MQTT 5.0 clients.""" - zh: """UTF-8 字符串，用于指定返回给客户端的响应主题，如 reqrsp/，此时请求和应答客户端都需要使用 reqrsp/ 前缀的主题来完成通讯。 -如希望禁用此功能，请在下方的文字框中输入\"\"；仅适用于 MQTT 5.0 客户端。""" - } - label: { - en: """Response Information""" - zh: """响应信息""" - } - } - - mqtt_server_keepalive { - desc { - en: """The keep alive duration required by EMQX. To use the setting from the client side, choose disabled from the drop-down list. Only applicable to MQTT 5.0 clients.""" - zh: """EMQX 要求的保活时间，如设为 disabled，则将使用客户端指定的保持连接时间；仅适用于 MQTT 5.0 客户端。""" - } - label: { - en: """Server Keep Alive""" - zh: """服务端保活时间""" - } - } - - mqtt_keepalive_backoff { - desc { - en: """The coefficient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" - zh: """EMQX 判定客户端保活超时使用的阈值系数。计算公式为：Keep Alive * Backoff * 2""" - } - label: { - en: """Keep Alive Backoff""" - zh: """保活超时阈值系数""" - } - } - - mqtt_max_subscriptions { - desc { - en: """Maximum number of subscriptions allowed per client.""" - zh: """允许每个客户端建立的最大订阅数量。""" - } - label: { - en: """Max Subscriptions""" - zh: """最大订阅数量""" - } - } - - mqtt_upgrade_qos { - desc { - en: """Force upgrade of QoS level according to subscription.""" - zh: """投递消息时，是否根据订阅主题时的 QoS 等级来强制提升派发的消息的 QoS 等级。""" - } - label: { - en: """Upgrade QoS""" - zh: """升级 QoS""" - } - } - - mqtt_max_inflight { - desc { - en: """Maximum number of QoS 1 and QoS 2 messages that are allowed to be delivered simultaneously before completing the acknowledgment.""" - zh: """允许在完成应答前同时投递的 QoS 1 和 QoS 2 消息的最大数量。""" - } - label: { - en: """Max Inflight""" - zh: """最大飞行窗口""" - } - - } - - mqtt_retry_interval { - desc { - en: """Retry interval for QoS 1/2 message delivering.""" - zh: """QoS 1/2 消息的重新投递间隔。""" - } - label: { - en: """Retry Interval""" - zh: """重试间隔""" - } - } - - mqtt_max_awaiting_rel { - desc { - en: """For each publisher session, the maximum number of outstanding QoS 2 messages pending on the client to send PUBREL. After reaching this limit, new QoS 2 PUBLISH requests will be rejected with `147(0x93)` until either PUBREL is received or timed out.""" - zh: """每个发布者的会话中，都存在一个队列来处理客户端发送的 QoS 2 消息。该队列会存储 QoS 2 消息的报文 ID 直到收到客户端的 PUBREL 或超时，达到队列长度的限制后，新的 QoS 2 消息发布会被拒绝，并返回 `147(0x93)` 错误。""" - } - label: { - en: """Max Awaiting PUBREL""" - zh: """PUBREL 等待队列长度""" - } - } - - mqtt_await_rel_timeout { - desc { - en: """For client to broker QoS 2 message, the time limit for the broker to wait before the `PUBREL` message is received. The wait is aborted after timed out, meaning the packet ID is freed for new `PUBLISH` requests. Receiving a stale `PUBREL` causes a warning level log. Note, the message is delivered to subscribers before entering the wait for PUBREL.""" - zh: """客户端发布 QoS 2 消息时，服务器等待 `PUBREL` 的最长时延。超过该时长后服务器会放弃等待，该PACKET ID 会被释放，从而允许后续新的 PUBLISH 消息使用。如果超时后收到 PUBREL，服务器将会产生一条告警日志。注意，向订阅客户端转发消息的动作发生在进入等待之前。""" - } - label: { - en: """Max Awaiting PUBREL TIMEOUT""" - zh: """PUBREL 最大等待时间""" - } - } - - mqtt_session_expiry_interval { - desc { - en: """Specifies how long the session will expire after the connection is disconnected, only for non-MQTT 5.0 connections.""" - zh: """指定会话将在连接断开后多久过期，仅适用于非 MQTT 5.0 的连接。""" - } - label: { - en: """Session Expiry Interval""" - zh: """会话过期间隔""" - } - } - - mqtt_max_mqueue_len { - desc { - en: """Maximum queue length. Enqueued messages when persistent client disconnected, or inflight window is full.""" - zh: """消息队列最大长度。持久客户端断开连接或飞行窗口已满时排队的消息长度。""" - } - label: { - en: """Max Message Queue Length""" - zh: """最大消息队列长度""" - } - } - - mqtt_mqueue_priorities { - desc { - en: """Topic priorities. Priority number [1-255] -There's no priority table by default, hence all messages are treated equal. - -**NOTE**: Comma and equal signs are not allowed for priority topic names. -**NOTE**: Messages for topics not in the priority table are treated as either highest or lowest priority depending on the configured value for mqtt.mqueue_default_priority. - -**Examples**: -To configure \"topic/1\" > \"topic/2\": -mqueue_priorities: {\"topic/1\": 10, \"topic/2\": 8}""" - zh: """主题优先级。取值范围 [1-255] -默认优先级表为空，即所有的主题优先级相同。 - -注：优先主题名称中不支持使用逗号和等号。 -注：不在此列表中的主题，被视为最高/最低优先级，这取决于mqtt.mqueue_default_priority 的配置 - -示例： -配置 \"topic/1\" > \"topic/2\": -mqueue_priorities: {\"topic/1\": 10, \"topic/2\": 8}""" - } - label: { - en: """Topic Priorities""" - zh: """主题优先级""" - } - } - - mqtt_mqueue_default_priority { - desc { - en: """Default topic priority, which will be used by topics not in Topic Priorities (mqueue_priorities).""" - zh: """默认的主题优先级，不在 主题优先级（mqueue_priorities） 中的主题将会使用该优先级。""" - } - label: { - en: """Default Topic Priorities""" - zh: """默认主题优先级""" - } - } - - mqtt_mqueue_store_qos0 { - desc { - en: """Specifies whether to store QoS 0 messages in the message queue while the connection is down but the session remains.""" - zh: """指定在连接断开但会话保持期间，是否需要在消息队列中存储 QoS 0 消息。""" - } - label: { - en: """Store QoS 0 Message""" - zh: """存储 QoS 0 消息""" - } - } - - mqtt_use_username_as_clientid { - desc { - en: """Whether to use Username as Client ID. -This setting takes effect later than Use Peer Certificate as Username and Use peer certificate as Client ID.""" - zh: """是否使用用户名作为客户端 ID。 -此设置的作用时间晚于 对端证书作为用户名 和 对端证书作为客户端 ID。""" - } - label: { - en: """Use Username as Client ID""" - zh: """用户名作为客户端 ID""" - } - } - - mqtt_peer_cert_as_username { - desc { - en: """Use the CN, DN field in the peer certificate or the entire certificate content as Username. Only works for the TLS connection. +mqtt_peer_cert_as_username.desc: +"""Use the CN, DN field in the peer certificate or the entire certificate content as Username. Only works for the TLS connection. Supported configurations are the following: - cn: CN field of the certificate - dn: DN field of the certificate - crt: Content of the DER or PEM certificate - pem: Convert DER certificate content to PEM format and use as Username - md5: MD5 value of the DER or PEM certificate""" - zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为用户名；仅适用于 TLS 连接。 -目前支持： -- cn: 取证书的 CN 字段 -- dn: 取证书的 DN 字段 -- crt: 取 DER 或 PEM 的证书内容 -- pem: 将 DER 证书转换为 PEM 格式作为用户名 -- md5: 取 DER 或 PEM 证书内容的 MD5 值""" - } - label: { - en: """Use Peer Certificate as Username""" - zh: """对端证书作为用户名""" - } - } - mqtt_peer_cert_as_clientid { - desc { - en: """Use the CN, DN field in the peer certificate or the entire certificate content as Client ID. Only works for the TLS connection. -Supported configurations are the following: -- cn: CN field of the certificate -- dn: DN field of the certificate -- crt: DER or PEM certificate -- pem: Convert DER certificate content to PEM format and use as Client ID -- md5: MD5 value of the DER or PEM certificate""" - zh: """使用对端证书中的 CN、DN 字段或整个证书内容来作为客户端 ID。仅适用于 TLS 连接； -目前支持： -- cn: 取证书的 CN 字段 -- dn: 取证书的 DN 字段 -- crt: 取 DER 或 PEM 证书的内容 -- pem: 将 DER 证书内容转换为 PEM 格式作为客户端 ID -- md5: 取 DER 或 PEM 证书内容的 MD5 值""" - } - label: { - en: """Use Peer Certificate as Client ID""" - zh: """对端证书作为客户端 ID""" - } - } +mqtt_peer_cert_as_username.label: +"""Use Peer Certificate as Username""" - broker { - desc { - en: """Message broker options.""" - zh: """Broker 相关配置项。""" - } - } +fields_cache_enable.desc: +"""Enable or disable the authorization cache.""" - broker_enable_session_registry { - desc { - en: """Enable session registry""" - zh: """是否启用 Session Registry""" - } - } +fields_cache_enable.label: +"""Enable or disable the authorization cache.""" - broker_session_locking_strategy { - desc { - en: """Session locking strategy in a cluster. - - `local`: only lock the session on the current node - - `one`: select only one remote node to lock the session - - `quorum`: select some nodes to lock the session - - `all`: lock the session on all the nodes in the cluster""" +fields_mqtt_quic_listener_disconnect_timeout_ms.desc: +"""How long to wait for an ACK before declaring a path dead and disconnecting. Default: 16000""" - zh: """Session 在集群中的锁策略。 - - `loca`：仅锁本节点的 Session； - - `one`：任选一个其它节点加锁； - - `quorum`：选择集群中半数以上的节点加锁； - - `all`：选择所有节点加锁。""" - } - } +fields_mqtt_quic_listener_disconnect_timeout_ms.label: +"""Disconnect timeout ms""" - broker_shared_subscription_strategy { - desc { - en: """Dispatch strategy for shared subscription. - - `random`: dispatch the message to a random selected subscriber - - `round_robin`: select the subscribers in a round-robin manner - - `round_robin_per_group`: select the subscribers in round-robin fashion within each shared subscriber group - - `local`: select random local subscriber otherwise select random cluster-wide - - `sticky`: always use the last selected subscriber to dispatch, until the subscriber disconnects. - - `hash_clientid`: select the subscribers by hashing the `clientIds` - - `hash_topic`: select the subscribers by hashing the source topic""" +mqtt_max_topic_alias.desc: +"""Maximum topic alias, 0 means no topic alias supported.""" - zh: """共享订阅消息派发策略。 - - `random`：随机挑选一个共享订阅者派发； - - `round_robin`：使用 round-robin 策略派发； - - `round_robin_per_group`：在共享组内循环选择下一个成员； - - `local`：选择随机的本地成员，否则选择随机的集群范围内成员; - - `sticky`：总是使用上次选中的订阅者派发，直到它断开连接； - - `hash_clientid`：通过对发送者的客户端 ID 进行 Hash 处理来选择订阅者; - - `hash_topic`：通过对源主题进行 Hash 处理来选择订阅者。""" - } - } +mqtt_max_topic_alias.label: +"""Max Topic Alias""" - broker_shared_dispatch_ack_enabled { - desc { - en: """Deprecated, will be removed in 5.1. -Enable/disable shared dispatch acknowledgement for QoS 1 and QoS 2 messages. -This should allow messages to be dispatched to a different subscriber in the group in case the picked (based on `shared_subscription_strategy`) subscriber is offline.""" +common_ssl_opts_schema_user_lookup_fun.desc: +"""EMQX-internal callback that is used to lookup pre-shared key (PSK) identity.""" - zh: """该配置项已废弃，会在 5.1 中移除。 -启用/禁用 QoS 1 和 QoS 2 消息的共享派发确认。 -开启后，允许将消息从未及时回复 ACK 的订阅者 （例如，客户端离线） 重新派发给另外一个订阅者。""" - } - } +common_ssl_opts_schema_user_lookup_fun.label: +"""SSL PSK user lookup fun""" - broker_route_batch_clean { - desc { - en: """Enable batch clean for deleted routes.""" - zh: """是否开启批量清除路由。""" - } - } +fields_listeners_wss.desc: +"""HTTPS websocket listeners.""" - shared_subscription_group_strategy { - desc { - en: """Per group dispatch strategy for shared subscription. -This config is a map from shared subscription group name to the strategy -name. The group name should be of format `[A-Za-z0-9]`. i.e. no -special characters are allowed.""" - zh: """设置共享订阅组为单位的分发策略。该配置是一个从组名到 -策略名的一个map，组名不得包含 `[A-Za-z0-9]` 之外的特殊字符。""" - } +fields_listeners_wss.label: +"""HTTPS websocket listeners""" - } +sysmon_top_max_procs.desc: +"""Stop collecting data when the number of processes +in the VM exceeds this value""" - shared_subscription_strategy_enum { - desc { - en: """Dispatch strategy for shared subscription. -- `random`: dispatch the message to a random selected subscriber -- `round_robin`: select the subscribers in a round-robin manner -- `round_robin_per_group`: select the subscribers in round-robin fashion within each shared subscriber group -- `sticky`: always use the last selected subscriber to dispatch, -until the subscriber disconnects. -- `hash`: select the subscribers by the hash of `clientIds` -- `local`: send to a random local subscriber. If local -subscriber was not found, send to a random subscriber cluster-wide""" - zh: """共享订阅的分发策略名称。 -- `random`：随机选择一个组内成员； -- `round_robin`：循环选择下一个成员； -- `round_robin_per_group`：在共享组内循环选择下一个成员； -- `sticky`：使用上一次选中的成员； -- `hash`：根据 ClientID 哈希映射到一个成员； -- `local`：随机分发到节点本地成成员，如果本地成员不存在，则随机分发到任意一个成员。""" +sysmon_top_max_procs.label: +"""Max procs""" - } - } +mqtt_use_username_as_clientid.desc: +"""Whether to use Username as Client ID. +This setting takes effect later than Use Peer Certificate as Username and Use peer certificate as Client ID.""" - broker_perf_route_lock_type { - desc { - en: """Performance tuning for subscribing/unsubscribing a wildcard topic. -Change this parameter only when there are many wildcard topics. +mqtt_use_username_as_clientid.label: +"""Use Username as Client ID""" -NOTE: when changing from/to `global` lock, it requires all nodes in the cluster to be stopped before the change. - - `key`: mnesia transactional updates with per-key locks. Recommended for a single-node setup. - - `tab`: mnesia transactional updates with table lock. Recommended for a cluster setup. - - `global`: updates are protected with a global lock. Recommended for large clusters.""" - zh: """通配主题订阅/取消订阅性能调优。 -建议仅当通配符主题较多时才更改此参数。 +mqtt_max_qos_allowed.desc: +"""Maximum QoS allowed.""" -注：当从/更改为 `global` 锁时，它要求集群中的所有节点在更改之前停止。 - - `key`：为 Mnesia 事务涉及到的每个 key 上锁，建议单节点时使用。 - - `tab`：为 Mnesia 事务涉及到的表上锁，建议在集群中使用。 - - `global`：所以更新操作都被全局的锁保护，仅建议在超大规模集群中使用。""" - } - } +mqtt_max_qos_allowed.label: +"""Max QoS""" - broker_perf_trie_compaction { - desc { - en: """Enable trie path compaction. -Enabling it significantly improves wildcard topic subscribe rate, if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', where ID is unique per subscriber. -Topic match performance (when publishing) may degrade if messages are mostly published to topics with large number of levels. +fields_mqtt_quic_listener_max_binding_stateless_operations.desc: +"""The maximum number of stateless operations that may be queued on a binding at any one time. Default: 100""" -NOTE: This is a cluster-wide configuration. It requires all nodes to be stopped before changing it.""" - zh: """是否开启主题表压缩存储。 -启用它会显着提高通配符主题订阅率，如果通配符主题具有唯一前缀，例如：'sensor/{{id}}/+/'，其中每个订阅者的 ID 是唯一的。 -如果消息主要发布到具有大量级别的主题，则主题匹配性能（发布时）可能会降低。 +fields_mqtt_quic_listener_max_binding_stateless_operations.label: +"""Max binding stateless operations""" -注意：这是一个集群范围的配置。 它要求在更改之前停止所有节点。""" - } - } +fields_mqtt_quic_listener_stream_recv_buffer_default.desc: +"""Stream initial buffer size. Default: 4096""" - sys_topics { - desc { - en: """System topics configuration.""" - zh: """系统主题配置。""" - } - } +fields_mqtt_quic_listener_stream_recv_buffer_default.label: +"""Stream recv buffer default""" - sys_msg_interval { - desc { - en: """Time interval of publishing `$SYS` messages.""" - zh: """发送 `$SYS` 主题的间隔时间。""" - } - } +fields_mqtt_quic_listener_pacing_enabled.desc: +"""Pace sending to avoid overfilling buffers on the path. Default: 1 (Enabled)""" - sys_heartbeat_interval { - desc { - en: """Time interval for publishing following heartbeat messages: - - `$SYS/brokers//uptime` - - `$SYS/brokers//datetime`""" - zh: """发送心跳系统消息的间隔时间，它包括： - - `$SYS/brokers//uptime` - - `$SYS/brokers//datetime`""" - } - } +fields_mqtt_quic_listener_pacing_enabled.label: +"""Pacing enabled""" - sys_event_messages { - desc { - en: """Client events messages.""" - zh: """客户端事件消息。""" - } - } +mqtt_max_subscriptions.desc: +"""Maximum number of subscriptions allowed per client.""" - sys_event_client_connected { - desc { - en: """Enable to publish client connected event messages""" - zh: """是否开启客户端已连接事件消息。""" - } - } +mqtt_max_subscriptions.label: +"""Max Subscriptions""" - sys_event_client_disconnected { - desc { - en: """Enable to publish client disconnected event messages.""" - zh: """是否开启客户端已断开连接事件消息。""" - } - } +persistent_session_builtin_messages_table.desc: +"""Performance tuning options for built-in messages table.""" - sys_event_client_subscribed { - desc { - en: """Enable to publish event message that client subscribed a topic successfully.""" - zh: """是否开启客户端已成功订阅主题事件消息。""" - } - } +persistent_session_builtin_messages_table.label: +"""Persistent messages""" - sys_event_client_unsubscribed { - desc { - en: """Enable to publish event message that client unsubscribed a topic successfully.""" - zh: """是否开启客户端已成功取消订阅主题事件消息。""" - } - } +sysmon_os_cpu_low_watermark.desc: +"""The threshold, as percentage of system CPU load, + for how much system cpu can be used before the corresponding alarm is cleared.""" +sysmon_os_cpu_low_watermark.label: +"""CPU low watermark""" -fields_authorization_no_match { - desc { - en: """Default access control action if the user or client matches no ACL rules, -or if no such user or client is found by the configurable authorization -sources such as built_in_database, an HTTP API, or a query against PostgreSQL. -Find more details in 'authorization.sources' config.""" - zh: """如果用户或客户端不匹配ACL规则，或者从可配置授权源(比如内置数据库、HTTP API 或 PostgreSQL 等。)内未找 -到此类用户或客户端时，模式的认访问控制操作。 -在“授权”中查找更多详细信息。""" - } - label: { - en: "Authorization no match" - zh: "未匹时的默认授权动作" - } -} +fields_mqtt_quic_listener_tls_server_max_send_buffer.desc: +"""How much Server TLS data to buffer. Default: 8192""" -fields_authorization_deny_action { - desc { - en: """The action when the authorization check rejects an operation.""" - zh: """授权检查拒绝操作时的操作。""" - } - label: { - en: "Authorization deny action" - zh: "授权检查拒绝操作时的操作" - } -} +fields_mqtt_quic_listener_tls_server_max_send_buffer.label: +"""TLS server max send buffer""" -fields_cache_enable { - desc { - en: """Enable or disable the authorization cache.""" - zh: """启用或禁用授权缓存。""" - } - label: { - en: "Enable or disable the authorization cache." - zh: "启用或禁用授权缓存" - } -} +base_listener_bind.desc: +"""IP address and port for the listening socket.""" -fields_cache_max_size { - desc { - en: """Maximum number of cached items.""" - zh: """缓存项的最大数量。""" - } - label: { - en: "Maximum number of cached items." - zh: "缓存项的最大数量" - } -} +base_listener_bind.label: +"""IP address and port""" -fields_cache_ttl { - desc { - en: """Time to live for the cached data.""" - zh: """缓存数据的生存时间。""" - } - label: { - en: "Time to live for the cached data." - zh: "缓存数据的生存时间。" - } -} +server_ssl_opts_schema_handshake_timeout.desc: +"""Maximum time duration allowed for the handshake to complete""" -fields_deflate_opts_level { - desc { - en: """Compression level.""" - zh: """压缩级别""" - } - label: { - en: "Compression level" - zh: "压缩级别" - } -} +server_ssl_opts_schema_handshake_timeout.label: +"""Handshake timeout""" -fields_deflate_opts_mem_level { - desc { - en: """Specifies the size of the compression state.
-Lower values decrease memory usage per connection.""" - zh: """指定压缩状态的大小
-较低的值会减少每个连接的内存使用。""" - } - label: { - en: "Size of the compression state" - zh: "压缩状态大小" - } -} +fields_deflate_opts_server_context_takeover.desc: +"""Takeover means the compression state is retained between server messages.""" -fields_deflate_opts_strategy { - desc { - en: """Specifies the compression strategy.""" - zh: """指定压缩策略。""" - } - label: { - en: "compression strategy" - zh: "指定压缩策略" - } -} +fields_deflate_opts_server_context_takeover.label: +"""Server context takeover""" -fields_deflate_opts_server_context_takeover { - desc { - en: """Takeover means the compression state is retained between server messages.""" - zh: """接管意味着在服务器消息之间保留压缩状态。""" - } - label: { - en: "Server context takeover" - zh: "服务上下文接管" - } -} +mqtt_session_expiry_interval.desc: +"""Specifies how long the session will expire after the connection is disconnected, only for non-MQTT 5.0 connections.""" -fields_deflate_opts_client_context_takeover { - desc { - en: """Takeover means the compression state is retained between client messages.""" - zh: """接管意味着在客户端消息之间保留压缩状态。""" - } - label: { - en: "Client context takeover" - zh: "客户端上下文接管" - } -} +mqtt_session_expiry_interval.label: +"""Session Expiry Interval""" -fields_deflate_opts_server_max_window_bits { - desc { - en: """Specifies the size of the compression context for the server.""" - zh: """指定服务器压缩上下文的大小。""" - } - label: { - en: "Server compression max window size" - zh: "服务器压缩窗口大小" - } -} +fields_listener_enabled.desc: +"""Enable listener.""" -fields_deflate_opts_client_max_window_bits { - desc { - en: """Specifies the size of the compression context for the client.""" - zh: """指定客户端压缩上下文的大小。""" - } - label: { - en: "Client compression max window size" - zh: "压缩窗口大小" - } -} +fields_listener_enabled.label: +"""Enable listener""" -client_ssl_opts_schema_enable { - desc { - en: """Enable TLS.""" - zh: """启用 TLS。""" - } - label: { - en: "Enable TLS." - zh: "启用 TLS" - } -} +mqtt.desc: +"""Global MQTT configuration. +The configs here work as default values which can be overridden in zone configs""" -common_ssl_opts_schema_cacertfile { - desc { - en: """Trusted PEM format CA certificates bundle file.
+crl_cache_refresh_http_timeout.desc: +"""The timeout for the HTTP request when fetching CRLs. This is a global setting for all listeners.""" + +crl_cache_refresh_http_timeout.label: +"""CRL Cache Refresh HTTP Timeout""" + +fields_tcp_opts_backlog.desc: +"""TCP backlog defines the maximum length that the queue of +pending connections can grow to.""" + +fields_tcp_opts_backlog.label: +"""TCP backlog length""" + +broker_route_batch_clean.desc: +"""Enable batch clean for deleted routes.""" + +fields_mqtt_quic_listener_initial_window_packets.desc: +"""The size (in packets) of the initial congestion window for a connection. Default: 10""" + +fields_mqtt_quic_listener_initial_window_packets.label: +"""Initial window packets""" + +flapping_detect_enable.desc: +"""Enable flapping connection detection feature.""" + +flapping_detect_enable.label: +"""Enable flapping detection""" + +sysmon_top_db_password.desc: +"""EMQX user password in the PostgreSQL database""" + +sysmon_top_db_password.label: +"""DB Password""" + +fields_ws_opts_check_origins.desc: +"""List of allowed origins.
See check_origin_enable.""" + +fields_ws_opts_check_origins.label: +"""Allowed origins""" + +fields_deflate_opts_client_context_takeover.desc: +"""Takeover means the compression state is retained between client messages.""" + +fields_deflate_opts_client_context_takeover.label: +"""Client context takeover""" + +base_listener_acceptors.desc: +"""The size of the listener's receiving pool.""" + +base_listener_acceptors.label: +"""Acceptors Num""" + +common_ssl_opts_schema_cacertfile.desc: +"""Trusted PEM format CA certificates bundle file.
The certificates in this file are used to verify the TLS peer's certificates. Append new certificates to the file if new CAs are to be trusted. There is no need to restart EMQX to have the updated file loaded, because the system regularly checks if file has been updated (and reload).
NOTE: invalidating (deleting) a certificate from the file will not affect already established connections.""" - zh: """受信任的PEM格式 CA 证书捆绑文件
-此文件中的证书用于验证TLS对等方的证书。 -如果要信任新 CA，请将新证书附加到文件中。 -无需重启EMQX即可加载更新的文件，因为系统会定期检查文件是否已更新（并重新加载）
-注意：从文件中失效（删除）证书不会影响已建立的连接。""" - } - label: { - en: "CACertfile" - zh: "CA 证书文件" - } -} -common_ssl_opts_schema_certfile { - desc { - en: """PEM format certificates chain file.
-The certificates in this file should be in reversed order of the certificate -issue chain. That is, the host's certificate should be placed in the beginning -of the file, followed by the immediate issuer certificate and so on. -Although the root CA certificate is optional, it should be placed at the end of -the file if it is to be added.""" - zh: """PEM格式证书链文件
-此文件中的证书应与证书颁发链的顺序相反。也就是说，主机的证书应该放在文件的开头， -然后是直接颁发者 CA 证书，依此类推，一直到根 CA 证书。 -根 CA 证书是可选的，如果想要添加，应加到文件到最末端。""" - } - label: { - en: "Certfile" - zh: "证书文件" - } -} +common_ssl_opts_schema_cacertfile.label: +"""CACertfile""" -common_ssl_opts_schema_keyfile { - desc { - en: """PEM format private key file.""" - zh: """PEM格式的私钥文件。""" - } - label: { - en: "Keyfile" - zh: "私钥文件" - } -} +fields_ws_opts_mqtt_path.desc: +"""WebSocket's MQTT protocol path. So the address of EMQX Broker's WebSocket is: +ws://{ip}:{port}/mqtt""" -common_ssl_opts_schema_verify { - desc { - en: """Enable or disable peer verification.""" - zh: """启用或禁用对等验证。""" - } - label: { - en: "Verify peer" - zh: "对等验证" - } -} +fields_ws_opts_mqtt_path.label: +"""WS MQTT Path""" -common_ssl_opts_schema_reuse_sessions { - desc { - en: """Enable TLS session reuse.""" - zh: """启用 TLS 会话重用。""" - } - label: { - en: "TLS session reuse" - zh: "TLS 会话重用" - } -} +sysmon_os_procmem_high_watermark.desc: +"""The threshold, as percentage of system memory, + for how much system memory can be allocated by one Erlang process before + the corresponding alarm is raised.""" -common_ssl_opts_schema_depth { - desc { - en: """Maximum number of non-self-issued intermediate certificates that can follow the peer certificate in a valid certification path. -So, if depth is 0 the PEER must be signed by the trusted ROOT-CA directly;
-if 1 the path can be PEER, Intermediate-CA, ROOT-CA;
-if 2 the path can be PEER, Intermediate-CA1, Intermediate-CA2, ROOT-CA.""" - zh: """在有效的证书路径中，可以跟随对等证书的非自颁发中间证书的最大数量。 -因此，如果深度为0，则对等方必须由受信任的根 CA 直接签名；
-如果是1，路径可以是 PEER、中间 CA、ROOT-CA；
-如果是2，则路径可以是PEER、中间 CA1、中间 CA2、ROOT-CA。""" - } - label: { - en: "CACert Depth" - zh: "CA 证书深度" - } -} +sysmon_os_procmem_high_watermark.label: +"""ProcMem high wartermark""" -common_ssl_opts_schema_password { - desc { - en: """String containing the user's password. Only used if the private key file is password-protected.""" - zh: """包含用户密码的字符串。仅在私钥文件受密码保护时使用。""" - } - label: { - en: "Keyfile passphrase" - zh: "秘钥文件密码" - } -} +fields_listeners_quic.desc: +"""QUIC listeners.""" -common_ssl_opts_schema_versions { - desc { - en: """All TLS/DTLS versions to be supported.
-NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config.
-In case PSK cipher suites are intended, make sure to configure -['tlsv1.2', 'tlsv1.1'] here.""" - zh: """支持所有TLS/DTLS版本
-注：PSK 的 Ciphers 无法在 tlsv1.3 中使用，如果打算使用 PSK 密码套件，请确保这里配置为 ["tlsv1.2","tlsv1.1"]。""" - } - label: { - en: "SSL versions" - zh: "SSL 版本" - } -} +fields_listeners_quic.label: +"""QUIC listeners""" -common_ssl_opts_schema_hibernate_after { - desc { - en: """Hibernate the SSL process after idling for amount of time reducing its memory footprint.""" - zh: """在闲置一定时间后休眠 SSL 进程，减少其内存占用。""" - } - label: { - en: "hibernate after" - zh: "闲置多久后休眠" - } -} +fields_listeners_ws.desc: +"""HTTP websocket listeners.""" -ciphers_schema_common { - desc { - en: """This config holds TLS cipher suite names separated by comma, -or as an array of strings. e.g. -"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256" or -["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]. -
-Ciphers (and their ordering) define the way in which the -client and server encrypts information over the network connection. -Selecting a good cipher suite is critical for the -application's data security, confidentiality and performance. +fields_listeners_ws.label: +"""HTTP websocket listeners""" -The names should be in OpenSSL string format (not RFC format). -All default values and examples provided by EMQX config -documentation are all in OpenSSL format.
+mqtt_retry_interval.desc: +"""Retry interval for QoS 1/2 message delivering.""" -NOTE: Certain cipher suites are only compatible with -specific TLS versions ('tlsv1.1', 'tlsv1.2' or 'tlsv1.3') -incompatible cipher suites will be silently dropped. -For instance, if only 'tlsv1.3' is given in the versions, -configuring cipher suites for other versions will have no effect. -
+mqtt_retry_interval.label: +"""Retry Interval""" -NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config
-If PSK cipher suites are intended, 'tlsv1.3' should be disabled from versions.
-PSK cipher suites: "RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, -RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, -RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, -RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"""" - zh: """此配置保存由逗号分隔的 TLS 密码套件名称，或作为字符串数组。例如 -"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256"或 -["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]。 -
-密码（及其顺序）定义了客户端和服务器通过网络连接加密信息的方式。 -选择一个好的密码套件对于应用程序的数据安全性、机密性和性能至关重要。 +stats_enable.desc: +"""Enable/disable statistic data collection.""" -名称应为 OpenSSL 字符串格式（而不是 RFC 格式）。 -EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
-注意：某些密码套件仅与特定的 TLS 版本兼容（'tlsv1.1'、'tlsv1.2'或'tlsv1.3'）。 -不兼容的密码套件将被自动删除。 +stats_enable.label: +"""Enable/disable statistic data collection.""" -例如，如果只有 versions 仅配置为 tlsv1.3。为其他版本配置密码套件将无效。 +fields_authorization_deny_action.desc: +"""The action when the authorization check rejects an operation.""" -
-注：PSK 的 Ciphers 不支持 tlsv1.3
-如果打算使用PSK密码套件 tlsv1.3。应在ssl.versions中禁用。 +fields_authorization_deny_action.label: +"""Authorization deny action""" -
-PSK 密码套件： -"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, -RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, -RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, -RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"""" - } - label: { - en: "" - zh: "" - } -} +fields_deflate_opts_server_max_window_bits.desc: +"""Specifies the size of the compression context for the server.""" -ciphers_schema_quic { - desc { - en: """This config holds TLS cipher suite names separated by comma, +fields_deflate_opts_server_max_window_bits.label: +"""Server compression max window size""" + +client_ssl_opts_schema_server_name_indication.desc: +"""Specify the host name to be used in TLS Server Name Indication extension.
+For instance, when connecting to "server.example.net", the genuine server +which accepts the connection and performs TLS handshake may differ from the +host the TLS client initially connects to, e.g. when connecting to an IP address +or when the host has multiple resolvable DNS records
+If not specified, it will default to the host name string which is used +to establish the connection, unless it is IP addressed used.
+The host name is then also used in the host name verification of the peer +certificate.
The special value 'disable' prevents the Server Name +Indication extension from being sent and disables the hostname +verification check.""" + +client_ssl_opts_schema_server_name_indication.label: +"""Server Name Indication""" + +fields_mqtt_quic_listener_retry_memory_limit.desc: +"""The percentage of available memory usable for handshake connections before stateless retry is used. Calculated as `N/65535`. Default: 65""" + +fields_mqtt_quic_listener_retry_memory_limit.label: +"""Retry memory limit""" + +force_shutdown_max_message_queue_len.desc: +"""Maximum message queue length.""" + +force_shutdown_max_message_queue_len.label: +"""Maximum mailbox queue length of process.""" + +sys_heartbeat_interval.desc: +"""Time interval for publishing following heartbeat messages: + - `$SYS/brokers//uptime` + - `$SYS/brokers//datetime`""" + +flapping_detect_ban_time.desc: +"""How long the flapping clientid will be banned.""" + +flapping_detect_ban_time.label: +"""Ban time""" + +sysmon_top_num_items.desc: +"""The number of top processes per monitoring group""" + +sysmon_top_num_items.label: +"""Top num items""" + +persistent_session_builtin_session_table.desc: +"""Performance tuning options for built-in session table.""" + +persistent_session_builtin_session_table.label: +"""Persistent session""" + +mqtt_upgrade_qos.desc: +"""Force upgrade of QoS level according to subscription.""" + +mqtt_upgrade_qos.label: +"""Upgrade QoS""" + +mqtt_shared_subscription.desc: +"""Whether to enable support for MQTT shared subscription.""" + +mqtt_shared_subscription.label: +"""Shared Subscription Available""" + +fields_tcp_opts_sndbuf.desc: +"""The TCP send buffer (OS kernel) for the connections.""" + +fields_tcp_opts_sndbuf.label: +"""TCP send buffer""" + +sysmon_os_mem_check_interval.desc: +"""The time interval for the periodic memory check.""" + +sysmon_os_mem_check_interval.label: +"""Mem check interval""" + +server_ssl_opts_schema_gc_after_handshake.desc: +"""Memory usage tuning. If enabled, will immediately perform a garbage collection after the TLS/SSL handshake.""" + +server_ssl_opts_schema_gc_after_handshake.label: +"""Perform GC after handshake""" + +fields_mqtt_quic_listener_ssl_options.desc: +"""TLS options for QUIC transport""" + +fields_mqtt_quic_listener_ssl_options.label: +"""TLS Options""" + +fields_ws_opts_mqtt_piggyback.desc: +"""Whether a WebSocket message is allowed to contain multiple MQTT packets.""" + +fields_ws_opts_mqtt_piggyback.label: +"""MQTT Piggyback""" + +base_listener_mountpoint.desc: +"""When publishing or subscribing, prefix all topics with a mountpoint string. +The prefixed string will be removed from the topic name when the message +is delivered to the subscriber. The mountpoint is a way that users can use +to implement isolation of message routing between different listeners. +For example if a client A subscribes to `t` with `listeners.tcp.\.mountpoint` +set to `some_tenant`, then the client actually subscribes to the topic +`some_tenant/t`. Similarly, if another client B (connected to the same listener +as the client A) sends a message to topic `t`, the message is routed +to all the clients subscribed `some_tenant/t`, so client A will receive the +message, with topic name `t`.
+Set to `""` to disable the feature.
+ +Variables in mountpoint string: + - ${clientid}: clientid + - ${username}: username""" + +base_listener_mountpoint.label: +"""mountpoint""" + +mqtt_max_awaiting_rel.desc: +"""For each publisher session, the maximum number of outstanding QoS 2 messages pending on the client to send PUBREL. After reaching this limit, new QoS 2 PUBLISH requests will be rejected with `147(0x93)` until either PUBREL is received or timed out.""" + +mqtt_max_awaiting_rel.label: +"""Max Awaiting PUBREL""" + +ciphers_schema_quic.desc: +"""This config holds TLS cipher suite names separated by comma, or as an array of strings. e.g. "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256" or ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]. @@ -1606,115 +465,744 @@ RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
NOTE: QUIC listener supports only 'tlsv1.3' ciphers""" - zh: """此配置保存由逗号分隔的 TLS 密码套件名称，或作为字符串数组。例如 -"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256"或 -["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]。 -
-密码（及其顺序）定义了客户端和服务器通过网络连接加密信息的方式。 -选择一个好的密码套件对于应用程序的数据安全性、机密性和性能至关重要。 +ciphers_schema_quic.label: +"""""" -名称应为 OpenSSL 字符串格式（而不是 RFC 格式）。 -EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
-注意：某些密码套件仅与特定的 TLS 版本兼容（'tlsv1.1'、'tlsv1.2'或'tlsv1.3'）。 -不兼容的密码套件将被自动删除。 +fields_mqtt_quic_listener_max_bytes_per_key.desc: +"""Maximum number of bytes to encrypt with a single 1-RTT encryption key before initiating key update. Default: 274877906944""" -例如，如果只有 versions 仅配置为 tlsv1.3。为其他版本配置密码套件将无效。 +fields_mqtt_quic_listener_max_bytes_per_key.label: +"""Max bytes per key""" -
-注：PSK 的 Ciphers 不支持 tlsv1.3
-如果打算使用PSK密码套件，tlsv1.3。应在ssl.versions中禁用。 +fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us.desc: +"""The time in microseconds to wait before reattempting MTU probing if max was not reached. Default: 600000000""" -
-PSK 密码套件： -"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, -RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, -RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, -RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
+fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us.label: +"""MTU discovery search complete timeout us""" -注：QUIC 监听器不支持 tlsv1.3 的 ciphers""" - } - label: { - en: "" - zh: "" - } -} +fields_ws_opts_check_origin_enable.desc: +"""If true, origin HTTP header will be + validated against the list of allowed origins configured in check_origins + parameter.""" -common_ssl_opts_schema_user_lookup_fun { - desc { - en: """EMQX-internal callback that is used to lookup pre-shared key (PSK) identity.""" - zh: """用于查找预共享密钥（PSK）标识的 EMQX 内部回调。""" - } - label: { - en: "SSL PSK user lookup fun" - zh: "SSL PSK 用户回调" - } -} +fields_ws_opts_check_origin_enable.label: +"""Check origin""" -common_ssl_opts_schema_secure_renegotiate { - desc { - en: """SSL parameter renegotiation is a feature that allows a client and a server -to renegotiate the parameters of the SSL connection on the fly. -RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, -you drop support for the insecure renegotiation, prone to MitM attacks.""" - zh: """SSL 参数重新协商是一种允许客户端和服务器动态重新协商 SSL 连接参数的功能。 -RFC 5746 定义了一种更安全的方法。通过启用安全的重新协商，您就失去了对不安全的重新协商的支持，从而容易受到 MitM 攻击。""" - } - label: { - en: "SSL renegotiate" - zh: "SSL 重新协商" - } -} +sysmon_vm_busy_dist_port.desc: +"""When the RPC connection used to communicate with other nodes in the cluster is overloaded, +there will be a busy_dist_port warning log, +and an MQTT message is published to system topic $SYS/sysmon/busy_dist_port.""" -server_ssl_opts_schema_dhfile { - desc { - en: """Path to a file containing PEM-encoded Diffie-Hellman parameters +sysmon_vm_busy_dist_port.label: +"""Enable Busy Distribution Port monitoring.""" + +mqtt_max_mqueue_len.desc: +"""Maximum queue length. Enqueued messages when persistent client disconnected, or inflight window is full.""" + +mqtt_max_mqueue_len.label: +"""Max Message Queue Length""" + +mqtt_max_inflight.desc: +"""Maximum number of QoS 1 and QoS 2 messages that are allowed to be delivered simultaneously before completing the acknowledgment.""" + +mqtt_max_inflight.label: +"""Max Inflight""" + +persistent_session_store_enabled.desc: +"""Use the database to store information about persistent sessions. +This makes it possible to migrate a client connection to another +cluster node if a node is stopped.""" + +persistent_session_store_enabled.label: +"""Enable persistent session store""" + +fields_deflate_opts_level.desc: +"""Compression level.""" + +fields_deflate_opts_level.label: +"""Compression level""" + +mqtt_server_keepalive.desc: +"""The keep alive duration required by EMQX. To use the setting from the client side, choose disabled from the drop-down list. Only applicable to MQTT 5.0 clients.""" + +mqtt_server_keepalive.label: +"""Server Keep Alive""" + +global_authentication.desc: +"""Default authentication configs for all MQTT listeners. + +For per-listener overrides see authentication in listener configs + +This option can be configured with: +
    +
  • []: The default value, it allows *ALL* logins
    • +
  • one: For example {enable:true,backend:"built_in_database",mechanism="password_based"}
    • +
  • chain: An array of structs.
    • +
+ +When a chain is configured, the login credentials are checked against the backends per the configured order, until an 'allow' or 'deny' decision can be made. + +If there is no decision after a full chain exhaustion, the login is rejected.""" + +fields_mqtt_quic_listener_load_balancing_mode.desc: +"""0: Disabled, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. default: 0""" + +fields_mqtt_quic_listener_load_balancing_mode.label: +"""Load balancing mode""" + +persistent_session_store_session_message_gc_interval.desc: +"""The starting interval for garbage collection of transient data for +persistent session messages. This does not affect the lifetime length +of persistent session messages.""" + +persistent_session_store_session_message_gc_interval.label: +"""Session message GC interval""" + +server_ssl_opts_schema_ocsp_refresh_http_timeout.desc: +"""The timeout for the HTTP request when checking OCSP responses.""" + +server_ssl_opts_schema_ocsp_refresh_http_timeout.label: +"""OCSP Refresh HTTP Timeout""" + +fields_tcp_opts_send_timeout.desc: +"""The TCP send timeout for the connections.""" + +fields_tcp_opts_send_timeout.label: +"""TCP send timeout""" + +sysmon_vm_process_high_watermark.desc: +"""The threshold, as percentage of processes, for how many + processes can simultaneously exist at the local node before the corresponding + alarm is raised.""" + +sysmon_vm_process_high_watermark.label: +"""Process high watermark""" + +fields_tcp_opts_buffer.desc: +"""The size of the user-space buffer used by the driver.""" + +fields_tcp_opts_buffer.label: +"""TCP user-space buffer""" + +server_ssl_opts_schema_honor_cipher_order.desc: +"""An important security setting, it forces the cipher to be set based + on the server-specified order instead of the client-specified order, + hence enforcing the (usually more properly configured) security + ordering of the server administrator.""" + +server_ssl_opts_schema_honor_cipher_order.label: +"""SSL honor cipher order""" + +conn_congestion_min_alarm_sustain_duration.desc: +"""Minimal time before clearing the alarm.
The alarm is cleared only when there's no pending data in
the queue, and at least min_alarm_sustain_durationmilliseconds passed since the last time we considered the connection 'congested'.
This is to avoid clearing and raising the alarm again too often.""" + +conn_congestion_min_alarm_sustain_duration.label: +"""Sustain duration""" + +fields_mqtt_quic_listener_keep_alive_interval_ms.desc: +"""How often to send PING frames to keep a connection alive.""" + +fields_mqtt_quic_listener_keep_alive_interval_ms.label: +"""Keep alive interval ms""" + +fields_mqtt_quic_listener_handshake_idle_timeout_ms.desc: +"""How long a handshake can idle before it is discarded""" + +fields_mqtt_quic_listener_handshake_idle_timeout_ms.label: +"""Handshake idle timeout ms""" + +broker_session_locking_strategy.desc: +"""Session locking strategy in a cluster. + - `local`: only lock the session on the current node + - `one`: select only one remote node to lock the session + - `quorum`: select some nodes to lock the session + - `all`: lock the session on all the nodes in the cluster""" + +persistent_store_ram_cache.desc: +"""Maintain a copy of the data in RAM for faster access.""" + +persistent_store_ram_cache.label: +"""RAM cache""" + +fields_mqtt_quic_listener_stream_recv_window_default.desc: +"""Initial stream receive window size. Default: 32678""" + +fields_mqtt_quic_listener_stream_recv_window_default.label: +"""Stream recv window default""" + +mqtt_mqueue_priorities.desc: +"""Topic priorities. Priority number [1-255] +There's no priority table by default, hence all messages are treated equal. + +**NOTE**: Comma and equal signs are not allowed for priority topic names. +**NOTE**: Messages for topics not in the priority table are treated as either highest or lowest priority depending on the configured value for mqtt.mqueue_default_priority. + +**Examples**: +To configure "topic/1" > "topic/2": +mqueue_priorities: {"topic/1": 10, "topic/2": 8}""" + +mqtt_mqueue_priorities.label: +"""Topic Priorities""" + +fields_rate_limit_conn_messages_in.desc: +"""Message limit for the external MQTT connections.""" + +fields_rate_limit_conn_messages_in.label: +"""connecting messages in""" + +fields_rate_limit_max_conn_rate.desc: +"""Maximum connections per second.""" + +fields_rate_limit_max_conn_rate.label: +"""Max connection rate""" + +alarm_size_limit.desc: +"""The maximum total number of deactivated alarms to keep as history.
When this limit is exceeded, the oldest deactivated alarms are deleted to cap the total number.""" + +alarm_size_limit.label: +"""Alarm size limit""" + +fields_cache_max_size.desc: +"""Maximum number of cached items.""" + +fields_cache_max_size.label: +"""Maximum number of cached items.""" + +fields_listeners_tcp.desc: +"""TCP listeners.""" + +fields_listeners_tcp.label: +"""TCP listeners""" + +conn_congestion_enable_alarm.desc: +"""Enable or disable connection congestion alarm.""" + +conn_congestion_enable_alarm.label: +"""Enable/disable congestion alarm""" + +fields_ws_opts_proxy_port_header.desc: +"""HTTP header used to pass information about the client port. Relevant when the EMQX cluster is deployed behind a load-balancer.""" + +fields_ws_opts_proxy_port_header.label: +"""Proxy port header""" + +overload_protection_enable.desc: +"""React on system overload or not.""" + +overload_protection_enable.label: +"""React on system overload or not""" + +fields_mqtt_quic_listener_minimum_mtu.desc: +"""The minimum MTU supported by a connection. This will be used as the starting MTU. Default: 1248""" + +fields_mqtt_quic_listener_minimum_mtu.label: +"""Minimum MTU""" + +sys_msg_interval.desc: +"""Time interval of publishing `$SYS` messages.""" + +mqtt_await_rel_timeout.desc: +"""For client to broker QoS 2 message, the time limit for the broker to wait before the `PUBREL` message is received. The wait is aborted after timed out, meaning the packet ID is freed for new `PUBLISH` requests. Receiving a stale `PUBREL` causes a warning level log. Note, the message is delivered to subscribers before entering the wait for PUBREL.""" + +mqtt_await_rel_timeout.label: +"""Max Awaiting PUBREL TIMEOUT""" + +common_ssl_opts_schema_verify.desc: +"""Enable or disable peer verification.""" + +common_ssl_opts_schema_verify.label: +"""Verify peer""" + +fields_listeners_ssl.desc: +"""SSL listeners.""" + +fields_listeners_ssl.label: +"""SSL listeners""" + +fields_deflate_opts_client_max_window_bits.desc: +"""Specifies the size of the compression context for the client.""" + +fields_deflate_opts_client_max_window_bits.label: +"""Client compression max window size""" + +common_ssl_opts_schema_keyfile.desc: +"""PEM format private key file.""" + +common_ssl_opts_schema_keyfile.label: +"""Keyfile""" + +sysmon_os_cpu_high_watermark.desc: +"""The threshold, as percentage of system CPU load, + for how much system cpu can be used before the corresponding alarm is raised.""" + +sysmon_os_cpu_high_watermark.label: +"""CPU high watermark""" + +flapping_detect_window_time.desc: +"""The time window for flapping detection.""" + +flapping_detect_window_time.label: +"""Window time""" + +mqtt_mqueue_default_priority.desc: +"""Default topic priority, which will be used by topics not in Topic Priorities (mqueue_priorities).""" + +mqtt_mqueue_default_priority.label: +"""Default Topic Priorities""" + +client_ssl_opts_schema_enable.desc: +"""Enable TLS.""" + +client_ssl_opts_schema_enable.label: +"""Enable TLS.""" + +fields_mqtt_quic_listener_mtu_discovery_missing_probe_count.desc: +"""The maximum number of stateless operations that may be queued on a binding at any one time. Default: 3""" + +fields_mqtt_quic_listener_mtu_discovery_missing_probe_count.label: +"""MTU discovery missing probe count""" + +fields_tcp_opts_recbuf.desc: +"""The TCP receive buffer (OS kernel) for the connections.""" + +fields_tcp_opts_recbuf.label: +"""TCP receive buffer""" + +sysmon_vm_process_check_interval.desc: +"""The time interval for the periodic process limit check.""" + +sysmon_vm_process_check_interval.label: +"""Process limit check interval""" + +fields_mqtt_quic_listener_server_resumption_level.desc: +"""Controls resumption tickets and/or 0-RTT server support. Default: 0 (No resumption)""" + +fields_mqtt_quic_listener_server_resumption_level.label: +"""Server resumption level""" + +fields_ws_opts_proxy_address_header.desc: +"""HTTP header used to pass information about the client IP address. +Relevant when the EMQX cluster is deployed behind a load-balancer.""" + +fields_ws_opts_proxy_address_header.label: +"""Proxy address header""" + +sysmon_os_sysmem_high_watermark.desc: +"""The threshold, as percentage of system memory, + for how much system memory can be allocated before the corresponding alarm is raised.""" + +sysmon_os_sysmem_high_watermark.label: +"""SysMem high wartermark""" + +fields_tcp_opts_high_watermark.desc: +"""The socket is set to a busy state when the amount of data queued internally +by the VM socket implementation reaches this limit.""" + +fields_tcp_opts_high_watermark.label: +"""TCP 高水位线""" + +fields_mqtt_quic_listener_stateless_operation_expiration_ms.desc: +"""The time limit between operations for the same endpoint, in milliseconds. Default: 100""" + +fields_mqtt_quic_listener_stateless_operation_expiration_ms.label: +"""Stateless operation expiration ms""" + +server_ssl_opts_schema_dhfile.desc: +"""Path to a file containing PEM-encoded Diffie-Hellman parameters to be used by the server if a cipher suite using Diffie-Hellman key exchange is negotiated. If not specified, default parameters are used.
NOTE: The dhfile option is not supported by TLS 1.3.""" - zh: """如果协商使用Diffie-Hellman密钥交换的密码套件，则服务器将使用包含PEM编码的Diffie-Hellman参数的文件的路径。如果未指定，则使用默认参数。
-注意：TLS 1.3不支持dhfile选项。""" - } - label: { - en: "SSL dhfile" - zh: "SSL dhfile" - } -} -server_ssl_opts_schema_fail_if_no_peer_cert { - desc { - en: """Used together with {verify, verify_peer} by an TLS/DTLS server. +server_ssl_opts_schema_dhfile.label: +"""SSL dhfile""" + +flapping_detect_max_count.desc: +"""The maximum number of disconnects allowed for a MQTT Client in `window_time`""" + +flapping_detect_max_count.label: +"""Max count""" + +mqtt_max_topic_levels.desc: +"""Maximum topic levels allowed.""" + +mqtt_max_topic_levels.label: +"""Max Topic Levels""" + +force_shutdown_max_heap_size.desc: +"""Total heap size""" + +force_shutdown_max_heap_size.label: +"""Total heap size""" + +persistent_store_on_disc.desc: +"""Save information about the persistent sessions on disc. +If this option is enabled, persistent sessions will survive full restart of the cluster. +Otherwise, all the data will be stored in RAM, and it will be lost when all the nodes in the cluster are stopped.""" + +persistent_store_on_disc.label: +"""Persist on disc""" + +mqtt_ignore_loop_deliver.desc: +"""Whether the messages sent by the MQTT v3.1.1/v3.1.0 client will be looped back to the publisher itself, similar to No Local in MQTT 5.0.""" + +mqtt_ignore_loop_deliver.label: +"""Ignore Loop Deliver""" + +common_ssl_opts_schema_certfile.desc: +"""PEM format certificates chain file.
+The certificates in this file should be in reversed order of the certificate +issue chain. That is, the host's certificate should be placed in the beginning +of the file, followed by the immediate issuer certificate and so on. +Although the root CA certificate is optional, it should be placed at the end of +the file if it is to be added.""" + +common_ssl_opts_schema_certfile.label: +"""Certfile""" + +mqtt_exclusive_subscription.desc: +"""Whether to enable support for MQTT exclusive subscription.""" + +mqtt_exclusive_subscription.label: +"""Exclusive Subscription""" + +mqtt_retain_available.desc: +"""Whether to enable support for MQTT retained message.""" + +mqtt_retain_available.label: +"""Retain Available""" + +fields_tcp_opts_reuseaddr.desc: +"""The SO_REUSEADDR flag for the connections.""" + +fields_tcp_opts_reuseaddr.label: +"""SO_REUSEADDR""" + +sysmon_vm_long_schedule.desc: +"""When the Erlang VM detect a task scheduled for too long, a warning level 'long_schedule' log is emitted, +and an MQTT message is published to the system topic $SYS/sysmon/long_schedule.""" + +sysmon_vm_long_schedule.label: +"""Enable Long Schedule monitoring.""" + +mqtt_keepalive_backoff.desc: +"""The coefficient EMQX uses to confirm whether the keep alive duration of the client expires. Formula: Keep Alive * Backoff * 2""" + +mqtt_keepalive_backoff.label: +"""Keep Alive Backoff""" + +force_gc_bytes.desc: +"""GC the process after specified number of bytes have passed through.""" + +force_gc_bytes.label: +"""Process GC bytes""" + +server_ssl_opts_schema_fail_if_no_peer_cert.desc: +"""Used together with {verify, verify_peer} by an TLS/DTLS server. If set to true, the server fails if the client does not have a certificate to send, that is, sends an empty certificate. If set to false, it fails only if the client sends an invalid certificate (an empty certificate is considered valid).""" - zh: """TLS/DTLS 服务器与 {verify，verify_peer} 一起使用。 -如果设置为true，则如果客户端没有要发送的证书，即发送空证书，服务器将失败。 -如果设置为false，则仅当客户端发送无效证书（空证书被视为有效证书）时才会失败。""" - } - label: { - en: "SSL fail if no peer cert" - zh: "没有证书则 SSL 失败" - } -} -server_ssl_opts_schema_honor_cipher_order { - desc { - en: """An important security setting, it forces the cipher to be set based - on the server-specified order instead of the client-specified order, - hence enforcing the (usually more properly configured) security - ordering of the server administrator.""" - zh: """一个重要的安全设置，它强制根据服务器指定的顺序而不是客户机指定的顺序设置密码，从而强制服务器管理员执行（通常配置得更正确）安全顺序。""" - } - label: { - en: "SSL honor cipher order" - zh: "SSL honor cipher order" - } -} +server_ssl_opts_schema_fail_if_no_peer_cert.label: +"""SSL fail if no peer cert""" -server_ssl_opts_schema_client_renegotiation { - desc { - en: """In protocols that support client-initiated renegotiation, +fields_ws_opts_compress.desc: +"""If true, compress WebSocket messages using zlib.
+The configuration items under deflate_opts belong to the compression-related parameter configuration.""" + +fields_ws_opts_compress.label: +"""Ws compress""" + +fields_mqtt_quic_listener_keep_alive_interval.desc: +"""How often to send PING frames to keep a connection alive. 0 means disabled.""" + +fields_mqtt_quic_listener_keep_alive_interval.label: +"""Keep Alive Interval""" + +fields_cache_ttl.desc: +"""Time to live for the cached data.""" + +fields_cache_ttl.label: +"""Time to live for the cached data.""" + +sys_topics.desc: +"""System topics configuration.""" + +sys_event_client_subscribed.desc: +"""Enable to publish event message that client subscribed a topic successfully.""" + +sysmon_top_db_port.desc: +"""Port of the PostgreSQL database that collects the data points.""" + +sysmon_top_db_port.label: +"""DB Port""" + +fields_mqtt_quic_listener_max_operations_per_drain.desc: +"""The maximum number of operations to drain per connection quantum. Default: 16""" + +fields_mqtt_quic_listener_max_operations_per_drain.label: +"""Max operations per drain""" + +fields_mqtt_quic_listener_datagram_receive_enabled.desc: +"""Advertise support for QUIC datagram extension. Reserve for the future. Default 0 (FALSE)""" + +fields_mqtt_quic_listener_datagram_receive_enabled.label: +"""Datagram receive enabled""" + +fields_mqtt_quic_listener_initial_rtt_ms.desc: +"""Initial RTT estimate.""" + +fields_mqtt_quic_listener_initial_rtt_ms.label: +"""Initial RTT ms""" + +overload_protection_backoff_gc.desc: +"""When at high load, skip forceful GC.""" + +overload_protection_backoff_gc.label: +"""Skip GC""" + +broker_perf_route_lock_type.desc: +"""Performance tuning for subscribing/unsubscribing a wildcard topic. +Change this parameter only when there are many wildcard topics. + +NOTE: when changing from/to `global` lock, it requires all nodes in the cluster to be stopped before the change. + - `key`: mnesia transactional updates with per-key locks. Recommended for a single-node setup. + - `tab`: mnesia transactional updates with table lock. Recommended for a cluster setup. + - `global`: updates are protected with a global lock. Recommended for large clusters.""" + +fields_tcp_opts_nodelay.desc: +"""The TCP_NODELAY flag for the connections.""" + +fields_tcp_opts_nodelay.label: +"""TCP_NODELAY""" + +sysmon_top_db_username.desc: +"""Username of the PostgreSQL database""" + +sysmon_top_db_username.label: +"""DB Username""" + +broker.desc: +"""Message broker options.""" + +force_gc_count.desc: +"""GC the process after this many received messages.""" + +force_gc_count.label: +"""Process GC messages num""" + +mqtt_max_clientid_len.desc: +"""Maximum allowed length of MQTT Client ID.""" + +mqtt_max_clientid_len.label: +"""Max Client ID Length""" + +fields_ws_opts_supported_subprotocols.desc: +"""Comma-separated list of supported subprotocols.""" + +fields_ws_opts_supported_subprotocols.label: +"""Supported subprotocols""" + +broker_shared_subscription_strategy.desc: +"""Dispatch strategy for shared subscription. + - `random`: dispatch the message to a random selected subscriber + - `round_robin`: select the subscribers in a round-robin manner + - `round_robin_per_group`: select the subscribers in round-robin fashion within each shared subscriber group + - `local`: select random local subscriber otherwise select random cluster-wide + - `sticky`: always use the last selected subscriber to dispatch, until the subscriber disconnects. + - `hash_clientid`: select the subscribers by hashing the `clientIds` + - `hash_topic`: select the subscribers by hashing the source topic""" + +fields_deflate_opts_mem_level.desc: +"""Specifies the size of the compression state.
+Lower values decrease memory usage per connection.""" + +fields_deflate_opts_mem_level.label: +"""Size of the compression state""" + +fields_mqtt_quic_listener_send_idle_timeout_ms.desc: +"""Reset congestion control after being idle for amount of time. Default: 1000""" + +fields_mqtt_quic_listener_send_idle_timeout_ms.label: +"""Send idle timeout ms""" + +base_listener_limiter.desc: +"""Type of the rate limit.""" + +base_listener_limiter.label: +"""Type of the rate limit.""" + +persistent_session_store_backend.desc: +"""Database management system used to store information about persistent sessions and messages. +- `builtin`: Use the embedded database (mria)""" + +persistent_session_store_backend.label: +"""Backend""" + +alarm_validity_period.desc: +"""Retention time of deactivated alarms. Alarms are not deleted immediately +when deactivated, but after the retention time.""" + +alarm_validity_period.label: +"""Alarm validity period""" + +server_ssl_opts_schema_ocsp_issuer_pem.desc: +"""PEM-encoded certificate of the OCSP issuer for the server certificate.""" + +server_ssl_opts_schema_ocsp_issuer_pem.label: +"""OCSP Issuer Certificate""" + +fields_tcp_opts_active_n.desc: +"""Specify the {active, N} option for this Socket.
+See: https://erlang.org/doc/man/inet.html#setopts-2""" + +fields_tcp_opts_active_n.label: +"""active_n""" + +listener_authentication.desc: +"""Per-listener authentication override. +Authentication can be one single authenticator instance or a chain of authenticators as an array. +When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.""" + +listener_authentication.label: +"""Per-listener authentication override""" + +fields_trace_payload_encode.desc: +"""Determine the format of the payload format in the trace file.
+`text`: Text-based protocol or plain text protocol. + It is recommended when payload is JSON encoded.
+`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
+`hidden`: payload is obfuscated as `******`""" + +fields_trace_payload_encode.label: +"""Payload encode""" + +mqtt_response_information.desc: +"""UTF-8 string, for creating the response topic, for example, if set to reqrsp/, the publisher/subscriber will communicate using the topic prefix reqrsp/. +To disable this feature, input "" in the text box below. Only applicable to MQTT 5.0 clients.""" + +mqtt_response_information.label: +"""Response Information""" + +persistent_session_store_max_retain_undelivered.desc: +"""The time messages that was not delivered to a persistent session +is stored before being garbage collected if the node the previous +session was handled on restarts of is stopped.""" + +persistent_session_store_max_retain_undelivered.label: +"""Max retain undelivered""" + +fields_mqtt_quic_listener_migration_enabled.desc: +"""Enable clients to migrate IP addresses and tuples. Requires a cooperative load-balancer, or no load-balancer. Default: 1 (Enabled)""" + +fields_mqtt_quic_listener_migration_enabled.label: +"""Migration enabled""" + +common_ssl_opts_schema_password.desc: +"""String containing the user's password. Only used if the private key file is password-protected.""" + +common_ssl_opts_schema_password.label: +"""Keyfile passphrase""" + +common_ssl_opts_schema_hibernate_after.desc: +"""Hibernate the SSL process after idling for amount of time reducing its memory footprint.""" + +common_ssl_opts_schema_hibernate_after.label: +"""hibernate after""" + +fields_mqtt_quic_listener_send_buffering_enabled.desc: +"""Buffer send data instead of holding application buffers until sent data is acknowledged. Default: 1 (Enabled)""" + +fields_mqtt_quic_listener_send_buffering_enabled.label: +"""Send buffering enabled""" + +sys_event_client_unsubscribed.desc: +"""Enable to publish event message that client unsubscribed a topic successfully.""" + +overload_protection_backoff_new_conn.desc: +"""When at high load, close new incoming connections.""" + +overload_protection_backoff_new_conn.label: +"""Close new connections""" + +server_ssl_opts_schema_ocsp_responder_url.desc: +"""URL for the OCSP responder to check the server certificate against.""" + +server_ssl_opts_schema_ocsp_responder_url.label: +"""OCSP Responder URL""" + +mqtt_idle_timeout.desc: +"""Configure the duration of time that a connection can remain idle (i.e., without any data transfer) before being: + - Automatically disconnected if no CONNECT package is received from the client yet. + - Put into hibernation mode to save resources if some CONNECT packages are already received. +Note: Please set the parameter with caution as long idle time will lead to resource waste.""" + +mqtt_idle_timeout.label: +"""Idle Timeout""" + +fields_mqtt_quic_listener_conn_flow_control_window.desc: +"""Connection-wide flow control window. Default: 16777216""" + +fields_mqtt_quic_listener_conn_flow_control_window.label: +"""Conn flow control window""" + +fields_mqtt_quic_listener_maximum_mtu.desc: +"""The maximum MTU supported by a connection. This will be the maximum probed value. Default: 1500""" + +fields_mqtt_quic_listener_maximum_mtu.label: +"""Maximum MTU""" + +sysmon_top_db_name.desc: +"""PostgreSQL database name""" + +sysmon_top_db_name.label: +"""DB Name""" + +mqtt_strict_mode.desc: +"""Whether to parse MQTT messages in strict mode. +In strict mode, invalid utf8 strings in for example client ID, topic name, etc. will cause the client to be disconnected.""" + +mqtt_strict_mode.label: +"""Strict Mode""" + +shared_subscription_group_strategy.desc: +"""Per group dispatch strategy for shared subscription. +This config is a map from shared subscription group name to the strategy +name. The group name should be of format `[A-Za-z0-9]`. i.e. no +special characters are allowed.""" + +fields_deflate_opts_strategy.desc: +"""Specifies the compression strategy.""" + +fields_deflate_opts_strategy.label: +"""compression strategy""" + +shared_subscription_strategy_enum.desc: +"""Dispatch strategy for shared subscription. +- `random`: dispatch the message to a random selected subscriber +- `round_robin`: select the subscribers in a round-robin manner +- `round_robin_per_group`: select the subscribers in round-robin fashion within each shared subscriber group +- `sticky`: always use the last selected subscriber to dispatch, +until the subscriber disconnects. +- `hash`: select the subscribers by the hash of `clientIds` +- `local`: send to a random local subscriber. If local +subscriber was not found, send to a random subscriber cluster-wide""" + +persistent_session_builtin_sess_msg_table.desc: +"""Performance tuning options for built-in session messages table.""" + +persistent_session_builtin_sess_msg_table.label: +"""Persistent session messages""" + +mqtt_mqueue_store_qos0.desc: +"""Specifies whether to store QoS 0 messages in the message queue while the connection is down but the session remains.""" + +mqtt_mqueue_store_qos0.label: +"""Store QoS 0 Message""" + +server_ssl_opts_schema_client_renegotiation.desc: +"""In protocols that support client-initiated renegotiation, the cost of resources of such an operation is higher for the server than the client. This can act as a vector for denial of service attacks. The SSL application already takes measures to counter-act such attempts, @@ -1722,1147 +1210,331 @@ but client-initiated renegotiation can be strictly disabled by setting this opti The default value is true. Note that disabling renegotiation can result in long-lived connections becoming unusable due to limits on the number of messages the underlying cipher suite can encipher.""" - zh: """在支持客户机发起的重新协商的协议中，这种操作的资源成本对于服务器来说高于客户机。 -这可能会成为拒绝服务攻击的载体。 -SSL 应用程序已经采取措施来反击此类尝试，但通过将此选项设置为 false，可以严格禁用客户端发起的重新协商。 -默认值为 true。请注意，由于基础密码套件可以加密的消息数量有限，禁用重新协商可能会导致长期连接变得不可用。""" - } - label: { - en: "SSL client renegotiation" - zh: "SSL 客户端冲协商" - } -} - -server_ssl_opts_schema_handshake_timeout { - desc { - en: """Maximum time duration allowed for the handshake to complete""" - zh: """握手完成所允许的最长时间""" - } - label: { - en: "Handshake timeout" - zh: "握手超时时间" - } -} - -server_ssl_opts_schema_gc_after_handshake { - desc { - en: """Memory usage tuning. If enabled, will immediately perform a garbage collection after the TLS/SSL handshake.""" - zh: """内存使用调优。如果启用，将在TLS/SSL握手完成后立即执行垃圾回收。TLS/SSL握手建立后立即进行GC。""" - } - label: { - en: "Perform GC after handshake" - zh: "握手后执行GC" - } -} - -server_ssl_opts_schema_enable_ocsp_stapling { - desc { - en: "Whether to enable Online Certificate Status Protocol (OCSP) stapling for the listener." - " If set to true, requires defining the OCSP responder URL and issuer PEM path." - zh: "是否为监听器启用 OCSP Stapling 功能。 如果设置为 true，" - "需要定义 OCSP Responder 的 URL 和证书签发者的 PEM 文件路径。" - } - label: { - en: "Enable OCSP Stapling" - zh: "启用 OCSP Stapling" - } -} - -server_ssl_opts_schema_ocsp_responder_url { - desc { - en: "URL for the OCSP responder to check the server certificate against." - zh: "用于检查服务器证书的 OCSP Responder 的 URL。" - } - label: { - en: "OCSP Responder URL" - zh: "OCSP Responder 的 URL" - } -} - -server_ssl_opts_schema_ocsp_issuer_pem { - desc { - en: "PEM-encoded certificate of the OCSP issuer for the server certificate." - zh: "服务器证书的 OCSP 签发者的 PEM 编码证书。" - } - label: { - en: "OCSP Issuer Certificate" - zh: "OCSP 签发者证书" - } -} - -server_ssl_opts_schema_ocsp_refresh_interval { - desc { - en: "The period to refresh the OCSP response for the server." - zh: "为服务器刷新OCSP响应的周期。" - } - label: { - en: "OCSP Refresh Interval" - zh: "OCSP 刷新间隔" - } -} - -server_ssl_opts_schema_ocsp_refresh_http_timeout { - desc { - en: "The timeout for the HTTP request when checking OCSP responses." - zh: "检查 OCSP 响应时，HTTP 请求的超时。" - } - label: { - en: "OCSP Refresh HTTP Timeout" - zh: "OCSP 刷新 HTTP 超时" - } -} - -server_ssl_opts_schema_enable_crl_check { - desc { - en: "Whether to enable CRL verification for this listener." - zh: "是否为该监听器启用 CRL 检查。" - } - label: { - en: "Enable CRL Check" - zh: "启用 CRL 检查" - } -} - -crl_cache_refresh_http_timeout { - desc { - en: "The timeout for the HTTP request when fetching CRLs. This is" - " a global setting for all listeners." - zh: "获取 CRLs 时 HTTP 请求的超时。 该配置对所有启用 CRL 检查的监听器监听器有效。" - } - label: { - en: "CRL Cache Refresh HTTP Timeout" - zh: "CRL 缓存刷新 HTTP 超时" - } -} - -crl_cache_refresh_interval { - desc { - en: "The period to refresh the CRLs from the servers. This is a global setting" - " for all URLs and listeners." - zh: "从服务器刷新CRL的周期。 该配置对所有 URL 和监听器有效。" - } - label: { - en: "CRL Cache Refresh Interval" - zh: "CRL 缓存刷新间隔" - } -} - -crl_cache_capacity { - desc { - en: "The maximum number of CRL URLs that can be held in cache. If the cache is at" - " full capacity and a new URL must be fetched, then it'll evict the oldest" - " inserted URL in the cache." - zh: "缓存中可容纳的 CRL URL 的最大数量。" - " 如果缓存的容量已满，并且必须获取一个新的 URL，" - "那么它将驱逐缓存中插入的最老的 URL。" - } - label: { - en: "CRL Cache Capacity" - zh: "CRL 缓存容量" - } -} - -fields_listeners_tcp { - desc { - en: """TCP listeners.""" - zh: """TCP 监听器。""" - } - label: { - en: "TCP listeners" - zh: "TCP 监听器" - } -} - -fields_listeners_ssl { - desc { - en: """SSL listeners.""" - zh: """SSL 监听器。""" - } - label: { - en: "SSL listeners" - zh: "SSL 监听器" - } -} - -fields_listeners_ws { - desc { - en: """HTTP websocket listeners.""" - zh: """HTTP websocket 监听器。""" - } - label: { - en: "HTTP websocket listeners" - zh: "HTTP websocket 监听器" - } -} - -fields_listeners_wss { - desc { - en: """HTTPS websocket listeners.""" - zh: """HTTPS websocket 监听器。""" - } - label: { - en: "HTTPS websocket listeners" - zh: "HTTPS websocket 监听器" - } -} - -fields_listeners_quic { - desc { - en: """QUIC listeners.""" - zh: """QUIC 监听器。""" - } - label: { - en: "QUIC listeners" - zh: "QUIC 监听器" - } -} - -fields_listener_enabled { - desc { - en: """Enable listener.""" - zh: """启停监听器。""" - } - label: { - en: "Enable listener" - zh: "启停监听器" - } -} - -fields_mqtt_quic_listener_certfile { - desc { - en: """Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead.""" - zh: """证书文件。在 5.1 中会被废弃，使用 .ssl_options.certfile 代替。""" - } - label: { - en: "Certificate file" - zh: "证书文件" - } -} - -fields_mqtt_quic_listener_keyfile { - desc { - en: """Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead.""" - zh: """私钥文件。在 5.1 中会被废弃，使用 .ssl_options.keyfile 代替。""" - } - label: { - en: "Key file" - zh: "私钥文件" - } -} - -fields_mqtt_quic_listener_idle_timeout { - desc { - en: """How long a connection can go idle before it is gracefully shut down. 0 to disable""" - zh: """一个连接在被关闭之前可以空闲多长时间。0表示禁用。""" - } - label: { - en: "Idle Timeout" - zh: "空闲超时时间" - } -} - -fields_mqtt_quic_listener_handshake_idle_timeout { - desc { - en: """How long a handshake can idle before it is discarded.""" - zh: """一个握手在被丢弃之前可以空闲多长时间。""" - } - label: { - en: "Handshake Idle Timeout" - zh: "握手空闲超时时间" - } -} - -fields_mqtt_quic_listener_keep_alive_interval { - desc { - en: """How often to send PING frames to keep a connection alive. 0 means disabled.""" - zh: """发送 PING 帧的频率，以保活连接. 设为 0 表示禁用。""" - } - label: { - en: "Keep Alive Interval" - zh: "PING 保活频率" - } -} - -fields_mqtt_quic_listener_ssl_options { - desc { - en: """TLS options for QUIC transport""" - zh: """QUIC 传输层的 TLS 选项""" - } - label: { - en: "TLS Options" - zh: "TLS 选项" - } -} - -base_listener_bind { - desc { - en: """IP address and port for the listening socket.""" - zh: """监听套接字的 IP 地址和端口。""" - } - label: { - en: "IP address and port" - zh: "IP 地址和端口" - } -} - -base_listener_acceptors { - desc { - en: """The size of the listener's receiving pool.""" - zh: """监听器接收池的大小。""" - } - label: { - en: "Acceptors Num" - zh: "接收器数量" - } -} - -fields_mqtt_quic_listener_max_bytes_per_key { - desc { - en: "Maximum number of bytes to encrypt with a single 1-RTT encryption key before initiating key update. Default: 274877906944" - zh: "在启动密钥更新之前，用单个 1-RTT 加密密钥加密的最大字节数。默认值：274877906944" - } - label { - en: "Max bytes per key" - zh: "每个密钥的最大字节数" - } -} - -fields_mqtt_quic_listener_handshake_idle_timeout_ms { - desc { - en: "How long a handshake can idle before it is discarded. Default: 10 000" - zh: "一个握手在被丢弃之前可以空闲多长时间。 默认值：10 000" - } - label { - en: "Handshake idle timeout ms" - zh: "握手空闲超时毫秒" - } -} - -fields_mqtt_quic_listener_tls_server_max_send_buffer { - desc { - en: "How much Server TLS data to buffer. Default: 8192" - zh: "缓冲多少TLS数据。 默认值：8192" - } - label { - en: "TLS server max send buffer" - zh: "TLS 服务器最大发送缓冲区" - } -} - -fields_mqtt_quic_listener_stream_recv_window_default { - desc { - en: "Initial stream receive window size. Default: 32678" - zh: "初始流接收窗口大小。 默认值：32678" - } - label { - en: "Stream recv window default" - zh: "流接收窗口默认" - } -} - -fields_mqtt_quic_listener_stream_recv_buffer_default { - desc { - en: "Stream initial buffer size. Default: 4096" - zh: "流的初始缓冲区大小。默认：4096" - } - label { - en: "Stream recv buffer default" - zh: "流媒体接收缓冲区默认值" - } -} - -fields_mqtt_quic_listener_conn_flow_control_window { - desc { - en: "Connection-wide flow control window. Default: 16777216" - zh: "连接的流控窗口。默认：16777216" - } - label { - en: "Conn flow control window" - zh: "流控窗口" - } -} - -fields_mqtt_quic_listener_max_stateless_operations { - desc { - en: "The maximum number of stateless operations that may be queued on a worker at any one time. Default: 16" - zh: "无状态操作的最大数量，在任何时候都可以在一个工作者上排队。默认值：16" - } - label { - en: "Max stateless operations" - zh: "最大无状态操作数" - } -} - -fields_mqtt_quic_listener_initial_window_packets { - desc { - en: "The size (in packets) of the initial congestion window for a connection. Default: 10" - zh: "一个连接的初始拥堵窗口的大小（以包为单位）。默认值：10" - } - label { - en: "Initial window packets" - zh: "初始窗口数据包" - } -} - -fields_mqtt_quic_listener_send_idle_timeout_ms { - desc { - en: "Reset congestion control after being idle for amount of time. Default: 1000" - zh: "在闲置一定时间后重置拥堵控制。默认值：1000" - } - label { - en: "Send idle timeout ms" - zh: "发送空闲超时毫秒" - } -} - -fields_mqtt_quic_listener_initial_rtt_ms { - desc { - en: "Initial RTT estimate." - zh: "初始RTT估计" - } - label { - en: "Initial RTT ms" - zh: "Initial RTT 毫秒" - } -} - -fields_mqtt_quic_listener_max_ack_delay_ms { - desc { - en: "How long to wait after receiving data before sending an ACK. Default: 25" - zh: "在收到数据后要等待多长时间才能发送一个ACK。默认值：25" - } - label { - en: "Max ack delay ms" - zh: "最大应答延迟 毫秒" - } -} - -fields_mqtt_quic_listener_disconnect_timeout_ms { - desc { - en: "How long to wait for an ACK before declaring a path dead and disconnecting. Default: 16000" - zh: "在判定路径无效和断开连接之前，要等待多长时间的ACK。默认：16000" - } - label { - en: "Disconnect timeout ms" - zh: "断开连接超时 毫秒" - } -} - -fields_mqtt_quic_listener_idle_timeout_ms { - desc { - en: "How long a connection can go idle before it is gracefully shut down. 0 to disable timeout" - zh: "一个连接在被优雅地关闭之前可以空闲多长时间。0 表示禁用超时" - } - label { - en: "Idle timeout ms" - zh: "空闲超时 毫秒" - } -} - -fields_mqtt_quic_listener_handshake_idle_timeout_ms { - desc { - en: "How long a handshake can idle before it is discarded" - zh: "一个握手在被丢弃之前可以空闲多长时间" - } - label { - en: "Handshake idle timeout ms" - zh: "握手空闲超时 毫秒" - } -} - -fields_mqtt_quic_listener_keep_alive_interval_ms { - desc { - en: "How often to send PING frames to keep a connection alive." - zh: "多长时间发送一次PING帧以保活连接。" - } - label { - en: "Keep alive interval ms" - zh: "保持活着的时间间隔 毫秒" - } -} - -fields_mqtt_quic_listener_peer_bidi_stream_count { - desc { - en: "Number of bidirectional streams to allow the peer to open." - zh: "允许对端打开的双向流的数量" - } - label { - en: "Peer bidi stream count" - zh: "对端双向流的数量" - } -} - -fields_mqtt_quic_listener_peer_unidi_stream_count { - desc { - en: "Number of unidirectional streams to allow the peer to open." - zh: "允许对端打开的单向流的数量" - } - label { - en: "Peer unidi stream count" - zh: "对端单向流的数量" - } -} - -fields_mqtt_quic_listener_retry_memory_limit { - desc { - en: "The percentage of available memory usable for handshake connections before stateless retry is used. Calculated as `N/65535`. Default: 65" - zh: "在使用无状态重试之前，可用于握手连接的可用内存的百分比。计算为`N/65535`。默认值：65" - } - label { - en: "Retry memory limit" - zh: "重试内存限制" - } -} - -fields_mqtt_quic_listener_load_balancing_mode { - desc { - en: "0: Disabled, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. default: 0" - zh: "0: 禁用, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. 默认: 0" - } - label { - en: "Load balancing mode" - zh: "负载平衡模式" - } -} - -fields_mqtt_quic_listener_max_operations_per_drain { - desc { - en: "The maximum number of operations to drain per connection quantum. Default: 16" - zh: "每个连接操作的最大耗费操作数。默认：16" - } - label { - en: "Max operations per drain" - zh: "每次操作最大操作数" - } -} - -fields_mqtt_quic_listener_send_buffering_enabled { - desc { - en: "Buffer send data instead of holding application buffers until sent data is acknowledged. Default: 1 (Enabled)" - zh: "缓冲发送数据，而不是保留应用缓冲区，直到发送数据被确认。默认值：1（启用）" - } - label { - en: "Send buffering enabled" - zh: "启用发送缓冲功能" - } -} - -fields_mqtt_quic_listener_pacing_enabled { - desc { - en: "Pace sending to avoid overfilling buffers on the path. Default: 1 (Enabled)" - zh: "有节奏的发送，以避免路径上的缓冲区过度填充。默认值：1（已启用）" - } - label { - en: "Pacing enabled" - zh: "启用节奏发送" - } -} - -fields_mqtt_quic_listener_migration_enabled { - desc { - en: "Enable clients to migrate IP addresses and tuples. Requires a cooperative load-balancer, or no load-balancer. Default: 1 (Enabled)" - zh: "开启客户端地址迁移功能。需要一个支持的负载平衡器，或者没有负载平衡器。默认值：1（已启用）" - } - label { - en: "Migration enabled" - zh: "启用地址迁移" - } -} - -fields_mqtt_quic_listener_datagram_receive_enabled { - desc { - en: "Advertise support for QUIC datagram extension. Reserve for the future. Default 0 (FALSE)" - zh: "宣传对QUIC Datagram 扩展的支持。为将来保留。默认为0（FALSE）" - } - label { - en: "Datagram receive enabled" - zh: "启用 Datagram 接收" - } -} - -fields_mqtt_quic_listener_server_resumption_level { - desc { - en: "Controls resumption tickets and/or 0-RTT server support. Default: 0 (No resumption)" - zh: "连接恢复 和/或 0-RTT 服务器支持。默认值：0（无恢复功能）" - } - label { - en: "Server resumption level" - zh: "服务端连接恢复支持" - } -} - -fields_mqtt_quic_listener_minimum_mtu { - desc { - en: "The minimum MTU supported by a connection. This will be used as the starting MTU. Default: 1248" - zh: "一个连接所支持的最小MTU。这将被作为起始MTU使用。默认值：1248" - } - label { - en: "Minimum MTU" - zh: "最小 MTU" - } -} - -fields_mqtt_quic_listener_maximum_mtu { - desc { - en: "The maximum MTU supported by a connection. This will be the maximum probed value. Default: 1500" - zh: "一个连接所支持的最大MTU。这将是最大的探测值。默认值：1500" - } - label { - en: "Maximum MTU" - zh: "最大 MTU" - } -} - -fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us { - desc { - en: "The time in microseconds to wait before reattempting MTU probing if max was not reached. Default: 600000000" - zh: "如果没有达到 max ，在重新尝试 MTU 探测之前要等待的时间，单位是微秒。默认值：600000000" - } - label { - en: "MTU discovery search complete timeout us" - zh: "" - } -} - -fields_mqtt_quic_listener_mtu_discovery_missing_probe_count { - desc { - en: "The maximum number of stateless operations that may be queued on a binding at any one time. Default: 3" - zh: "在任何时候都可以在一个绑定上排队的无状态操作的最大数量。默认值：3" - } - label { - en: "MTU discovery missing probe count" - zh: "MTU发现丢失的探针数量" - } -} - -fields_mqtt_quic_listener_max_binding_stateless_operations { - desc { - en: "The maximum number of stateless operations that may be queued on a binding at any one time. Default: 100" - zh: "在任何时候可以在一个绑定上排队的无状态操作的最大数量。默认值：100" - } - label { - en: "Max binding stateless operations" - zh: "最大绑定无状态操作" - } -} - -fields_mqtt_quic_listener_stateless_operation_expiration_ms { - desc { - en: "The time limit between operations for the same endpoint, in milliseconds. Default: 100" - zh: "同一个对端的操作之间的时间限制，单位是毫秒。 默认：100" - } - label { - en: "Stateless operation expiration ms" - zh: "无状态操作过期 毫秒" - } -} - -base_listener_max_connections { - desc { - en: """The maximum number of concurrent connections allowed by the listener.""" - zh: """监听器允许的最大并发连接数。""" - } - label: { - en: "Max connections" - zh: "最大并发连接数" - } -} - -base_listener_mountpoint { - desc { - en: """When publishing or subscribing, prefix all topics with a mountpoint string. -The prefixed string will be removed from the topic name when the message -is delivered to the subscriber. The mountpoint is a way that users can use -to implement isolation of message routing between different listeners. -For example if a client A subscribes to `t` with `listeners.tcp.\.mountpoint` -set to `some_tenant`, then the client actually subscribes to the topic -`some_tenant/t`. Similarly, if another client B (connected to the same listener -as the client A) sends a message to topic `t`, the message is routed -to all the clients subscribed `some_tenant/t`, so client A will receive the -message, with topic name `t`.
-Set to `""` to disable the feature.
- -Variables in mountpoint string: - - ${clientid}: clientid - - ${username}: username""" - zh: """发布或订阅时，请在所有主题前面加上 mountpoint 字符串。 - -将消息传递给订阅者时，将从主题名称中删除带前缀的字符串。挂载点是一种用户可以用来实现不同侦听器之间消息路由隔离的方法。 - -例如，如果客户机 A 使用 listeners.tcp.\.mountpoint 设置为'some_tenant'，那么客户端实际上订阅了主题'some_tenant/t'。
-类似地，如果另一个客户端B（与客户端A连接到同一个侦听器）向主题 't' 发送消息，该消息将路由到所有订阅了'some_租户/t'的客户端，因此客户端 A 将接收主题名为't'的消息
- -设置为"" 以禁用该功能
- -mountpoint 字符串中的变量： -- ${clientid}: clientid -- ${username}: username""" - } - label: { - en: "mountpoint" - zh: "mountpoint" - } -} - -base_listener_zone { - desc { - en: """The configuration zone to which the listener belongs.""" - zh: """监听器所属的配置组。""" - } - label: { - en: "Zone" - zh: "配置组" - } -} - -base_listener_limiter { - desc { - en: """Type of the rate limit.""" - zh: """速率限制类型""" - } - label: { - en: "Type of the rate limit." - zh: "速率限制类型" - } -} - -base_listener_enable_authn { - desc { - en: """Set true (default) to enable client authentication on this listener, the authentication + +server_ssl_opts_schema_client_renegotiation.label: +"""SSL client renegotiation""" + +server_ssl_opts_schema_enable_crl_check.desc: +"""Whether to enable CRL verification for this listener.""" + +server_ssl_opts_schema_enable_crl_check.label: +"""Enable CRL Check""" + +fields_mqtt_quic_listener_peer_bidi_stream_count.desc: +"""Number of bidirectional streams to allow the peer to open.""" + +fields_mqtt_quic_listener_peer_bidi_stream_count.label: +"""Peer bidi stream count""" + +fields_mqtt_quic_listener_max_stateless_operations.desc: +"""The maximum number of stateless operations that may be queued on a worker at any one time. Default: 16""" + +fields_mqtt_quic_listener_max_stateless_operations.label: +"""Max stateless operations""" + +fields_ws_opts_idle_timeout.desc: +"""Close transport-layer connections from the clients that have not sent MQTT CONNECT message within this interval.""" + +fields_ws_opts_idle_timeout.label: +"""WS idle timeout""" + +fields_mqtt_quic_listener_max_ack_delay_ms.desc: +"""How long to wait after receiving data before sending an ACK. Default: 25""" + +fields_mqtt_quic_listener_max_ack_delay_ms.label: +"""Max ack delay ms""" + +base_listener_zone.desc: +"""The configuration zone to which the listener belongs.""" + +base_listener_zone.label: +"""Zone""" + +fields_mqtt_quic_listener_handshake_idle_timeout.desc: +"""How long a handshake can idle before it is discarded.""" + +fields_mqtt_quic_listener_handshake_idle_timeout.label: +"""Handshake Idle Timeout""" + +force_gc_enable.desc: +"""Enable forced garbage collection.""" + +force_gc_enable.label: +"""Enable forced garbage collection""" + +fields_ws_opts_allow_origin_absence.desc: +"""If false and check_origin_enable is + true, the server will reject requests that don't have origin + HTTP header.""" + +fields_ws_opts_allow_origin_absence.label: +"""Allow origin absence""" + +common_ssl_opts_schema_versions.desc: +"""All TLS/DTLS versions to be supported.
+NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config.
+In case PSK cipher suites are intended, make sure to configure +['tlsv1.2', 'tlsv1.1'] here.""" + +common_ssl_opts_schema_versions.label: +"""SSL versions""" + +mqtt_listener_proxy_protocol_timeout.desc: +"""Timeout for proxy protocol. EMQX will close the TCP connection if proxy protocol packet is not received within the timeout.""" + +mqtt_listener_proxy_protocol_timeout.label: +"""Proxy protocol timeout""" + +fields_mqtt_quic_listener_idle_timeout.desc: +"""How long a connection can go idle before it is gracefully shut down. 0 to disable""" + +fields_mqtt_quic_listener_idle_timeout.label: +"""Idle Timeout""" + +common_ssl_opts_schema_secure_renegotiate.desc: +"""SSL parameter renegotiation is a feature that allows a client and a server +to renegotiate the parameters of the SSL connection on the fly. +RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, +you drop support for the insecure renegotiation, prone to MitM attacks.""" + +common_ssl_opts_schema_secure_renegotiate.label: +"""SSL renegotiate""" + +sysmon_vm_busy_port.desc: +"""When a port (e.g. TCP socket) is overloaded, there will be a busy_port warning log, +and an MQTT message is published to the system topic $SYS/sysmon/busy_port.""" + +sysmon_vm_busy_port.label: +"""Enable Busy Port monitoring.""" + +sys_event_client_connected.desc: +"""Enable to publish client connected event messages""" + +sysmon_vm_process_low_watermark.desc: +"""The threshold, as percentage of processes, for how many + processes can simultaneously exist at the local node before the corresponding + alarm is cleared.""" + +sysmon_vm_process_low_watermark.label: +"""Process low watermark""" + +mqtt_max_packet_size.desc: +"""Maximum MQTT packet size allowed.""" + +mqtt_max_packet_size.label: +"""Max Packet Size""" + +common_ssl_opts_schema_reuse_sessions.desc: +"""Enable TLS session reuse.""" + +common_ssl_opts_schema_reuse_sessions.label: +"""TLS session reuse""" + +common_ssl_opts_schema_depth.desc: +"""Maximum number of non-self-issued intermediate certificates that can follow the peer certificate in a valid certification path. +So, if depth is 0 the PEER must be signed by the trusted ROOT-CA directly;
+if 1 the path can be PEER, Intermediate-CA, ROOT-CA;
+if 2 the path can be PEER, Intermediate-CA1, Intermediate-CA2, ROOT-CA.""" + +common_ssl_opts_schema_depth.label: +"""CACert Depth""" + +sysmon_vm_long_gc.desc: +"""When an Erlang process spends long time to perform garbage collection, a warning level long_gc log is emitted, +and an MQTT message is published to the system topic $SYS/sysmon/long_gc.""" + +sysmon_vm_long_gc.label: +"""Enable Long GC monitoring.""" + +fields_mqtt_quic_listener_keyfile.desc: +"""Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead.""" + +fields_mqtt_quic_listener_keyfile.label: +"""Key file""" + +mqtt_peer_cert_as_clientid.desc: +"""Use the CN, DN field in the peer certificate or the entire certificate content as Client ID. Only works for the TLS connection. +Supported configurations are the following: +- cn: CN field of the certificate +- dn: DN field of the certificate +- crt: DER or PEM certificate +- pem: Convert DER certificate content to PEM format and use as Client ID +- md5: MD5 value of the DER or PEM certificate""" + +mqtt_peer_cert_as_clientid.label: +"""Use Peer Certificate as Client ID""" + +persistent_session_store_message_gc_interval.desc: +"""The starting interval for garbage collection of undelivered messages to +a persistent session. This affects how often the "max_retain_undelivered" +is checked for removal.""" + +persistent_session_store_message_gc_interval.label: +"""Message GC interval""" + +broker_shared_dispatch_ack_enabled.desc: +"""Deprecated, will be removed in 5.1. +Enable/disable shared dispatch acknowledgement for QoS 1 and QoS 2 messages. +This should allow messages to be dispatched to a different subscriber in the group in case the picked (based on `shared_subscription_strategy`) subscriber is offline.""" + +base_listener_enable_authn.desc: +"""Set true (default) to enable client authentication on this listener, the authentication process goes through the configured authentication chain. When set to false to allow any clients with or without authentication information such as username or password to log in. When set to quick_deny_anonymous, it behaves like when set to true, but clients will be denied immediately without going through any authenticators if username is not provided. This is useful to fence off anonymous clients early.""" - zh: """配置 true （默认值）启用客户端进行身份认证，通过检查认配置的认认证器链来决定是否允许接入。 -配置 false 时，将不对客户端做任何认证，任何客户端，不论是不是携带用户名等认证信息，都可以接入。 -配置 quick_deny_anonymous 时，行为跟 true 类似，但是会对匿名 -客户直接拒绝，不做使用任何认证器对客户端进行身份检查。""" - } - label: { - en: "Enable authentication" - zh: "启用身份认证" - } -} -mqtt_listener_access_rules { - desc { - en: """The access control rules for this listener.
See: https://github.com/emqtt/esockd#allowdeny""" - zh: """此监听器的访问控制规则。""" - } - label: { - en: "Access rules" - zh: "访问控制规则" - } -} +base_listener_enable_authn.label: +"""Enable authentication""" -mqtt_listener_proxy_protocol { - desc { - en: """Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx.
+force_shutdown_enable.desc: +"""Enable `force_shutdown` feature.""" + +force_shutdown_enable.label: +"""Enable `force_shutdown` feature""" + +broker_enable_session_registry.desc: +"""Enable session registry""" + +overload_protection_backoff_delay.desc: +"""The maximum duration of delay for background task execution during high load conditions.""" + +overload_protection_backoff_delay.label: +"""Delay Time""" + +ciphers_schema_common.desc: +"""This config holds TLS cipher suite names separated by comma, +or as an array of strings. e.g. +"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256" or +["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]. +
+Ciphers (and their ordering) define the way in which the +client and server encrypts information over the network connection. +Selecting a good cipher suite is critical for the +application's data security, confidentiality and performance. + +The names should be in OpenSSL string format (not RFC format). +All default values and examples provided by EMQX config +documentation are all in OpenSSL format.
+ +NOTE: Certain cipher suites are only compatible with +specific TLS versions ('tlsv1.1', 'tlsv1.2' or 'tlsv1.3') +incompatible cipher suites will be silently dropped. +For instance, if only 'tlsv1.3' is given in the versions, +configuring cipher suites for other versions will have no effect. +
+ +NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config
+If PSK cipher suites are intended, 'tlsv1.3' should be disabled from versions.
+PSK cipher suites: "RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, +RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, +RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, +RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"""" + +ciphers_schema_common.label: +"""""" + +sys_event_client_disconnected.desc: +"""Enable to publish client disconnected event messages.""" + +crl_cache_refresh_interval.desc: +"""The period to refresh the CRLs from the servers. This is a global setting for all URLs and listeners.""" + +crl_cache_refresh_interval.label: +"""CRL Cache Refresh Interval""" + +mqtt_listener_proxy_protocol.desc: +"""Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx.
See: https://www.haproxy.com/blog/haproxy/proxy-protocol/""" - zh: """如果EMQX集群部署在 HAProxy 或 Nginx 之后，请启用代理协议 V1/2
-详情见: https://www.haproxy.com/blog/haproxy/proxy-protocol/""" - } - label: { - en: "Proxy protocol" - zh: "Proxy protocol" - } -} -mqtt_listener_proxy_protocol_timeout { - desc { - en: """Timeout for proxy protocol. EMQX will close the TCP connection if proxy protocol packet is not received within the timeout.""" - zh: """代理协议超时。如果在超时时间内未收到代理协议数据包，EMQX将关闭TCP连接。""" - } - label: { - en: "Proxy protocol timeout" - zh: "Proxy protocol 超时时间" - } -} +mqtt_listener_proxy_protocol.label: +"""Proxy protocol""" -global_authentication { - desc { - en: """Default authentication configs for all MQTT listeners. +mqtt_listener_access_rules.desc: +"""The access control rules for this listener.
See: https://github.com/emqtt/esockd#allowdeny""" -For per-listener overrides see authentication in listener configs +mqtt_listener_access_rules.label: +"""Access rules""" -This option can be configured with: -
    -
  • []: The default value, it allows *ALL* logins
    • -
  • one: For example {enable:true,backend:\"built_in_database\",mechanism=\"password_based\"}
    • -
  • chain: An array of structs.
    • -
+server_ssl_opts_schema_enable_ocsp_stapling.desc: +"""Whether to enable Online Certificate Status Protocol (OCSP) stapling for the listener. If set to true, requires defining the OCSP responder URL and issuer PEM path.""" -When a chain is configured, the login credentials are checked against the backends per the configured order, until an 'allow' or 'deny' decision can be made. +server_ssl_opts_schema_enable_ocsp_stapling.label: +"""Enable OCSP Stapling""" -If there is no decision after a full chain exhaustion, the login is rejected.""" - zh: """全局 MQTT 监听器的默认认证配置。 为每个监听器配置认证参考监听器器配置中的authentication 配置。 +fields_tcp_opts_send_timeout_close.desc: +"""Close the connection if send timeout.""" -该配置可以被配置为： -
    -
  • []: 默认值，允许所有的登录请求 -
  • 配置为单认证器，例如 {enable:true,backend:\"built_in_database\",mechanism=\"password_based\"}
    • -
  • 配置为认证器数组
    • -
+fields_tcp_opts_send_timeout_close.label: +"""TCP send timeout close""" -当配置为认证链后，登录凭证会按照配置的顺序进行检查，直到做出allow 或 deny的结果。 +sysmon_os_cpu_check_interval.desc: +"""The time interval for the periodic CPU check.""" -如果在所有的认证器都执行完后，还是没有结果，登录将被拒绝。""" - } -} +sysmon_os_cpu_check_interval.label: +"""The time interval for the periodic CPU check.""" -listener_authentication { - desc { - en: """Per-listener authentication override. -Authentication can be one single authenticator instance or a chain of authenticators as an array. -When authenticating a login (username, client ID, etc.) the authenticators are checked in the configured order.""" - zh: """监听器认证重载。 -认证配置可以是单个认证器实例，也可以是一个认证器数组组成的认证链。 -执行登录验证时（用户名、客户端 ID 等），将按配置的顺序执行。""" - } - label: { - en: "Per-listener authentication override" - zh: "每个监听器的认证覆盖" - } -} +sysmon_top_sample_interval.desc: +"""Specifies how often process top should be collected""" -fields_rate_limit_max_conn_rate { - desc { - en: """Maximum connections per second.""" - zh: """每秒最大连接数。""" - } - label: { - en: "Max connection rate" - zh: "每秒最大连接数" - } -} +sysmon_top_sample_interval.label: +"""Top sample interval""" -fields_rate_limit_conn_messages_in { - desc { - en: """Message limit for the external MQTT connections.""" - zh: """外部 MQTT 连接的消息限制。""" - } - label: { - en: "connecting messages in" - zh: "外部 MQTT 连接的消息限制" - } -} +fields_mqtt_quic_listener_idle_timeout_ms.desc: +"""How long a connection can go idle before it is gracefully shut down. 0 to disable timeout""" -fields_rate_limit_conn_bytes_in { - desc { - en: """Limit the rate of receiving packets for a MQTT connection. -The rate is counted by bytes of packets per second.""" - zh: """限制 MQTT 连接接收数据包的速率。 速率以每秒的数据包字节数计算。""" - } - label: { - en: "Connection bytes in" - zh: "数据包速率" - } -} +fields_mqtt_quic_listener_idle_timeout_ms.label: +"""Idle timeout ms""" -client_ssl_opts_schema_server_name_indication { - desc { - en: """Specify the host name to be used in TLS Server Name Indication extension.
-For instance, when connecting to "server.example.net", the genuine server -which accepts the connection and performs TLS handshake may differ from the -host the TLS client initially connects to, e.g. when connecting to an IP address -or when the host has multiple resolvable DNS records
-If not specified, it will default to the host name string which is used -to establish the connection, unless it is IP addressed used.
-The host name is then also used in the host name verification of the peer -certificate.
The special value 'disable' prevents the Server Name -Indication extension from being sent and disables the hostname -verification check.""" - zh: """指定要在 TLS 服务器名称指示扩展中使用的主机名。
-例如，当连接到 "server.example.net" 时，接受连接并执行 TLS 握手的真正服务器可能与 TLS 客户端最初连接到的主机不同， -例如，当连接到 IP 地址时，或者当主机具有多个可解析的 DNS 记录时
-如果未指定，它将默认为使用的主机名字符串 -建立连接，除非使用 IP 地址
-然后，主机名也用于对等机的主机名验证证书
-特殊值 disable 阻止发送服务器名称指示扩展，并禁用主机名验证检查。""" - } - label: { - en: "Server Name Indication" - zh: "服务器名称指示" - } -} - -fields_tcp_opts_active_n { - desc { - en: """Specify the {active, N} option for this Socket.
-See: https://erlang.org/doc/man/inet.html#setopts-2""" - zh: """为此套接字指定{active，N}选项
-See: https://erlang.org/doc/man/inet.html#setopts-2""" - } - label: { - en: "active_n" - zh: "active_n" - } -} - -fields_tcp_opts_backlog { - desc { - en: """TCP backlog defines the maximum length that the queue of -pending connections can grow to.""" - zh: """TCP backlog 定义了挂起连接队列可以增长到的最大长度。""" - } - label: { - en: "TCP backlog length" - zh: "TCP 连接队列长度" - } -} - -fields_tcp_opts_send_timeout { - desc { - en: """The TCP send timeout for the connections.""" - zh: """连接的 TCP 发送超时。""" - } - label: { - en: "TCP send timeout" - zh: "TCP 发送超时" - } -} - -fields_tcp_opts_send_timeout_close { - desc { - en: """Close the connection if send timeout.""" - zh: """如果发送超时，则关闭连接。""" - } - label: { - en: "TCP send timeout close" - zh: "TCP 发送超时关闭连接" - } -} - -fields_tcp_opts_recbuf { - desc { - en: """The TCP receive buffer (OS kernel) for the connections.""" - zh: """连接的 TCP 接收缓冲区（OS 内核）。""" - } - label: { - en: "TCP receive buffer" - zh: "TCP 接收缓冲区" - } -} - -fields_tcp_opts_sndbuf { - desc { - en: """The TCP send buffer (OS kernel) for the connections.""" - zh: """连接的 TCP 发送缓冲区（OS 内核）。""" - } - label: { - en: "TCP send buffer" - zh: "TCP 发送缓冲区" - } -} - -fields_tcp_opts_buffer { - desc { - en: """The size of the user-space buffer used by the driver.""" - zh: """驱动程序使用的用户空间缓冲区的大小。""" - } - label: { - en: "TCP user-space buffer" - zh: "TCP 用户态缓冲区" - } -} - -fields_tcp_opts_high_watermark { - desc { - en: """The socket is set to a busy state when the amount of data queued internally -by the VM socket implementation reaches this limit.""" - zh: """当 VM 套接字实现内部排队的数据量达到此限制时，套接字将设置为忙碌状态。""" - } - label: { - en: "TCP 高水位线" - zh: "" - } -} - -fields_tcp_opts_nodelay { - desc { - en: """The TCP_NODELAY flag for the connections.""" - zh: """连接的 TCP_NODELAY 标识""" - } - label: { - en: "TCP_NODELAY" - zh: "TCP_NODELAY" - } -} - -fields_tcp_opts_reuseaddr { - desc { - en: """The SO_REUSEADDR flag for the connections.""" - zh: """连接的 SO_REUSEADDR 标识。""" - } - label: { - en: "SO_REUSEADDR" - zh: "SO_REUSEADDR" - } -} - -fields_trace_payload_encode { - desc { - en: """Determine the format of the payload format in the trace file.
-`text`: Text-based protocol or plain text protocol. - It is recommended when payload is JSON encoded.
-`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.
-`hidden`: payload is obfuscated as `******`""" - zh: """确定跟踪文件中有效负载格式的格式。
-`text`：基于文本的协议或纯文本协议。 -建议在有效负载为JSON编码时使用
-`hex`：二进制十六进制编码。当有效负载是自定义二进制协议时，建议使用此选项
-`hidden`：有效负载被模糊化为 `******`""" - } - label: { - en: "Payload encode" - zh: "有效负载编码" - } -} - -fields_ws_opts_mqtt_path { - desc { - en: """WebSocket's MQTT protocol path. So the address of EMQX Broker's WebSocket is: -ws://{ip}:{port}/mqtt""" - zh: """WebSocket 的 MQTT 协议路径。因此，EMQX Broker的WebSocket地址为： -ws://{ip}:{port}/mqtt""" - } - label: { - en: "WS MQTT Path" - zh: "WS MQTT 路径" - } -} - -fields_ws_opts_mqtt_piggyback { - desc { - en: """Whether a WebSocket message is allowed to contain multiple MQTT packets.""" - zh: """WebSocket消息是否允许包含多个 MQTT 数据包。""" - } - label: { - en: "MQTT Piggyback" - zh: "MQTT Piggyback" - } -} - -fields_ws_opts_compress { - desc { - en: """If true, compress WebSocket messages using zlib.
-The configuration items under deflate_opts belong to the compression-related parameter configuration.""" - zh: """如果 true，则使用zlib 压缩 WebSocket 消息
-deflate_opts 下的配置项属于压缩相关参数配置。""" - } - label: { - en: "Ws compress" - zh: "Ws 压缩" - } -} - -fields_ws_opts_idle_timeout { - desc { - en: """Close transport-layer connections from the clients that have not sent MQTT CONNECT message within this interval.""" - zh: """关闭在此间隔内未发送 MQTT CONNECT 消息的客户端的传输层连接。""" - } - label: { - en: "WS idle timeout" - zh: "WS 空闲时间" - } -} - -fields_ws_opts_max_frame_size { - desc { - en: """The maximum length of a single MQTT packet.""" - zh: """单个 MQTT 数据包的最大长度。""" - } - label: { - en: "Max frame size" - zh: "最大数据包长度" - } -} - -fields_ws_opts_fail_if_no_subprotocol { - desc { - en: """If true, the server will return an error when +fields_ws_opts_fail_if_no_subprotocol.desc: +"""If true, the server will return an error when the client does not carry the Sec-WebSocket-Protocol field.
Note: WeChat applet needs to disable this verification.""" - zh: """如果true，当客户端未携带Sec WebSocket Protocol字段时，服务器将返回一个错误。 -
注意：微信小程序需要禁用此验证。""" - } - label: { - en: "Fail if no subprotocol" - zh: "无 subprotocol 则失败" - } -} -fields_ws_opts_supported_subprotocols { - desc { - en: """Comma-separated list of supported subprotocols.""" - zh: """逗号分隔的 subprotocols 支持列表。""" - } - label: { - en: "Supported subprotocols" - zh: "Subprotocols 支持列表" - } -} +fields_ws_opts_fail_if_no_subprotocol.label: +"""Fail if no subprotocol""" -fields_ws_opts_check_origin_enable { - desc { - en: """If true, origin HTTP header will be - validated against the list of allowed origins configured in check_origins - parameter.""" - zh: """如果true，originHTTP 头将根据check_origins参数中配置的允许来源列表进行验证。""" - } - label: { - en: "Check origin" - zh: "检查 origin" - } -} +mqtt_wildcard_subscription.desc: +"""Whether to enable support for MQTT wildcard subscription.""" -fields_ws_opts_allow_origin_absence { - desc { - en: """If false and check_origin_enable is - true, the server will reject requests that don't have origin - HTTP header.""" - zh: """If false and check_origin_enable is true, the server will reject requests that don't have origin HTTP header.""" - } - label: { - en: "Allow origin absence" - zh: "允许 origin 缺失" - } -} +mqtt_wildcard_subscription.label: +"""Wildcard Subscription Available""" -fields_ws_opts_check_origins { - desc { - en: """List of allowed origins.
See check_origin_enable.""" - zh: """允许的 origins 列表""" - } - label: { - en: "Allowed origins" - zh: "允许的 origins" - } -} +server_ssl_opts_schema_ocsp_refresh_interval.desc: +"""The period to refresh the OCSP response for the server.""" -fields_ws_opts_proxy_address_header { - desc { - en: """HTTP header used to pass information about the client IP address. -Relevant when the EMQX cluster is deployed behind a load-balancer.""" - zh: """HTTP 头，用于传递有关客户端 IP 地址的信息。 -当 EMQX 集群部署在负载平衡器后面时，这一点非常重要。""" - } - label: { - en: "Proxy address header" - zh: "客户端地址头" - } -} +server_ssl_opts_schema_ocsp_refresh_interval.label: +"""OCSP Refresh Interval""" -fields_ws_opts_proxy_port_header { - desc { - en: """HTTP header used to pass information about the client port. Relevant when the EMQX cluster is deployed behind a load-balancer.""" - zh: """HTTP 头，用于传递有关客户端端口的信息。当 EMQX 集群部署在负载平衡器后面时，这一点非常重要。""" - } - label: { - en: "Proxy port header" - zh: "客户端端口头" - } -} +overload_protection_backoff_hibernation.desc: +"""When at high load, skip process hibernation.""" + +overload_protection_backoff_hibernation.label: +"""Skip hibernation""" + +fields_ws_opts_max_frame_size.desc: +"""The maximum length of a single MQTT packet.""" + +fields_ws_opts_max_frame_size.label: +"""Max frame size""" + +sys_event_messages.desc: +"""Client events messages.""" + +broker_perf_trie_compaction.desc: +"""Enable trie path compaction. +Enabling it significantly improves wildcard topic subscribe rate, if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', where ID is unique per subscriber. +Topic match performance (when publishing) may degrade if messages are mostly published to topics with large number of levels. + +NOTE: This is a cluster-wide configuration. It requires all nodes to be stopped before changing it.""" + +sysmon_vm_large_heap.desc: +"""When an Erlang process consumed a large amount of memory for its heap space, +the system will write a warning level large_heap log, and an MQTT message is published to +the system topic $SYS/sysmon/large_heap.""" + +sysmon_vm_large_heap.label: +"""Enable Large Heap monitoring.""" } diff --git a/rel/i18n/emqx_slow_subs_api.hocon b/rel/i18n/emqx_slow_subs_api.hocon index 92862bc98..edf473487 100644 --- a/rel/i18n/emqx_slow_subs_api.hocon +++ b/rel/i18n/emqx_slow_subs_api.hocon @@ -1,66 +1,30 @@ emqx_slow_subs_api { - clear_records_api { - desc { - en: "Clear current data and re count slow topic" - zh: "清除当前记录，然后重新开始统计" - } - } +clear_records_api.desc: +"""Clear current data and re count slow topic""" - get_records_api { - desc { - en: "View slow topics statistics record data" - zh: "查看慢订阅的统计数据" - } - } +clientid.desc: +"""Message clientid""" - get_setting_api { - desc { - en: "View slow subs settings" - zh: "查看配置" - } - } +get_records_api.desc: +"""View slow topics statistics record data""" - update_setting_api { - desc { - en: "Update slow subs settings" - zh: "更新配置" - } - } +get_setting_api.desc: +"""View slow subs settings""" - clientid { - desc { - en: "Message clientid" - zh: "消息的客户端 ID" - } - } +last_update_time.desc: +"""The timestamp of last update""" - node { - desc { - en: "Message node name" - zh: "消息的节点名称" - } - } +node.desc: +"""Message node name""" - topic { - desc { - en: "Message topic" - zh: "消息的主题" - } - } +timespan.desc: +"""Timespan for message transmission""" - timespan { - desc { - en: "Timespan for message transmission" - zh: "消息的传输耗时" - } - } +topic.desc: +"""Message topic""" - last_update_time { - desc { - en: "The timestamp of last update" - zh: "记录的更新时间戳" - } - } +update_setting_api.desc: +"""Update slow subs settings""" } diff --git a/rel/i18n/emqx_slow_subs_schema.hocon b/rel/i18n/emqx_slow_subs_schema.hocon index e65e802c2..4164db75a 100644 --- a/rel/i18n/emqx_slow_subs_schema.hocon +++ b/rel/i18n/emqx_slow_subs_schema.hocon @@ -1,38 +1,18 @@ emqx_slow_subs_schema { - enable { - desc { - en: "Enable this feature" - zh: "开启慢订阅" - } - } +enable.desc: +"""Enable this feature""" - threshold { - desc { - en: "The latency threshold for statistics" - zh: "慢订阅统计的阈值" - } - } +expire_interval.desc: +"""The eviction time of the record, which in the statistics record table""" - expire_interval { - desc { - en: "The eviction time of the record, which in the statistics record table" - zh: "慢订阅记录的有效时间" - } - } +stats_type.desc: +"""The method to calculate the latency""" - top_k_num { - desc { - en: "The maximum number of records in the slow subscription statistics record table" - zh: "慢订阅统计表的记录数量上限" - } - } +threshold.desc: +"""The latency threshold for statistics""" - stats_type { - desc { - en: "The method to calculate the latency" - zh: "慢订阅的统计类型" - } - } +top_k_num.desc: +"""The maximum number of records in the slow subscription statistics record table""" } diff --git a/rel/i18n/emqx_statsd_api.hocon b/rel/i18n/emqx_statsd_api.hocon index 2721188bd..d8bab13a7 100644 --- a/rel/i18n/emqx_statsd_api.hocon +++ b/rel/i18n/emqx_statsd_api.hocon @@ -1,16 +1,9 @@ emqx_statsd_api { - get_statsd_config_api { - desc { - en: """List the configuration of StatsD metrics collection and push service.""" - zh: """列出 StatsD 指标采集和推送服务的的配置。""" - } - } +get_statsd_config_api.desc: +"""List the configuration of StatsD metrics collection and push service.""" + +update_statsd_config_api.desc: +"""Update the configuration of StatsD metrics collection and push service.""" - update_statsd_config_api { - desc { - en: """Update the configuration of StatsD metrics collection and push service.""" - zh: """更新 StatsD 指标采集和推送服务的配置。""" - } - } } diff --git a/rel/i18n/emqx_statsd_schema.hocon b/rel/i18n/emqx_statsd_schema.hocon index 46d654a46..fc21710c4 100644 --- a/rel/i18n/emqx_statsd_schema.hocon +++ b/rel/i18n/emqx_statsd_schema.hocon @@ -1,61 +1,30 @@ emqx_statsd_schema { - get_statsd_config_api { - desc { - en: """List the configuration of StatsD metrics collection and push service.""" - zh: """列出 StatsD 指标采集和推送服务的的配置。""" - } - } +enable.desc: +"""Enable or disable StatsD metrics collection and push service.""" - update_statsd_config_api { - desc { - en: """Update the configuration of StatsD metrics collection and push service.""" - zh: """更新 StatsD 指标采集和推送服务的配置。""" - } - } +flush_interval.desc: +"""The push interval for metrics.""" - statsd { - desc { - en: """StatsD metrics collection and push configuration.""" - zh: """StatsD 指标采集与推送配置。""" - } - label { - en: """StatsD""" - zh: """StatsD""" - } - } +get_statsd_config_api.desc: +"""List the configuration of StatsD metrics collection and push service.""" - server { - desc { - en: """StatsD server address.""" - zh: """StatsD 服务器地址。""" - } - } +sample_interval.desc: +"""The sampling interval for metrics.""" - sample_interval { - desc { - en: """The sampling interval for metrics.""" - zh: """指标的采样间隔。""" - } - } +server.desc: +"""StatsD server address.""" - flush_interval { - desc { - en: """The push interval for metrics.""" - zh: """指标的推送间隔。""" - } - } - tags { - desc { - en: """The tags for metrics.""" - zh: """指标的标签。""" - } - } +statsd.desc: +"""StatsD metrics collection and push configuration.""" + +statsd.label: +"""StatsD""" + +tags.desc: +"""The tags for metrics.""" + +update_statsd_config_api.desc: +"""Update the configuration of StatsD metrics collection and push service.""" - enable { - desc { - en: """Enable or disable StatsD metrics collection and push service.""" - zh: """启用或禁用 StatsD 指标采集和推送服务。""" - } - } } diff --git a/rel/i18n/emqx_stomp_schema.hocon b/rel/i18n/emqx_stomp_schema.hocon index 3d166abb5..05d5b9d18 100644 --- a/rel/i18n/emqx_stomp_schema.hocon +++ b/rel/i18n/emqx_stomp_schema.hocon @@ -1,32 +1,16 @@ emqx_stomp_schema { - stomp { - desc { - en: """The Stomp Gateway configuration. + +stom_frame_max_body_length.desc: +"""Maximum number of bytes of Body allowed per Stomp packet""" + +stom_frame_max_headers.desc: +"""The maximum number of Header""" + +stomp.desc: +"""The Stomp Gateway configuration. This gateway supports v1.2/1.1/1.0""" - zh: """Stomp 网关配置。当前实现支持 v1.2/1.1/1.0 协议版本""" - } - } - - stom_frame_max_headers { - desc { - en: """The maximum number of Header""" - zh: """允许的 Header 最大数量""" - } - } - - stomp_frame_max_headers_length { - desc { - en: """The maximum string length of the Header Value""" - zh: """允许的 Header 字符串的最大长度""" - } - } - - stom_frame_max_body_length { - desc { - en: """Maximum number of bytes of Body allowed per Stomp packet""" - zh: """允许的 Stomp 报文 Body 的最大字节数""" - } - } +stomp_frame_max_headers_length.desc: +"""The maximum string length of the Header Value""" } diff --git a/rel/i18n/emqx_telemetry_api.hocon b/rel/i18n/emqx_telemetry_api.hocon index a8f562065..5c61b8d3c 100644 --- a/rel/i18n/emqx_telemetry_api.hocon +++ b/rel/i18n/emqx_telemetry_api.hocon @@ -1,121 +1,54 @@ emqx_telemetry_api { - get_telemetry_status_api { - desc { - en: """Get telemetry status""" - zh: """获取遥测状态""" - } - } +active_modules.desc: +"""Get active modules""" - update_telemetry_status_api { - desc { - en: """Enable or disable telemetry""" - zh: """更新遥测状态""" - } - } +active_plugins.desc: +"""Get active plugins""" - get_telemetry_data_api { - desc { - en: """Get telemetry data""" - zh: """获取遥测数据""" - } - } +emqx_version.desc: +"""Get emqx version""" - enable { - desc { - en: """Enable telemetry""" - zh: """启用遥测""" - } - } +enable.desc: +"""Enable telemetry""" - emqx_version { - desc { - en: """Get emqx version""" - zh: """获取 emqx 版本""" - } - } +get_telemetry_data_api.desc: +"""Get telemetry data""" - license { - desc { - en: """Get license information""" - zh: """获取 license 信息""" - } - } +get_telemetry_status_api.desc: +"""Get telemetry status""" - os_name { - desc { - en: """Get OS name""" - zh: """获取操作系统名称""" - } - } +license.desc: +"""Get license information""" - os_version { - desc { - en: """Get OS version""" - zh: """获取操作系统版本""" - } - } +messages_received.desc: +"""Get number of messages received""" - otp_version { - desc { - en: """Get Erlang OTP version""" - zh: """获取 OTP 版本""" - } - } +messages_sent.desc: +"""Get number of messages sent""" - up_time { - desc { - en: """Get uptime""" - zh: """获取运行时间""" - } - } +nodes_uuid.desc: +"""Get nodes UUID""" - uuid { - desc { - en: """Get UUID""" - zh: """获取 UUID""" - } - } +num_clients.desc: +"""Get number of clients""" - nodes_uuid { - desc { - en: """Get nodes UUID""" - zh: """获取节点 UUID""" - } - } +os_name.desc: +"""Get OS name""" - active_plugins { - desc { - en: """Get active plugins""" - zh: """获取活跃插件""" - } - } +os_version.desc: +"""Get OS version""" - active_modules { - desc { - en: """Get active modules""" - zh: """获取活跃模块""" - } - } +otp_version.desc: +"""Get Erlang OTP version""" - num_clients { - desc { - en: """Get number of clients""" - zh: """获取客户端数量""" - } - } +up_time.desc: +"""Get uptime""" - messages_received { - desc { - en: """Get number of messages received""" - zh: """获取接收到的消息数量""" - } - } +update_telemetry_status_api.desc: +"""Enable or disable telemetry""" + +uuid.desc: +"""Get UUID""" - messages_sent { - desc { - en: """Get number of messages sent""" - zh: """获取发送的消息数量""" - } - } } diff --git a/rel/i18n/emqx_topic_metrics_api.hocon b/rel/i18n/emqx_topic_metrics_api.hocon index 22f038d4e..94c58f0cd 100644 --- a/rel/i18n/emqx_topic_metrics_api.hocon +++ b/rel/i18n/emqx_topic_metrics_api.hocon @@ -1,240 +1,105 @@ emqx_topic_metrics_api { - get_topic_metrics_api { - desc { - en: """List topic metrics""" - zh: """获取主题监控数据""" - } - } - reset_topic_metrics_api{ - desc { - en: """Reset telemetry status""" - zh: """重置主题监控状态""" - } - } +message_qos1_in_rate.desc: +"""QoS1 in messages rate""" - post_topic_metrics_api { - desc { - en: """Create topic metrics""" - zh: """创建主题监控数据""" - } - } +message_out_count.desc: +"""Out messages count""" - gat_topic_metrics_data_api { - desc { - en: """Get topic metrics""" - zh: """获取主题监控数据""" - } - } +message_qos2_out_rate.desc: +"""QoS2 out messages rate""" - delete_topic_metrics_data_api { - desc { - en: """Delete topic metrics""" - zh: """删除主题监控数据""" - } - } +message_qos0_in_rate.desc: +"""QoS0 in messages rate""" - topic_metrics_api_response409 { - desc { - en: """Conflict. Topic metrics exceeded max limit 512""" - zh: """冲突。主题监控数据超过最大限制512""" - } - } +get_topic_metrics_api.desc: +"""List topic metrics""" - topic_metrics_api_response400 { - desc { - en: """Bad request. Already exists or bad topic name""" - zh: """错误请求。已存在或错误的主题名称""" - } - } +reset_time.desc: +"""Reset time. In rfc3339. Nullable if never reset""" - topic_metrics_api_response404 { - desc { - en: """Not Found. Topic metrics not found""" - zh: """未找到。主题监控数据未找到""" - } - } +topic_metrics_api_response400.desc: +"""Bad request. Already exists or bad topic name""" - reset_topic_desc { - desc { - en: """Topic Name. If this parameter is not present,all created topic metrics will be reset.""" - zh: """主题名称。如果此参数不存在，则所有创建的主题监控数据都将重置。""" - } - } +reset_topic_desc.desc: +"""Topic Name. If this parameter is not present,all created topic metrics will be reset.""" - topic { - desc { - en: """Topic""" - zh: """主题""" - } - } +topic_metrics_api_response409.desc: +"""Conflict. Topic metrics exceeded max limit 512""" - topic_in_body { - desc { - en: """Raw topic string""" - zh: """主题字符串""" - } - } +post_topic_metrics_api.desc: +"""Create topic metrics""" - topic_in_path { - desc { - en: """Topic string. Notice: Topic string in url path must be encoded""" - zh: """主题字符串。注意：主题字符串在url路径中必须编码""" - } - } +message_dropped_rate.desc: +"""Dropped messages rate""" - action { - desc { - en: """Action. Only support reset""" - zh: """操作，仅支持 reset""" - } - } +message_qos2_in_rate.desc: +"""QoS2 in messages rate""" - create_time { - desc { - en: """Create time""" - zh: """创建时间。标准 rfc3339 时间格式，例如：2018-01-01T12:00:00Z""" - } - } +message_in_rate.desc: +"""In messages rate""" - reset_time { - desc { - en: """Reset time. In rfc3339. Nullable if never reset""" - zh: """重置时间。标准 rfc3339 时间格式，例如：2018-01-01T12:00:00Z。如果从未重置则为空""" - } - } +message_qos0_out_rate.desc: +"""QoS0 out messages rate""" - metrics { - desc { - en: """Metrics""" - zh: """监控数据""" - } - } +message_qos2_in_count.desc: +"""QoS2 in messages count""" - message_dropped_count { - desc { - en: """Dropped messages count""" - zh: """丢弃消息数量""" - } - } +message_dropped_count.desc: +"""Dropped messages count""" - message_dropped_rate { - desc { - en: """Dropped messages rate""" - zh: """丢弃消息速率""" - } - } +topic_metrics_api_response404.desc: +"""Not Found. Topic metrics not found""" - message_in_count { - desc { - en: """In messages count""" - zh: """接收消息数量""" - } - } +topic_in_path.desc: +"""Topic string. Notice: Topic string in url path must be encoded""" - message_in_rate { - desc { - en: """In messages rate""" - zh: """接收消息速率""" - } - } +action.desc: +"""Action. Only support reset""" - message_out_count { - desc { - en: """Out messages count""" - zh: """发送消息数量""" - } - } +message_qos0_in_count.desc: +"""QoS0 in messages count""" - message_out_rate { - desc { - en: """Out messages rate""" - zh: """发送消息速率""" - } - } +message_qos1_out_rate.desc: +"""QoS1 out messages rate""" - message_qos0_in_count { - desc { - en: """QoS0 in messages count""" - zh: """QoS0 接收消息数量""" - } - } +topic.desc: +"""Topic""" - message_qos0_in_rate { - desc { - en: """QoS0 in messages rate""" - zh: """QoS0 接收消息速率""" - } - } +reset_topic_metrics_api.desc: +"""Reset telemetry status""" - message_qos0_out_count { - desc { - en: """QoS0 out messages count""" - zh: """QoS0 发送消息数量""" - } - } +create_time.desc: +"""Create time""" - message_qos0_out_rate { - desc { - en: """QoS0 out messages rate""" - zh: """QoS0 发送消息速率""" - } - } +metrics.desc: +"""Metrics""" - message_qos1_in_count { - desc { - en: """QoS1 in messages count""" - zh: """QoS1 接收消息数量""" - } - } +message_qos1_out_count.desc: +"""QoS1 out messages count""" - message_qos1_in_rate { - desc { - en: """QoS1 in messages rate""" - zh: """QoS1 接收消息速率""" - } - } +gat_topic_metrics_data_api.desc: +"""Get topic metrics""" - message_qos1_out_count { - desc { - en: """QoS1 out messages count""" - zh: """QoS1 发送消息数量""" - } - } +message_qos1_in_count.desc: +"""QoS1 in messages count""" - message_qos1_out_rate { - desc { - en: """QoS1 out messages rate""" - zh: """QoS1 发送消息速率""" - } - } +delete_topic_metrics_data_api.desc: +"""Delete topic metrics""" - message_qos2_in_count { - desc { - en: """QoS2 in messages count""" - zh: """QoS2 接收消息数量""" - } - } +message_qos0_out_count.desc: +"""QoS0 out messages count""" - message_qos2_in_rate { - desc { - en: """QoS2 in messages rate""" - zh: """QoS2 接收消息速率""" - } - } +topic_in_body.desc: +"""Raw topic string""" - message_qos2_out_count { - desc { - en: """QoS2 out messages count""" - zh: """QoS2 发送消息数量""" - } - } +message_in_count.desc: +"""In messages count""" - message_qos2_out_rate { - desc { - en: """QoS2 out messages rate""" - zh: """QoS2 发送消息速率""" - } - } +message_qos2_out_count.desc: +"""QoS2 out messages count""" + +message_out_rate.desc: +"""Out messages rate""" } diff --git a/rel/i18n/zh/emqx_authn_api.hocon b/rel/i18n/zh/emqx_authn_api.hocon new file mode 100644 index 000000000..bd8332ac7 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_api.hocon @@ -0,0 +1,96 @@ +emqx_authn_api { + +authentication_get.desc: +"""列出全局认证链上的认证器。""" + +authentication_id_delete.desc: +"""删除全局认证链上的指定认证器。""" + +authentication_id_get.desc: +"""获取全局认证链上的指定认证器。""" + +authentication_id_position_put.desc: +"""更改全局认证链上指定认证器的顺序。""" + +authentication_id_put.desc: +"""更新全局认证链上的指定认证器。""" + +authentication_id_status_get.desc: +"""获取全局认证链上指定认证器的状态。""" + +authentication_id_users_get.desc: +"""获取全局认证链上指定认证器中的用户数据。""" + +authentication_id_users_post.desc: +"""为全局认证链上的指定认证器创建用户数据。""" + +authentication_id_users_user_id_delete.desc: +"""删除全局认证链上指定认证器中的指定用户数据。""" + +authentication_id_users_user_id_get.desc: +"""获取全局认证链上指定认证器中的指定用户数据。""" + +authentication_id_users_user_id_put.desc: +"""更新全局认证链上指定认证器中的指定用户数据。""" + +authentication_post.desc: +"""为全局认证链创建认证器。""" + +is_superuser.desc: +"""是否是超级用户""" + +like_user_id.desc: +"""使用用户 ID （username 或 clientid）模糊查询。""" + +like_user_id.label: +"""like_user_id""" + +listeners_listener_id_authentication_get.desc: +"""列出监听器认证链上的认证器。""" + +listeners_listener_id_authentication_id_delete.desc: +"""删除监听器认证链上的指定认证器。""" + +listeners_listener_id_authentication_id_get.desc: +"""获取监听器认证链上的指定认证器。""" + +listeners_listener_id_authentication_id_position_put.desc: +"""更改监听器认证链上指定认证器的顺序。""" + +listeners_listener_id_authentication_id_put.desc: +"""更新监听器认证链上的指定认证器。""" + +listeners_listener_id_authentication_id_status_get.desc: +"""获取监听器认证链上指定认证器的状态。""" + +listeners_listener_id_authentication_id_users_get.desc: +"""列出监听器认证链上指定认证器中的用户数据。""" + +listeners_listener_id_authentication_id_users_post.desc: +"""为监听器认证链上的指定认证器创建用户数据。""" + +listeners_listener_id_authentication_id_users_user_id_delete.desc: +"""删除监听器认证链上指定认证器中的指定用户数据。""" + +listeners_listener_id_authentication_id_users_user_id_get.desc: +"""获取监听器认证链上指定认证器中的指定用户数据。""" + +listeners_listener_id_authentication_id_users_user_id_put.desc: +"""更新监听器认证链上指定认证器中的指定用户数据。""" + +listeners_listener_id_authentication_post.desc: +"""在监听器认证链上创建认证器。""" + +param_auth_id.desc: +"""认证器 ID。""" + +param_listener_id.desc: +"""监听器 ID。""" + +param_position.desc: +"""认证者在链中的位置。可能的值是 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'""" + +param_user_id.desc: +"""用户 ID。""" + +} diff --git a/rel/i18n/zh/emqx_authn_http.hocon b/rel/i18n/zh/emqx_authn_http.hocon new file mode 100644 index 000000000..17c922b33 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_http.hocon @@ -0,0 +1,45 @@ +emqx_authn_http { + +body.desc: +"""HTTP request body。""" + +body.label: +"""Request Body""" + +get.desc: +"""使用 HTTP Server 作为认证服务的认证器的配置项 (使用 GET 请求)。""" + +headers.desc: +"""HTTP Headers 列表""" + +headers.label: +"""请求头""" + +headers_no_content_type.desc: +"""HTTP Headers 列表 (无 content-type) 。""" + +headers_no_content_type.label: +"""请求头(无 content-type)""" + +method.desc: +"""HTTP 请求方法。""" + +method.label: +"""请求方法""" + +post.desc: +"""使用 HTTP Server 作为认证服务的认证器的配置项 (使用 POST 请求)。""" + +request_timeout.desc: +"""HTTP 请求超时时长。""" + +request_timeout.label: +"""请求超时时间""" + +url.desc: +"""认证 HTTP 服务器地址。""" + +url.label: +"""URL""" + +} diff --git a/rel/i18n/zh/emqx_authn_jwt.hocon b/rel/i18n/zh/emqx_authn_jwt.hocon new file mode 100644 index 000000000..2aa27c1de --- /dev/null +++ b/rel/i18n/zh/emqx_authn_jwt.hocon @@ -0,0 +1,118 @@ +emqx_authn_jwt { + +acl_claim_name.desc: +"""JWT claim name to use for getting ACL rules.""" + +acl_claim_name.label: +"""ACL claim name""" + +algorithm.desc: +"""JWT 签名算法，支持 HMAC (配置为 hmac-based）和 RSA、ECDSA (配置为 public-key)。""" + +algorithm.label: +"""JWT 签名算法""" + +cacertfile.desc: +"""包含 PEM 编码的 CA 证书的文件的路径。""" + +cacertfile.label: +"""CA 证书文件""" + +certfile.desc: +"""包含用户证书的文件的路径。""" + +certfile.label: +"""证书文件""" + +enable.desc: +"""启用/禁用 SSL。""" + +enable.label: +"""启用/禁用 SSL""" + +endpoint.desc: +"""JWKS 端点， 它是一个以 JWKS 格式返回服务端的公钥集的只读端点。""" + +endpoint.label: +"""JWKS Endpoint""" + +from.desc: +"""要从中获取 JWT 的字段。""" + +from.label: +"""源字段""" + +hmac-based.desc: +"""用于认证的 JWT 使用 HMAC 算法签发时的配置。""" + +jwks.desc: +"""用于认证的 JWTs 需要从 JWKS 端点获取时的配置。""" + +keyfile.desc: +"""包含 PEM 编码的用户私钥的文件的路径。""" + +keyfile.label: +"""私钥文件""" + +public-key.desc: +"""用于认证的 JWT 使用 RSA 或 ECDSA 算法签发时的配置。""" + +public_key.desc: +"""用于验证 JWT 的公钥。""" + +public_key.label: +"""公钥""" + +refresh_interval.desc: +"""JWKS 刷新间隔。""" + +refresh_interval.label: +"""JWKS 刷新间隔""" + +secret.desc: +"""使用 HMAC 算法时用于验证 JWT 的密钥""" + +secret.label: +"""Secret""" + +secret_base64_encoded.desc: +"""密钥是否为 Base64 编码。""" + +secret_base64_encoded.label: +"""密钥是否为 Base64 编码""" + +server_name_indication.desc: +"""服务器名称指示（SNI）。""" + +server_name_indication.label: +"""服务器名称指示""" + +ssl.desc: +"""SSL 选项。""" + +ssl.label: +"""SSL 选项""" + +use_jwks.desc: +"""是否使用 JWKS。""" + +use_jwks.label: +"""是否使用 JWKS""" + +verify.desc: +"""指定握手过程中是否校验对端证书。""" + +verify.label: +"""Verify""" + +verify_claims.desc: +"""需要验证的自定义声明列表，它是一个名称/值对列表。 +值可以使用以下占位符： +- ${username}: 将在运行时被替换为客户端连接时使用的用户名 +- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符 +认证时将验证 JWT（取自 Password 字段）中 claims 的值是否与 verify_claims 中要求的相匹配。""" + +verify_claims.label: +"""Verify Claims""" + +} diff --git a/rel/i18n/zh/emqx_authn_mnesia.hocon b/rel/i18n/zh/emqx_authn_mnesia.hocon new file mode 100644 index 000000000..1ba394627 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_mnesia.hocon @@ -0,0 +1,12 @@ +emqx_authn_mnesia { + +authentication.desc: +"""使用内置数据库作为认证数据源的认证器的配置项。""" + +user_id_type.desc: +"""指定使用客户端ID `clientid` 还是用户名 `username` 进行认证。""" + +user_id_type.label: +"""认证 ID 类型""" + +} diff --git a/rel/i18n/zh/emqx_authn_mongodb.hocon b/rel/i18n/zh/emqx_authn_mongodb.hocon new file mode 100644 index 000000000..01419e2b9 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_mongodb.hocon @@ -0,0 +1,45 @@ +emqx_authn_mongodb { + +collection.desc: +"""存储认证数据的集合。""" + +collection.label: +"""集合""" + +filter.desc: +"""在查询中定义过滤条件的条件表达式。 +过滤器支持如下占位符： +- ${username}: 将在运行时被替换为客户端连接时使用的用户名 +- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符""" + +filter.label: +"""过滤器""" + +is_superuser_field.desc: +"""文档中用于定义用户是否具有超级用户权限的字段。""" + +is_superuser_field.label: +"""超级用户字段""" + +password_hash_field.desc: +"""文档中用于存放密码散列的字段。""" + +password_hash_field.label: +"""密码散列字段""" + +replica-set.desc: +"""使用 MongoDB (Replica Set) 作为认证数据源的认证器的配置项。""" + +salt_field.desc: +"""文档中用于存放盐值的字段。""" + +salt_field.label: +"""盐值字段""" + +sharded-cluster.desc: +"""使用 MongoDB (Sharded Cluster) 作为认证数据源的认证器的配置项。""" + +standalone.desc: +"""使用 MongoDB (Standalone) 作为认证数据源的认证器的配置项。""" + +} diff --git a/rel/i18n/zh/emqx_authn_mysql.hocon b/rel/i18n/zh/emqx_authn_mysql.hocon new file mode 100644 index 000000000..e718ad723 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_mysql.hocon @@ -0,0 +1,18 @@ +emqx_authn_mysql { + +authentication.desc: +"""使用 MySQL 作为认证数据源的认证器的配置项。""" + +query.desc: +"""用于查询密码散列等用于认证的数据的 SQL 语句。""" + +query.label: +"""查询语句""" + +query_timeout.desc: +"""SQL 查询的超时时间。""" + +query_timeout.label: +"""查询超时""" + +} diff --git a/rel/i18n/zh/emqx_authn_pgsql.hocon b/rel/i18n/zh/emqx_authn_pgsql.hocon new file mode 100644 index 000000000..97bf608d2 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_pgsql.hocon @@ -0,0 +1,12 @@ +emqx_authn_pgsql { + +authentication.desc: +"""使用 PostgreSQL 作为认证数据源的认证器的配置项。""" + +query.desc: +"""用于查询密码散列等用于认证的数据的 SQL 语句。""" + +query.label: +"""查询语句""" + +} diff --git a/rel/i18n/zh/emqx_authn_redis.hocon b/rel/i18n/zh/emqx_authn_redis.hocon new file mode 100644 index 000000000..e3d6b9d96 --- /dev/null +++ b/rel/i18n/zh/emqx_authn_redis.hocon @@ -0,0 +1,18 @@ +emqx_authn_redis { + +cluster.desc: +"""使用 Redis (Cluster) 作为认证数据源的认证器的配置项。""" + +cmd.desc: +"""用于查询密码散列等用于认证的数据的 Redis Command，目前仅支持 HGET 与 HMGET。""" + +cmd.label: +"""Command""" + +sentinel.desc: +"""使用 Redis (Sentinel) 作为认证数据源的认证器的配置项。""" + +standalone.desc: +"""使用 Redis (Standalone) 作为认证数据源的认证器的配置项。""" + +} diff --git a/rel/i18n/zh/emqx_authn_schema.hocon b/rel/i18n/zh/emqx_authn_schema.hocon new file mode 100644 index 000000000..e6e76e8ab --- /dev/null +++ b/rel/i18n/zh/emqx_authn_schema.hocon @@ -0,0 +1,135 @@ +emqx_authn_schema { + +backend.desc: +"""后端类型。""" + +backend.label: +"""后端类型""" + +enable.desc: +"""设为 true 或 false 以启用或禁用此认证数据源。""" + +enable.label: +"""启用""" + +failed.desc: +"""请求失败次数。""" + +failed.label: +"""失败""" + +matched.desc: +"""请求命中次数。""" + +matched.label: +"""已命中""" + +mechanism.desc: +"""认证机制。""" + +mechanism.label: +"""认证机制""" + +metrics.desc: +"""资源统计指标。""" + +metrics.label: +"""指标""" + +metrics_failed.desc: +"""在当前实例中找到需要的认证信息，并且实例返回认证失败的次数。""" + +metrics_failed.label: +"""实例认证失败的次数""" + +metrics_nomatch.desc: +"""在当前实例中没有找到需要的认证信息，实例被忽略的次数。""" + +metrics_nomatch.label: +"""实例被忽略的次数""" + +metrics_rate.desc: +"""实例被触发的速率。触发速率等于匹配速率 + 忽略速率，单位：次/秒。""" + +metrics_rate.label: +"""实例被触发的速率""" + +metrics_rate_last5m.desc: +"""实例5分钟内平均触发速率，单位：次/秒。""" + +metrics_rate_last5m.label: +"""实例5分钟内平均触发速率""" + +metrics_rate_max.desc: +"""实例曾经达到的最高触发速率，单位：次/秒。""" + +metrics_rate_max.label: +"""实例曾经达到的最高触发速率""" + +metrics_success.desc: +"""在当前实例中找到需要的认证信息，并且实例返回认证成功的次数。""" + +metrics_success.label: +"""实例认证成功的次数""" + +metrics_total.desc: +"""当前实例被触发的总次数。""" + +metrics_total.label: +"""当前实例被触发的总次数""" + +node.desc: +"""节点名称。""" + +node.label: +"""节点名称。""" + +node_error.desc: +"""节点上产生的错误。""" + +node_error.label: +"""节点产生的错误""" + +node_metrics.desc: +"""每个节点上资源的统计指标。""" + +node_metrics.label: +"""节点资源指标""" + +node_status.desc: +"""每个节点上资源的状态。""" + +node_status.label: +"""节点资源状态""" + +rate.desc: +"""命中速率，单位：次/秒。""" + +rate.label: +"""速率""" + +rate_last5m.desc: +"""5分钟内平均命中速率，单位：次/秒。""" + +rate_last5m.label: +"""5分钟内速率""" + +rate_max.desc: +"""最大命中速率，单位：次/秒。""" + +rate_max.label: +"""最大速率""" + +status.desc: +"""资源状态。""" + +status.label: +"""状态""" + +success.desc: +"""请求成功次数。""" + +success.label: +"""成功""" + +} diff --git a/rel/i18n/zh/emqx_authn_user_import_api.hocon b/rel/i18n/zh/emqx_authn_user_import_api.hocon new file mode 100644 index 000000000..a546066bb --- /dev/null +++ b/rel/i18n/zh/emqx_authn_user_import_api.hocon @@ -0,0 +1,9 @@ +emqx_authn_user_import_api { + +authentication_id_import_users_post.desc: +"""为全局认证链上的指定认证器导入用户数据。""" + +listeners_listener_id_authentication_id_import_users_post.desc: +"""为监听器认证链上的指定认证器导入用户数据。""" + +} diff --git a/rel/i18n/zh/emqx_authz_api_cache.hocon b/rel/i18n/zh/emqx_authz_api_cache.hocon new file mode 100644 index 000000000..94486e3e0 --- /dev/null +++ b/rel/i18n/zh/emqx_authz_api_cache.hocon @@ -0,0 +1,6 @@ +emqx_authz_api_cache { + +authorization_cache_delete.desc: +"""清除集群中所有授权数据缓存。""" + +} diff --git a/rel/i18n/zh/emqx_authz_api_mnesia.hocon b/rel/i18n/zh/emqx_authz_api_mnesia.hocon new file mode 100644 index 000000000..bc5121cc3 --- /dev/null +++ b/rel/i18n/zh/emqx_authz_api_mnesia.hocon @@ -0,0 +1,87 @@ +emqx_authz_api_mnesia { + +action.desc: +"""被授权的行为 (发布/订阅/所有)""" + +action.label: +"""行为""" + +clientid.desc: +"""客户端标识符""" + +clientid.label: +"""客户端标识符""" + +fuzzy_clientid.desc: +"""使用字串匹配模糊搜索客户端标识符""" + +fuzzy_clientid.label: +"""客户端标识符子串""" + +fuzzy_username.desc: +"""使用字串匹配模糊搜索用户名""" + +fuzzy_username.label: +"""用户名子串""" + +permission.desc: +"""权限""" + +permission.label: +"""权限""" + +rules_all_delete.desc: +"""删除 `all` 规则""" + +rules_all_get.desc: +"""列出为所有客户端启用的规则列表""" + +rules_all_post.desc: +"""创建/更新 为所有客户端启用的规则列表。""" + +rules_delete.desc: +"""清除内置数据库中的所有类型（'users' 、'clients' 、'all'）的所有规则""" + +topic.desc: +"""在指定主题上的规则""" + +topic.label: +"""主题""" + +user_clientid_delete.desc: +"""删除内置数据库中指定客户端标识符类型的规则记录""" + +user_clientid_get.desc: +"""获取内置数据库中指定客户端标识符类型的规则记录""" + +user_clientid_put.desc: +"""更新内置数据库中指定客户端标识符类型的规则记录""" + +user_username_delete.desc: +"""删除内置数据库中指定用户名类型的规则记录""" + +user_username_get.desc: +"""获取内置数据库中指定用户名类型的规则记录""" + +user_username_put.desc: +"""更新内置数据库中指定用户名类型的规则记录""" + +username.desc: +"""用户名""" + +username.label: +"""用户名""" + +users_clientid_get.desc: +"""获取内置数据库中所有客户端标识符类型的规则记录""" + +users_clientid_post.desc: +"""添加内置数据库中客户端标识符类型的规则记录""" + +users_username_get.desc: +"""获取内置数据库中所有用户名类型的规则记录""" + +users_username_post.desc: +"""添加内置数据库中用户名类型的规则记录""" + +} diff --git a/rel/i18n/zh/emqx_authz_api_schema.hocon b/rel/i18n/zh/emqx_authz_api_schema.hocon new file mode 100644 index 000000000..41e3819cd --- /dev/null +++ b/rel/i18n/zh/emqx_authz_api_schema.hocon @@ -0,0 +1,90 @@ +emqx_authz_api_schema { + +body.desc: +"""HTTP 请求体。""" + +body.label: +"""请求体""" + +cmd.desc: +"""访问控制数据查询命令。""" + +cmd.label: +"""查询命令""" + +collection.desc: +"""`MongoDB` 授权数据集。""" + +collection.label: +"""数据集""" + +enable.desc: +"""设为 true 或 false 以启用或禁用此访问控制数据源。""" + +enable.label: +"""enable""" + +filter.desc: +"""在查询中定义过滤条件的条件表达式。 +过滤器支持如下占位符： +- ${username}: 将在运行时被替换为客户端连接时使用的用户名 +- ${clientid}: 将在运行时被替换为客户端连接时使用的客户端标识符""" + +filter.label: +"""过滤器""" + +headers.desc: +"""HTTP Headers 列表""" + +headers.label: +"""请求头""" + +headers_no_content_type.desc: +"""HTTP Headers 列表（无 content-type）。""" + +headers_no_content_type.label: +"""请求头(无 content-type)""" + +method.desc: +"""HTTP 请求方法。""" + +method.label: +"""method""" + +position.desc: +"""认证数据源位置。""" + +position.label: +"""位置""" + +query.desc: +"""访问控制数据查询语句。""" + +query.label: +"""查询语句""" + +request_timeout.desc: +"""请求超时时间。""" + +request_timeout.label: +"""请求超时""" + +rules.desc: +"""静态授权文件规则。""" + +rules.label: +"""规则""" + +type.desc: +"""数据后端类型。""" + +type.label: +"""type""" + +url.desc: +"""认证服务器 URL。""" + +url.label: +"""url""" + +} diff --git a/rel/i18n/zh/emqx_authz_api_settings.hocon b/rel/i18n/zh/emqx_authz_api_settings.hocon new file mode 100644 index 000000000..c78fb7a0b --- /dev/null +++ b/rel/i18n/zh/emqx_authz_api_settings.hocon @@ -0,0 +1,9 @@ +emqx_authz_api_settings { + +authorization_settings_get.desc: +"""获取授权配置""" + +authorization_settings_put.desc: +"""更新授权配置""" + +} diff --git a/rel/i18n/zh/emqx_authz_api_sources.hocon b/rel/i18n/zh/emqx_authz_api_sources.hocon new file mode 100644 index 000000000..8e9bfef9c --- /dev/null +++ b/rel/i18n/zh/emqx_authz_api_sources.hocon @@ -0,0 +1,48 @@ +emqx_authz_api_sources { + +authorization_sources_get.desc: +"""列出所有授权数据源""" + +authorization_sources_post.desc: +"""添加授权数据源""" + +authorization_sources_type_delete.desc: +"""删除指定类型的授权数据源""" + +authorization_sources_type_get.desc: +"""获取指定类型的授权数据源""" + +authorization_sources_type_move_post.desc: +"""更新授权数据源的优先执行顺序""" + +authorization_sources_type_put.desc: +"""更新指定类型的授权数据源""" + +authorization_sources_type_status_get.desc: +"""获取指定授权数据源的状态""" + +source.desc: +"""授权数据源""" + +source.label: +"""数据源""" + +source_config.desc: +"""数据源配置""" + +source_config.label: +"""数据源配置""" + +source_type.desc: +"""数据源类型""" + +source_type.label: +"""数据源类型""" + +sources.desc: +"""授权数据源列表""" + +sources.label: +"""数据源列表""" + +} diff --git a/rel/i18n/zh/emqx_authz_schema.hocon b/rel/i18n/zh/emqx_authz_schema.hocon new file mode 100644 index 000000000..3dd6d1c01 --- /dev/null +++ b/rel/i18n/zh/emqx_authz_schema.hocon @@ -0,0 +1,285 @@ +emqx_authz_schema { + +deny.desc: +"""授权失败的次数。""" + +deny.label: +"""授权失败次数""" + +redis_sentinel.desc: +"""使用 Redis 授权（哨兵模式）。""" + +redis_sentinel.label: +"""redis_sentinel""" + +rate.desc: +"""命中速率，单位：次/秒。""" + +rate.label: +"""速率""" + +status.desc: +"""资源状态。""" + +status.label: +"""状态""" + +method.desc: +"""HTTP 请求方法""" + +method.label: +"""method""" + +query.desc: +"""访问控制数据查询语句/查询命令。""" + +query.label: +"""查询语句""" + +metrics_total.desc: +"""授权实例被触发的总次数。""" + +metrics_total.label: +"""授权实例被触发的总次数""" + +redis_cluster.desc: +"""使用 Redis 授权（集群模式）。""" + +redis_cluster.label: +"""redis_cluster""" + +mysql.desc: +"""使用 MySOL 数据库授权""" + +mysql.label: +"""mysql""" + +postgresql.desc: +"""使用 PostgreSQL 数据库授权""" + +postgresql.label: +"""postgresql""" + +mongo_rs.desc: +"""使用 MongoDB 授权（副本集模式）""" + +mongo_rs.label: +"""mongo_rs""" + +type.desc: +"""数据后端类型""" + +type.label: +"""type""" + +mongo_sharded.desc: +"""使用 MongoDB 授权（分片集群模式）。""" + +mongo_sharded.label: +"""mongo_sharded""" + +body.desc: +"""HTTP request body。""" + +body.label: +"""Request Body""" + +url.desc: +"""授权 HTTP 服务器地址。""" + +url.label: +"""URL""" + +node.desc: +"""节点名称。""" + +node.label: +"""节点名称。""" + +headers.desc: +"""HTTP Headers 列表""" + +headers.label: +"""请求头""" + +rate_last5m.desc: +"""5分钟内平均命中速率，单位：次/秒。""" + +rate_last5m.label: +"""5分钟内速率""" + +headers_no_content_type.desc: +"""HTTP Headers 列表 (无 content-type) 。""" + +headers_no_content_type.label: +"""请求头(无 content-type)""" + +node_error.desc: +"""节点上产生的错误。""" + +node_error.label: +"""节点产生的错误""" + +mnesia.desc: +"""使用内部数据库授权（mnesia）。""" + +mnesia.label: +"""mnesia""" + +enable.desc: +"""设为 true 或 false 以启用或禁用此访问控制数据源""" + +enable.label: +"""enable""" + +matched.desc: +"""请求命中次数。""" + +matched.label: +"""已命中""" + +node_status.desc: +"""每个节点上资源的状态。""" + +node_status.label: +"""节点资源状态""" + +rate_max.desc: +"""最大命中速率，单位：次/秒。""" + +rate_max.label: +"""最大速率""" + +filter.desc: +"""在查询中定义过滤条件的条件表达式。 +过滤器支持如下占位符：
+- ${username}：将在运行时被替换为客户端连接时使用的用户名
+- ${clientid}：将在运行时被替换为客户端连接时使用的客户端标识符""" + +filter.label: +"""过滤器""" + +path.desc: +"""包含 ACL 规则的文件路径。 +如果在启动 EMQX 节点前预先配置该路径， +那么可以将该文件置于任何 EMQX 可以访问到的位置。 + +如果从 EMQX Dashboard 或 HTTP API 创建或修改了规则集， +那么EMQX将会生成一个新的文件并将它存放在 `data_dir` 下的 `authz` 子目录中， +并从此弃用旧的文件。""" + +path.label: +"""path""" + +redis_single.desc: +"""使用 Redis 授权（单实例）。""" + +redis_single.label: +"""redis_single""" + +failed.desc: +"""请求失败次数。""" + +failed.label: +"""失败""" + +metrics.desc: +"""资源统计指标。""" + +metrics.label: +"""指标""" + +authorization.desc: +"""客户端授权相关配置""" + +authorization.label: +"""授权""" + +collection.desc: +"""`MongoDB` 授权数据集。""" + +collection.label: +"""数据集""" + +mongo_single.desc: +"""使用 MongoDB 授权（单实例）。""" + +mongo_single.label: +"""mongo_single""" + +file.desc: +"""使用静态文件授权""" + +file.label: +"""文件""" + +http_post.desc: +"""使用外部 HTTP 服务器授权(POST 请求)。""" + +http_post.label: +"""http_post""" + +request_timeout.desc: +"""HTTP 请求超时时长。""" + +request_timeout.label: +"""请求超时时间""" + +allow.desc: +"""授权成功的次数。""" + +allow.label: +"""授权成功次数""" + +cmd.desc: +"""访问控制数据查查询命令""" + +cmd.label: +"""查询命令""" + +nomatch.desc: +"""没有匹配到任何授权规则的次数。""" + +nomatch.label: +"""没有匹配到任何授权规则的次数""" + +sources.desc: +"""授权数据源。
+授权（ACL）数据源的列表。 +它被设计为一个数组，而不是一个散列映射， +所以可以作为链式访问控制。
+ +当授权一个 'publish' 或 'subscribe' 行为时， +该配置列表中的所有数据源将按顺序进行检查。 +如果在某个客户端未找到时(使用 ClientID 或 Username)。 +将会移动到下一个数据源。直至得到 'allow' 或 'deny' 的结果。
+ +如果在任何数据源中都未找到对应的客户端信息。 +配置的默认行为 ('authorization.no_match') 将生效。
+ +注意： +数据源使用 'type' 进行标识。 +使用同一类型的数据源多于一次不被允许。""" + +sources.label: +"""数据源""" + +node_metrics.desc: +"""每个节点上资源的统计指标。""" + +node_metrics.label: +"""节点资源指标""" + +success.desc: +"""请求成功次数。""" + +success.label: +"""成功""" + +http_get.desc: +"""使用外部 HTTP 服务器授权(GET 请求)。""" + +http_get.label: +"""http_get""" + +} diff --git a/rel/i18n/zh/emqx_auto_subscribe_api.hocon b/rel/i18n/zh/emqx_auto_subscribe_api.hocon new file mode 100644 index 000000000..7a42ece2c --- /dev/null +++ b/rel/i18n/zh/emqx_auto_subscribe_api.hocon @@ -0,0 +1,12 @@ +emqx_auto_subscribe_api { + +list_auto_subscribe_api.desc: +"""获取自动订阅主题列表""" + +update_auto_subscribe_api.desc: +"""更新自动订阅主题列表""" + +update_auto_subscribe_api_response409.desc: +"""超出自定订阅主题列表长度限制""" + +} diff --git a/rel/i18n/zh/emqx_auto_subscribe_schema.hocon b/rel/i18n/zh/emqx_auto_subscribe_schema.hocon new file mode 100644 index 000000000..f4156fe34 --- /dev/null +++ b/rel/i18n/zh/emqx_auto_subscribe_schema.hocon @@ -0,0 +1,48 @@ +emqx_auto_subscribe_schema { + +auto_subscribe.desc: +"""设备登录成功之后，通过预设的订阅表示符，为设备自动完成订阅。支持使用占位符。""" + +auto_subscribe.label: +"""自动订阅""" + +nl.desc: +"""缺省值为0， +MQTT v3.1.1：如果设备订阅了自己发布消息的主题，那么将收到自己发布的所有消息。 +MQTT v5：如果设备在订阅时将此选项设置为 1，那么服务端将不会向设备转发自己发布的消息""" + +nl.label: +"""No Local""" + +qos.desc: +"""缺省值为 0，服务质量， +QoS 0：消息最多传递一次，如果当时客户端不可用，则会丢失该消息。 +QoS 1：消息传递至少 1 次。 +QoS 2：消息仅传送一次。""" + +qos.label: +"""服务质量""" + +rap.desc: +"""缺省值为 0，这一选项用来指定服务端向客户端转发消息时是否要保留其中的 RETAIN 标识，注意这一选项不会影响保留消息中的 RETAIN 标识。因此当 Retain As Publish 选项被设置为 0 时，客户端直接依靠消息中的 RETAIN 标识来区分这是一个正常的转发消息还是一个保留消息，而不是去判断消息是否是自己订阅后收到的第一个消息（转发消息甚至可能会先于保留消息被发送，视不同 Broker 的具体实现而定）。""" + +rap.label: +"""Retain As Publish""" + +rh.desc: +"""指定订阅建立时服务端是否向客户端发送保留消息， +可选值 0：只要客户端订阅成功，服务端就发送保留消息。 +可选值 1：客户端订阅成功且该订阅此前不存在，服务端才发送保留消息。毕竟有些时候客户端重新发起订阅可能只是为了改变一下 QoS，并不意味着它想再次接收保留消息。 +可选值 2：即便客户订阅成功，服务端也不会发送保留消息。""" + +rh.label: +"""Retain Handling""" + +topic.desc: +"""订阅标识符，支持使用占位符，例如 client/${clientid}/username/${username}/host/${host}/port/${port} +必填，且不可为空字符串""" + +topic.label: +"""订阅标识符""" + +} diff --git a/rel/i18n/zh/emqx_bridge_api.hocon b/rel/i18n/zh/emqx_bridge_api.hocon new file mode 100644 index 000000000..06887a711 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_api.hocon @@ -0,0 +1,100 @@ +emqx_bridge_api { + +desc_api1.desc: +"""列出所有 Bridge""" + +desc_api1.label: +"""列出所有 Bridge""" + +desc_api2.desc: +"""通过类型和名字创建 Bridge""" + +desc_api2.label: +"""创建 Bridge""" + +desc_api3.desc: +"""通过 ID 获取 Bridge""" + +desc_api3.label: +"""获取 Bridge""" + +desc_api4.desc: +"""通过 ID 更新 Bridge""" + +desc_api4.label: +"""更新 Bridge""" + +desc_api5.desc: +"""通过 ID 删除 Bridge""" + +desc_api5.label: +"""删除 Bridge""" + +desc_api6.desc: +"""通过 ID 重置 Bridge 的计数""" + +desc_api6.label: +"""重置 Bridge 计数""" + +desc_api7.desc: +"""停止或启用所有节点上的桥接""" + +desc_api7.label: +"""集群 Bridge 操作""" + +desc_api8.desc: +"""在某个节点上停止/重新启动 Bridge。""" + +desc_api8.label: +"""单节点 Bridge 操作""" + +desc_api9.desc: +"""通过给定的 ID 测试创建一个新的桥接。
+ID 的格式必须为 ’{type}:{name}”""" + +desc_api9.label: +"""测试桥接创建""" + +desc_bridge_metrics.desc: +"""通过 Id 来获取桥接的指标信息""" + +desc_bridge_metrics.label: +"""获取桥接的指标""" + +desc_enable_bridge.desc: +"""启用或禁用所有节点上的桥接""" + +desc_enable_bridge.label: +"""是否启用集群内的桥接""" + +desc_param_path_enable.desc: +"""是否启用桥接""" + +desc_param_path_enable.label: +"""启用桥接""" + +desc_param_path_id.desc: +"""Bridge ID ， 格式为 {type}:{name}""" + +desc_param_path_id.label: +"""Bridge ID""" + +desc_param_path_node.desc: +"""节点名，比如 emqx@127.0.0.1""" + +desc_param_path_node.label: +"""节点名""" + +desc_param_path_operation_cluster.desc: +"""集群可用操作：停止、重新启动""" + +desc_param_path_operation_cluster.label: +"""集群可用操作""" + +desc_param_path_operation_on_node.desc: +"""节点可用操作：停止、重新启动""" + +desc_param_path_operation_on_node.label: +"""节点可用操作""" + +} diff --git a/rel/i18n/zh/emqx_bridge_kafka.hocon b/rel/i18n/zh/emqx_bridge_kafka.hocon new file mode 100644 index 000000000..31bae51d3 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_kafka.hocon @@ -0,0 +1,354 @@ +emqx_bridge_kafka { + +connect_timeout.desc: +"""建立 TCP 连接时的最大等待时长（若启用认证，这个等待时长也包含完成认证所需时间）。""" + +connect_timeout.label: +"""连接超时""" + +producer_opts.desc: +"""本地 MQTT 数据源和 Kafka 桥接的配置。""" + +producer_opts.label: +"""MQTT 到 Kafka""" + +min_metadata_refresh_interval.desc: +"""刷新 Kafka broker 和 Kafka 主题元数据段最短时间间隔。设置太小可能会增加 Kafka 压力。""" + +min_metadata_refresh_interval.label: +"""元数据刷新最小间隔""" + +kafka_producer.desc: +"""Kafka Producer 配置。""" + +kafka_producer.label: +"""Kafka Producer""" + +producer_buffer.desc: +"""配置消息缓存的相关参数。 + +当 EMQX 需要发送的消息超过 Kafka 处理能力，或者当 Kafka 临时下线时，EMQX 内部会将消息缓存起来。""" + +producer_buffer.label: +"""消息缓存""" + +socket_send_buffer.desc: +"""TCP socket 的发送缓存调优。默认值是针对高吞吐量的一个推荐值。""" + +socket_send_buffer.label: +"""Socket 发送缓存大小""" + +desc_name.desc: +"""桥接名字，可读描述""" + +desc_name.label: +"""桥接名字""" + +consumer_offset_commit_interval_seconds.desc: +"""指定 Kafka 消费组偏移量提交的时间间隔。""" + +consumer_offset_commit_interval_seconds.label: +"""偏移提交间隔""" + +consumer_max_batch_bytes.desc: +"""设置每次从 Kafka 拉取数据的字节数。如该配置小于 Kafka 消息的大小，可能会影响消费性能。""" + +consumer_max_batch_bytes.label: +"""拉取字节数""" + +socket_receive_buffer.desc: +"""TCP socket 的收包缓存调优。默认值是针对高吞吐量的一个推荐值。""" + +socket_receive_buffer.label: +"""Socket 收包缓存大小""" + +consumer_topic_mapping.desc: +"""指定 Kafka 主题和 MQTT 主题之间的映射关系。 应至少包含一项。""" + +consumer_topic_mapping.label: +"""主题映射关系""" + +producer_kafka_opts.desc: +"""Kafka 生产者参数。""" + +producer_kafka_opts.label: +"""生产者参数""" + +kafka_topic.desc: +"""Kafka 主题名称""" + +kafka_topic.label: +"""Kafka 主题名称""" + +consumer_kafka_topic.desc: +"""指定从哪个 Kafka 主题消费消息。""" + +consumer_kafka_topic.label: +"""Kafka 主题""" + +auth_username_password.desc: +"""基于用户名密码的认证。""" + +auth_username_password.label: +"""用户名密码认证""" + +auth_sasl_password.desc: +"""SASL 认证的密码。""" + +auth_sasl_password.label: +"""密码""" + +kafka_message_timestamp.desc: +"""生成 Kafka 消息时间戳的模版。该时间必需是一个整型数值（可以是字符串格式）例如 1661326462115 或 '1661326462115'。当所需的输入字段不存在，或不是一个整型时，则会使用当前系统时间。""" + +kafka_message_timestamp.label: +"""消息的时间戳""" + +buffer_mode.desc: +"""消息缓存模式。 +memory: 所有的消息都缓存在内存里。如果 EMQX 服务重启，缓存的消息会丢失。 +disk: 缓存到磁盘上。EMQX 重启后会继续发送重启前未发送完成的消息。 +hybrid: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制（配置项 segment_bytes 描述了该限制）后，后续的消息会缓存到磁盘上。与 memory 模式一样，如果 EMQX 服务重启，缓存的消息会丢失。""" + +buffer_mode.label: +"""缓存模式""" + +consumer_mqtt_qos.desc: +"""转发 MQTT 消息时使用的 QoS。""" + +consumer_mqtt_qos.label: +"""QoS""" + +consumer_key_encoding_mode.desc: +"""通过 MQTT 转发之前，如何处理 Kafka 消息的 Key。none 使用 Kafka 消息中的 Key 原始值，不进行编码。 注意：在这种情况下，Key 必须是一个有效的 UTF-8 字符串。 +base64 对收到的密钥或值使用 base-64 编码。""" + +consumer_key_encoding_mode.label: +"""Key 编码模式""" + +auth_gssapi_kerberos.desc: +"""使用 GSSAPI/Kerberos 认证。""" + +auth_gssapi_kerberos.label: +"""GSSAPI/Kerberos""" + +consumer_mqtt_opts.desc: +"""本地 MQTT 消息转发。""" + +consumer_mqtt_opts.label: +"""MQTT 转发""" + +auth_kerberos_principal.desc: +"""SASL GSSAPI 认证方法的 Kerberos principal，例如 client_name@MY.KERBEROS.REALM.MYDOMAIN.COM注意：这里使用的 realm 需要配置在 EMQX 服务器的 /etc/krb5.conf 中""" + +auth_kerberos_principal.label: +"""Kerberos Principal""" + +socket_opts.desc: +"""更多 Socket 参数设置。""" + +socket_opts.label: +"""Socket 参数""" + +consumer_mqtt_topic.desc: +"""设置 Kafka 消息向哪个本地 MQTT 主题转发消息。""" + +consumer_mqtt_topic.label: +"""MQTT主题""" + +consumer_offset_reset_policy.desc: +"""如不存在偏移量历史记录或历史记录失效，消费者应使用哪个偏移量开始消费。""" + +consumer_offset_reset_policy.label: +"""偏移重置策略""" + +partition_count_refresh_interval.desc: +"""配置 Kafka 刷新分区数量的时间间隔。 +EMQX 发现 Kafka 分区数量增加后，会开始按 partition_strategy 配置，把消息发送到新的分区中。""" + +partition_count_refresh_interval.label: +"""分区数量刷新间隔""" + +max_batch_bytes.desc: +"""最大消息批量字节数。大多数 Kafka 环境的默认最低值是 1 MB，EMQX 的默认值比 1 MB 更小是因为需要补偿 Kafka 消息编码所需要的额外字节（尤其是当每条消息都很小的情况下）。当单个消息的大小超过该限制时，它仍然会被发送，（相当于该批量中只有单个消息）。""" + +max_batch_bytes.label: +"""最大批量字节数""" + +required_acks.desc: +"""设置 Kafka leader 在返回给 EMQX 确认之前需要等待多少个 follower 的确认。 + +all_isr: 需要所有的在线复制者都确认。 +leader_only: 仅需要分区 leader 确认。 +none: 无需 Kafka 回复任何确认。""" + +required_acks.label: +"""Kafka 确认数量""" + +metadata_request_timeout.desc: +"""刷新元数据时最大等待时长。""" + +metadata_request_timeout.label: +"""元数据请求超时""" + +desc_type.desc: +"""桥接类型""" + +desc_type.label: +"""桥接类型""" + +socket_nodelay.desc: +"""设置‘true’让系统内核立即发送。否则当需要发送的内容很少时，可能会有一定延迟（默认 40 毫秒）。""" + +socket_nodelay.label: +"""是否关闭延迟发送""" + +authentication.desc: +"""认证参数。""" + +authentication.label: +"""认证""" + +buffer_memory_overload_protection.desc: +"""缓存模式是 memory 或 hybrid 时适用。当系统处于高内存压力时，从队列中丢弃旧的消息以减缓内存增长。内存压力值由配置项 sysmon.os.sysmem_high_watermark 决定。注意，该配置仅在 Linux 系统中有效。""" + +buffer_memory_overload_protection.label: +"""内存过载保护""" + +auth_sasl_mechanism.desc: +"""SASL 认证方法名称。""" + +auth_sasl_mechanism.label: +"""认证方法""" + +config_enable.desc: +"""启用（true）或停用该（false）Kafka 数据桥接。""" + +config_enable.label: +"""启用或停用""" + +consumer_mqtt_payload.desc: +"""用于转换收到的 Kafka 消息的模板。 默认情况下，它将使用 JSON 格式来序列化来自 Kafka 的所有字段。 这些字段包括：headers：一个包含字符串键值对的 JSON 对象。 +key：Kafka 消息的键（使用选择的编码方式编码）。 +offset：消息的偏移量。 +topic：Kafka 主题。 +ts: 消息的时间戳。 +ts_type：消息的时间戳类型，值可能是： create， append 或 undefined。 +value: Kafka 消息值（使用选择的编码方式编码）。""" + +consumer_mqtt_payload.label: +"""MQTT Payload Template""" + +consumer_opts.desc: +"""本地 MQTT 转发 和 Kafka 消费者配置。""" + +consumer_opts.label: +"""MQTT 到 Kafka""" + +kafka_consumer.desc: +"""Kafka 消费者配置。""" + +kafka_consumer.label: +"""Kafka 消费者""" + +desc_config.desc: +"""Kafka 桥接配置""" + +desc_config.label: +"""Kafka 桥接配置""" + +consumer_value_encoding_mode.desc: +"""通过 MQTT 转发之前，如何处理 Kafka 消息的 Value。none 使用 Kafka 消息中的 Value 原始值，不进行编码。 注意：在这种情况下，Value 必须是一个有效的 UTF-8 字符串。 +base64 对收到的 Value 使用 base-64 编码。""" + +consumer_value_encoding_mode.label: +"""Value 编码模式""" + +buffer_per_partition_limit.desc: +"""为每个 Kafka 分区设置的最大缓存字节数。当超过这个上限之后，老的消息会被丢弃，为新的消息腾出空间。""" + +buffer_per_partition_limit.label: +"""Kafka 分区缓存上限""" + +bootstrap_hosts.desc: +"""用逗号分隔的 host[:port] 主机列表。默认端口号为 9092。""" + +bootstrap_hosts.label: +"""主机列表""" + +consumer_max_rejoin_attempts.desc: +"""消费组成员允许重新加入小组的最大次数。如超过该配置次数后仍未能成功加入消费组，则会在等待一段时间后重试。""" + +consumer_max_rejoin_attempts.label: +"""最大的重新加入尝试""" + +kafka_message_key.desc: +"""生成 Kafka 消息 Key 的模版。如果模版生成后为空值，则会使用 Kafka 的 NULL ，而非空字符串。""" + +kafka_message_key.label: +"""消息的 Key""" + +kafka_message.desc: +"""用于生成 Kafka 消息的模版。""" + +kafka_message.label: +"""Kafka 消息模版""" + +mqtt_topic.desc: +"""MQTT 主题数据源由桥接指定，或留空由规则动作指定。""" + +mqtt_topic.label: +"""源 MQTT 主题""" + +kafka_message_value.desc: +"""生成 Kafka 消息 Value 的模版。如果模版生成后为空值，则会使用 Kafka 的 NULL，而非空字符串。""" + +kafka_message_value.label: +"""消息的 Value""" + +partition_strategy.desc: +"""设置消息发布时应该如何选择 Kafka 分区。 + +random: 为每个消息随机选择一个分区。 +key_dispatch: Hash Kafka message key to a partition number""" + +partition_strategy.label: +"""分区选择策略""" + +buffer_segment_bytes.desc: +"""当缓存模式是 disk 或 hybrid 时适用。该配置用于指定缓存到磁盘上的文件的大小。""" + +buffer_segment_bytes.label: +"""缓存文件大小""" + +consumer_kafka_opts.desc: +"""Kafka消费者配置。""" + +consumer_kafka_opts.label: +"""Kafka 消费者""" + +max_inflight.desc: +"""设置 Kafka 生产者（每个分区一个）在收到 Kafka 的确认前最多发送多少个请求（批量）。调大这个值通常可以增加吞吐量，但是，当该值设置大于 1 时存在消息乱序的风险。""" + +max_inflight.label: +"""飞行窗口""" + +auth_sasl_username.desc: +"""SASL 认证的用户名。""" + +auth_sasl_username.label: +"""用户名""" + +auth_kerberos_keytab_file.desc: +"""SASL GSSAPI 认证方法的 Kerberos keytab 文件。注意：该文件需要上传到 EMQX 服务器中，且运行 EMQX 服务的系统账户需要有读取权限。""" + +auth_kerberos_keytab_file.label: +"""Kerberos keytab 文件""" + +compression.desc: +"""压缩方法。""" + +compression.label: +"""压缩""" + +} diff --git a/rel/i18n/zh/emqx_bridge_mqtt_schema.hocon b/rel/i18n/zh/emqx_bridge_mqtt_schema.hocon new file mode 100644 index 000000000..669d50398 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_mqtt_schema.hocon @@ -0,0 +1,21 @@ +emqx_bridge_mqtt_schema { + +config.desc: +"""MQTT Bridge 的配置。""" + +config.label: +"""配置""" + +desc_name.desc: +"""Bridge 名字，Bridge 的可读描述""" + +desc_name.label: +"""Bridge 名字""" + +desc_type.desc: +"""Bridge 的类型""" + +desc_type.label: +"""Bridge 类型""" + +} diff --git a/rel/i18n/zh/emqx_bridge_schema.hocon b/rel/i18n/zh/emqx_bridge_schema.hocon new file mode 100644 index 000000000..7512efa67 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_schema.hocon @@ -0,0 +1,158 @@ +emqx_bridge_schema { + +bridges_mqtt.desc: +"""桥接到另一个 MQTT Broker 的 MQTT Bridge""" + +bridges_mqtt.label: +"""MQTT Bridge""" + +bridges_webhook.desc: +"""转发消息到 HTTP 服务器的 WebHook""" + +bridges_webhook.label: +"""WebHook""" + +desc_bridges.desc: +"""MQTT Bridge 配置""" + +desc_bridges.label: +"""MQTT Bridge 配置""" + +desc_enable.desc: +"""启用/禁用 Bridge""" + +desc_enable.label: +"""启用/禁用 Bridge""" + +desc_metrics.desc: +"""Bridge 计数""" + +desc_metrics.label: +"""Bridge 计数""" + +desc_node_metrics.desc: +"""节点的计数器""" + +desc_node_metrics.label: +"""节点的计数器""" + +desc_node_name.desc: +"""节点的名字""" + +desc_node_name.label: +"""节点名字""" + +desc_node_status.desc: +"""节点的状态""" + +desc_node_status.label: +"""节点的状态""" + +desc_status.desc: +"""Bridge 的连接状态
+- connecting: 启动时的初始状态。
+- connected: 桥接驱动健康检查正常。
+- disconnected: 当桥接无法通过健康检查。
+- stopped: 桥接处于停用状态。
+- inconsistent: 集群中有各节点汇报的状态不一致。""" + +desc_status.label: +"""Bridge 状态""" + +desc_status_reason.desc: +"""桥接连接失败的原因。""" + +desc_status_reason.label: +"""失败原因""" + +metric_dropped.desc: +"""被丢弃的消息个数。""" + +metric_dropped.label: +"""丢弃""" + +metric_dropped_other.desc: +"""因为其他原因被丢弃的消息个数。""" + +metric_dropped_other.label: +"""其他丢弃""" + +metric_dropped_queue_full.desc: +"""因为队列已满被丢弃的消息个数。""" + +metric_dropped_queue_full.label: +"""队列已满被丢弃""" + +metric_dropped_resource_not_found.desc: +"""因为资源不存在被丢弃的消息个数。""" + +metric_dropped_resource_not_found.label: +"""资源不存在被丢弃""" + +metric_dropped_resource_stopped.desc: +"""因为资源已停用被丢弃的消息个数。""" + +metric_dropped_resource_stopped.label: +"""资源停用被丢弃""" + +metric_inflight.desc: +"""已异步地发送但没有收到 ACK 的消息个数。""" + +metric_inflight.label: +"""已发送未确认""" + +metric_matched.desc: +"""Bridge 被匹配到（被请求）的次数。""" + +metric_matched.label: +"""匹配次数""" + +metric_queuing.desc: +"""当前被缓存到磁盘队列的消息个数。""" + +metric_queuing.label: +"""被缓存""" + +metric_rate.desc: +"""执行操作的速率，次/秒""" + +metric_rate.label: +"""速率""" + +metric_rate_last5m.desc: +"""5 分钟平均速率，次/秒""" + +metric_rate_last5m.label: +"""5 分钟平均速率""" + +metric_rate_max.desc: +"""执行操作的最大速率，次/秒""" + +metric_rate_max.label: +"""执行操作的最大速率""" + +metric_received.desc: +"""从远程系统收到的消息个数。""" + +metric_received.label: +"""已接收""" + +metric_retried.desc: +"""重试的次数。""" + +metric_retried.label: +"""已重试""" + +metric_sent_failed.desc: +"""发送失败的消息个数。""" + +metric_sent_failed.label: +"""发送失败""" + +metric_sent_success.desc: +"""已经发送成功的消息个数。""" + +metric_sent_success.label: +"""发送成功""" + +} diff --git a/rel/i18n/zh/emqx_bridge_webhook_schema.hocon b/rel/i18n/zh/emqx_bridge_webhook_schema.hocon new file mode 100644 index 000000000..d7dd9dae0 --- /dev/null +++ b/rel/i18n/zh/emqx_bridge_webhook_schema.hocon @@ -0,0 +1,87 @@ +emqx_bridge_webhook_schema { + +config_body.desc: +"""HTTP 请求的正文。
+如果没有设置该字段，请求正文将是包含所有可用字段的 JSON object。
+如果该 webhook 是由于收到 MQTT 消息触发的，'所有可用字段' 将是 MQTT 消息的 +上下文信息；如果该 webhook 是由于规则触发的，'所有可用字段' 则为触发事件的上下文信息。
+允许使用带有变量的模板。""" + +config_body.label: +"""HTTP 请求正文""" + +config_direction.desc: +"""已废弃，Bridge 的方向，必须是 egress""" + +config_direction.label: +"""Bridge 方向""" + +config_enable.desc: +"""启用/禁用 Bridge""" + +config_enable.label: +"""启用/禁用 Bridge""" + +config_headers.desc: +"""HTTP 请求的标头。
+允许使用带有变量的模板。""" + +config_headers.label: +"""HTTP 请求标头""" + +config_local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 HTTP 服务器。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HTTP 服务器。""" + +config_local_topic.label: +"""本地 Topic""" + +config_max_retries.desc: +"""HTTP 请求失败最大重试次数""" + +config_max_retries.label: +"""HTTP 请求重试次数""" + +config_method.desc: +"""HTTP 请求的方法。 所有可用的方法包括：post、put、get、delete。
+允许使用带有变量的模板。""" + +config_method.label: +"""HTTP 请求方法""" + +config_request_timeout.desc: +"""HTTP 请求超时""" + +config_request_timeout.label: +"""HTTP 请求超时""" + +config_url.desc: +"""HTTP Bridge 的 URL。
+路径中允许使用带变量的模板，但是 host， port 不允许使用变量模板。
+例如， http://localhost:9901/${topic} 是允许的， +但是 http://${host}:9901/message +或 http://localhost:${port}/message +不允许。""" + +config_url.label: +"""HTTP Bridge""" + +desc_config.desc: +"""HTTP Bridge 配置""" + +desc_config.label: +"""HTTP Bridge 配置""" + +desc_name.desc: +"""Bridge 名字，Bridge 的可读描述""" + +desc_name.label: +"""Bridge 名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""Bridge 类型""" + +} diff --git a/rel/i18n/zh/emqx_coap_api.hocon b/rel/i18n/zh/emqx_coap_api.hocon new file mode 100644 index 000000000..f3bee543e --- /dev/null +++ b/rel/i18n/zh/emqx_coap_api.hocon @@ -0,0 +1,27 @@ +emqx_coap_api { + +content_type.desc: +"""Payload 类型""" + +message_id.desc: +"""消息 ID""" + +method.desc: +"""请求 Method 类型""" + +payload.desc: +"""Payload 内容""" + +response_code.desc: +"""应答码""" + +send_coap_request.desc: +"""发送 CoAP 消息到指定客户端""" + +timeout.desc: +"""请求超时时间""" + +token.desc: +"""消息 Token, 可以为空""" + +} diff --git a/rel/i18n/zh/emqx_coap_schema.hocon b/rel/i18n/zh/emqx_coap_schema.hocon new file mode 100644 index 000000000..132c0f03a --- /dev/null +++ b/rel/i18n/zh/emqx_coap_schema.hocon @@ -0,0 +1,37 @@ +emqx_coap_schema { + +coap.desc: +"""CoAP 网关配置。 +该网关的实现基于 RFC-7252 和 https://core-wg.github.io/coap-pubsub/draft-ietf-core-pubsub.html""" + +coap_connection_required.desc: +"""是否开启连接模式。 +连接模式是非标准协议的功能。它维护 CoAP 客户端上线、认证、和连接状态的保持""" + +coap_heartbeat.desc: +"""CoAP 网关要求客户端的最小心跳间隔时间。 +当 connection_required 开启后，该参数用于检查客户端连接是否存活""" + +coap_notify_type.desc: +"""投递给 CoAP 客户端的通知消息类型。当客户端 Observe 一个资源（或订阅某个主题）时，网关会向客户端推送新产生的消息。其消息类型可设置为：
+ - non: 不需要客户端返回确认消息;
+ - con: 需要客户端返回一个确认消息;
+ - qos: 取决于消息的 QoS 等级; QoS 0 会以 `non` 类型下发，QoS 1/2 会以 `con` 类型下发""" + +coap_publish_qos.desc: +"""客户端发布请求的默认 QoS 等级。 +当 CoAP 客户端发起发布请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
+ - qos0、qos1、qos2: 设置为固定的 QoS 等级
+ - coap: 依据发布操作的 CoAP 报文类型来动态决定
+ * 当发布请求为 `non-confirmable` 类型时，取值为 qos0
+ * 当发布请求为 `confirmable` 类型时，取值为 qos1""" + +coap_subscribe_qos.desc: +"""客户端订阅请求的默认 QoS 等级。 +当 CoAP 客户端发起订阅请求时，如果未携带 `qos` 参数则会使用该默认值。默认值可设置为：
+ - qos0、 qos1、qos2: 设置为固定的 QoS 等级
+ - coap: 依据订阅操作的 CoAP 报文类型来动态决定
+ * 当订阅请求为 `non-confirmable` 类型时，取值为 qos0
+ * 当订阅请求为 `confirmable` 类型时，取值为 qos1""" + +} diff --git a/rel/i18n/zh/emqx_conf_schema.hocon b/rel/i18n/zh/emqx_conf_schema.hocon new file mode 100644 index 000000000..4c1edbdee --- /dev/null +++ b/rel/i18n/zh/emqx_conf_schema.hocon @@ -0,0 +1,774 @@ +emqx_conf_schema { + +common_handler_drop_mode_qlen.desc: +"""当缓冲的日志事件数大于此值时，新的日志事件将被丢弃。起到过载保护的功能。 +为了使过载保护算法正常工作必须要： sync_mode_qlen =< drop_mode_qlen =< flush_qlen 且 drop_mode_qlen > 1 +要禁用某些模式，请执行以下操作。 +- 如果sync_mode_qlen被设置为0，所有的日志事件都被同步处理。也就是说，异步日志被禁用。 +- 如果sync_mode_qlen被设置为与drop_mode_qlen相同的值，同步模式被禁用。也就是说，处理程序总是以异步模式运行，除非调用drop或flushing。 +- 如果drop_mode_qlen被设置为与flush_qlen相同的值，则drop模式被禁用，永远不会发生。""" + +common_handler_drop_mode_qlen.label: +"""进入丢弃模式的队列长度""" + +cluster_mcast_addr.desc: +"""指定多播 IPv4 地址。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_addr.label: +"""多播地址""" + +desc_cluster_dns.desc: +"""DNS SRV 记录服务发现。""" + +desc_cluster_dns.label: +"""DNS SRV 记录服务发现""" + +cluster_dns_name.desc: +"""指定 DNS A 记录的名字。emqx 会通过访问这个 DNS A 记录来获取 IP 地址列表。 +当cluster.discovery_strategy 为 dns 时有效。""" + +cluster_dns_name.label: +"""DNS名称""" + +rpc_keyfile.desc: +"""rpc.certfile 的私钥文件的路径。
+注意：此文件内容是私钥，所以需要设置权限为 600。""" + +rpc_keyfile.label: +"""RPC 私钥文件""" + +cluster_mcast_recbuf.desc: +"""接收数据报的内核级缓冲区的大小。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_recbuf.label: +"""多播接收数据缓冲区""" + +cluster_autoheal.desc: +"""集群脑裂自动恢复机制开关。""" + +cluster_autoheal.label: +"""节点脑裂自动修复机制""" + +log_overload_kill_enable.desc: +"""日志处理进程过载时为保护自己节点其它的业务能正常，强制杀死日志处理进程。""" + +log_overload_kill_enable.label: +"""日志处理进程过载保护""" + +node_etc_dir.desc: +"""etc 存放目录""" + +node_etc_dir.label: +"""Etc 目录""" + +cluster_proto_dist.desc: +"""分布式 Erlang 集群协议类型。可选值为:
+- inet_tcp: 使用 IPv4
+- inet_tls: 使用 TLS，需要配合 etc/ssl_dist.conf 一起使用。""" + +cluster_proto_dist.label: +"""集群内部通信协议""" + +log_burst_limit_enable.desc: +"""启用日志限流保护机制。""" + +log_burst_limit_enable.label: +"""日志限流保护""" + +dist_buffer_size.desc: +"""Erlang分布式缓冲区的繁忙阈值，单位是KB。""" + +dist_buffer_size.label: +"""Erlang分布式缓冲区的繁忙阈值(KB)""" + +common_handler_max_depth.desc: +"""Erlang 内部格式日志格式化和 Erlang 进程消息队列检查的最大深度。""" + +common_handler_max_depth.label: +"""最大深度""" + +desc_log.desc: +"""EMQX 日志记录支持日志事件的多个接收器。 每个接收器由一个_log handler_表示，可以独立配置。""" + +desc_log.label: +"""日志""" + +common_handler_flush_qlen.desc: +"""如果缓冲日志事件的数量增长大于此阈值，则会发生冲刷（删除）操作。 日志处理进程会丢弃缓冲的日志消息。 +来缓解自身不会由于内存瀑涨而影响其它业务进程。日志内容会提醒有多少事件被删除。""" + +common_handler_flush_qlen.label: +"""冲刷阈值""" + +common_handler_chars_limit.desc: +"""设置单个日志消息的最大长度。 如果超过此长度，则日志消息将被截断。最小可设置的长度为100。 +注意：如果日志格式为 JSON，限制字符长度可能会导致截断不完整的 JSON 数据。""" + +common_handler_chars_limit.label: +"""单条日志长度限制""" + +cluster_k8s_namespace.desc: +"""当使用 k8s 方式并且 cluster.k8s.address_type 指定为 dns 类型时， +可设置 emqx 节点名的命名空间。与 cluster.k8s.suffix 一起使用用以拼接得到节点名列表。""" + +cluster_k8s_namespace.label: +"""K8s 命名空间""" + +node_name.desc: +"""节点名。格式为 \@\。其中 可以是 IP 地址，也可以是 FQDN。 +详见 http://erlang.org/doc/reference_manual/distributed.html。""" + +node_name.label: +"""节点名""" + +rpc_port_discovery.desc: +"""manual: 通过 tcp_server_port 来发现端口。 +
stateless: 使用无状态的方式来发现端口，使用如下算法。如果节点名称是 +emqxN@127.0.0.1, N 是一个数字，那么监听端口就是 5370 + N。""" + +rpc_port_discovery.label: +"""RPC 端口发现策略""" + +log_overload_kill_restart_after.desc: +"""处理进程停止后，会在该延迟时间后自动重新启动。除非该值设置为 infinity，这会阻止任何后续的重启。""" + +log_overload_kill_restart_after.label: +"""处理进程重启延迟""" + +log_file_handler_max_size.desc: +"""此参数控制日志文件轮换。 `infinity` 意味着日志文件将无限增长，否则日志文件将在达到 `max_size`（以字节为单位）时进行轮换。 +与 rotation count配合使用。如果 counter 为 10，则是10个文件轮换。""" + +log_file_handler_max_size.label: +"""日志文件轮换大小""" + +desc_log_file_handler.desc: +"""日志处理进程将日志事件打印到文件。""" + +desc_log_file_handler.label: +"""文件日志处理进程""" + +rpc_socket_keepalive_count.desc: +"""keepalive 探测消息发送失败的次数，直到 RPC 连接被认为已经断开。""" + +rpc_socket_keepalive_count.label: +"""RPC Socket Keepalive 次数""" + +cluster_etcd_server.desc: +"""指定 etcd 服务的地址。如有多个服务使用逗号 , 分隔。 +当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" + +cluster_etcd_server.label: +"""Etcd 服务器地址""" + +db_backend.desc: +"""配置后端数据库驱动，默认值为 rlog 它适用于大规模的集群。 +mnesia 是备选数据库，适合中小集群。""" + +db_backend.label: +"""内置数据库""" + +desc_authorization.desc: +"""授权相关""" + +desc_authorization.label: +"""授权""" + +cluster_etcd_ssl.desc: +"""当使用 TLS 连接 etcd 时的配置选项。 +当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" + +cluster_etcd_ssl.label: +"""Etcd SSL 选项""" + +rpc_insecure_fallback.desc: +"""兼容旧的无鉴权模式""" + +rpc_insecure_fallback.label: +"""向后兼容旧的无鉴权模式""" + +cluster_mcast_buffer.desc: +"""用户级缓冲区的大小。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_buffer.label: +"""多播用户级缓冲区""" + +rpc_authentication_timeout.desc: +"""远程节点认证的超时时间。""" + +rpc_authentication_timeout.label: +"""RPC 认证超时时间""" + +cluster_call_retry_interval.desc: +"""当集群间调用出错时，多长时间重试一次。""" + +cluster_call_retry_interval.label: +"""重试时间间隔""" + +cluster_mcast_sndbuf.desc: +"""外发数据报的内核级缓冲区的大小。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_sndbuf.label: +"""多播发送缓存区""" + +rpc_driver.desc: +"""集群间通信使用的传输协议。""" + +rpc_driver.label: +"""RPC 驱动""" + +max_ets_tables.desc: +"""Erlang ETS 表的最大数量""" + +max_ets_tables.label: +"""Erlang 表的最大数量""" + +desc_db.desc: +"""内置数据库的配置。""" + +desc_db.label: +"""数据库""" + +desc_cluster_etcd.desc: +"""使用 'etcd' 服务的服务发现。""" + +desc_cluster_etcd.label: +"""'etcd' 服务的服务发现""" + +cluster_name.desc: +"""EMQX集群名称。每个集群都有一个唯一的名称。服务发现时会用于做路径的一部分。""" + +cluster_name.label: +"""集群名称""" + +log_rotation_enable.desc: +"""启用日志轮换功能。启动后生成日志文件后缀会加上对应的索引数字，比如：log/emqx.log.1。 +系统会默认生成*.siz/*.idx用于记录日志位置，请不要手动修改这两个文件。""" + +log_rotation_enable.label: +"""日志轮换""" + +cluster_call_cleanup_interval.desc: +"""清理过期事务的时间间隔""" + +cluster_call_cleanup_interval.label: +"""清理间隔""" + +desc_cluster_static.desc: +"""静态节点服务发现。新节点通过连接一个节点来加入集群。""" + +desc_cluster_static.label: +"""静态节点服务发现""" + +db_default_shard_transport.desc: +"""定义用于推送事务日志的默认传输。
+这可以在 db.shard_transports 中基于每个分片被覆盖。 +gen_rpc 使用 gen_rpc 库， +distr 使用 Erlang 发行版。""" + +db_default_shard_transport.label: +"""事务日志传输默认协议""" + +cluster_static_seeds.desc: +"""集群中的EMQX节点名称列表， +指定固定的节点列表，多个节点间使用逗号 , 分隔。 +当 cluster.discovery_strategy 为 static 时，此配置项才有效。 +适合于节点数量较少且固定的集群。""" + +cluster_static_seeds.label: +"""集群静态节点""" + +log_overload_kill_qlen.desc: +"""允许的最大队列长度。""" + +log_overload_kill_qlen.label: +"""最大队列长度""" + +node_backtrace_depth.desc: +"""错误信息中打印的最大堆栈层数""" + +node_backtrace_depth.label: +"""最大堆栈导数""" + +desc_log_burst_limit.desc: +"""短时间内产生的大量日志事件可能会导致问题，例如： + - 日志文件变得非常大 + - 日志文件轮换过快，有用信息被覆盖 + - 对系统的整体性能影响 + +日志突发限制功能可以暂时禁用日志记录以避免这些问题。""" + +desc_log_burst_limit.label: +"""日志突发限制""" + +common_handler_enable.desc: +"""启用此日志处理进程。""" + +common_handler_enable.label: +"""启用日志处理进程""" + +cluster_k8s_service_name.desc: +"""指定 Kubernetes 中 EMQX 的服务名。 +当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。""" + +cluster_k8s_service_name.label: +"""K8s 服务别名""" + +log_rotation_count.desc: +"""轮换的最大日志文件数。""" + +log_rotation_count.label: +"""最大日志文件数""" + +node_cookie.desc: +"""分布式 Erlang 集群使用的 cookie 值。集群间保持一致""" + +node_cookie.label: +"""节点 Cookie""" + +db_role.desc: +"""选择节点的角色。
+core 节点提供数据的持久性，并负责写入。建议将核心节点放置在不同的机架或不同的可用区。
+repliant 节点是临时工作节点。 从集群中删除它们，不影响数据库冗余
+建议复制节点多于核心节点。
+注意：该参数仅在设置backend时生效到 rlog。""" + +db_role.label: +"""数据库角色""" + +rpc_tcp_server_port.desc: +"""RPC 本地服务使用的 TCP 端口。
+只有当 rpc.port_discovery 设置为 manual 时，此配置才会生效。""" + +rpc_tcp_server_port.label: +"""RPC TCP 服务监听端口""" + +desc_console_handler.desc: +"""日志处理进程将日志事件打印到 EMQX 控制台。""" + +desc_console_handler.label: +"""Console Handler""" + +node_applications.desc: +"""当新EMQX 加入集群时，应重启的Erlang应用程序的列表。""" + +node_applications.label: +"""应用""" + +log_burst_limit_max_count.desc: +"""在 `window_time` 间隔内处理的最大日志事件数。 达到限制后，将丢弃连续事件，直到 `window_time` 结束。""" + +log_burst_limit_max_count.label: +"""日志事件数""" + +rpc_tcp_client_num.desc: +"""设置本节点与远程节点之间的 RPC 通信通道的最大数量。""" + +rpc_tcp_client_num.label: +"""RPC TCP 客户端数量""" + +cluster_k8s_address_type.desc: +"""当使用 k8s 方式集群时，address_type 用来从 Kubernetes 接口的应答里获取什么形式的 Host 列表。 +指定 cluster.k8s.address_type 为 ip，则将从 Kubernetes 接口中获取集群中其他节点 +的IP地址。""" + +cluster_k8s_address_type.label: +"""K8s 地址类型""" + +rpc_socket_sndbuf.desc: +"""TCP 调节参数。TCP 发送缓冲区大小。""" + +rpc_socket_sndbuf.label: +"""RPC 套接字发送缓冲区大小""" + +cluster_mcast_ttl.desc: +"""指定多播的 Time-To-Live 值。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_ttl.label: +"""多播TTL""" + +db_core_nodes.desc: +"""当前节点连接的核心节点列表。
+注意：该参数仅在设置backend时生效到 rlog +并且设置role为replicant时生效。
+该值需要在手动或静态集群发现机制下设置。
+如果使用了自动集群发现机制（如etcd），则不需要设置该值。""" + +db_core_nodes.label: +"""数据库核心节点""" + +log_file_handler_file.desc: +"""日志文件路径及名字。""" + +log_file_handler_file.label: +"""日志文件名字""" + +node_dist_net_ticktime.desc: +"""系统调优参数，此配置将覆盖 vm.args 文件里的 -kernel net_ticktime 参数。当一个节点持续无响应多久之后，认为其已经宕机并断开连接。""" + +node_dist_net_ticktime.label: +"""节点间心跳间隔""" + +desc_cluster_k8s.desc: +"""Kubernetes 服务发现。""" + +desc_cluster_k8s.label: +"""Kubernetes 服务发现""" + +desc_cluster_mcast.desc: +"""UDP 组播服务发现。""" + +desc_cluster_mcast.label: +"""UDP 组播服务发现""" + +rpc_cacertfile.desc: +"""验证 rpc.certfile 的 CA 证书文件的路径。
+注意：集群中所有节点的证书必须使用同一个 CA 签发。""" + +rpc_cacertfile.label: +"""RPC CA 证书文件""" + +desc_node.desc: +"""节点名称、Cookie、配置文件、数据目录和 Erlang 虚拟机（BEAM）启动参数。""" + +desc_node.label: +"""节点""" + +cluster_k8s_apiserver.desc: +"""指定 Kubernetes API Server。如有多个 Server 使用逗号 , 分隔。 +当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。""" + +cluster_k8s_apiserver.label: +"""K8s 服务地址""" + +common_handler_supervisor_reports.desc: +"""Supervisor 报告的类型。默认为 error 类型。
+ - error：仅记录 Erlang 进程中的错误。 + - progress：除了 error 信息外，还需要记录进程启动的详细信息。""" + +common_handler_supervisor_reports.label: +"""报告类型""" + +node_data_dir.desc: +"""节点数据存放目录，可能会自动创建的子目录如下：
+- `mnesia/`。EMQX的内置数据库目录。例如，`mnesia/emqx@127.0.0.1`。
+如果节点要被重新命名（例如，`emqx@10.0.1.1`）。旧目录应该首先被删除。
+- `configs`。在启动时生成的配置，以及集群/本地覆盖的配置。
+- `patches`: 热补丁文件将被放在这里。
+- `trace`: 日志跟踪文件。
+ +**注意**: 一个数据dir不能被两个或更多的EMQX节点同时使用。""" + +node_data_dir.label: +"""节点数据目录""" + +cluster_k8s_suffix.desc: +"""当使用 k8s 方式并且 cluster.k8s.address_type 指定为 dns 类型时，可设置 emqx 节点名的后缀。 +与 cluster.k8s.namespace 一起使用用以拼接得到节点名列表。""" + +cluster_k8s_suffix.label: +"""K8s 前缀""" + +db_rpc_module.desc: +"""集群间推送事务日志到复制节点使用的协议。""" + +db_rpc_module.label: +"""RPC协议""" + +cluster_etcd_prefix.desc: +"""指定 etcd 路径的前缀。每个节点在 etcd 中都会创建一个路径: +v2/keys///
+当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" + +cluster_etcd_prefix.label: +"""Etcd 路径前缀""" + +cluster_mcast_iface.desc: +"""指定节点发现服务需要绑定到本地 IP 地址。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_iface.label: +"""多播绑定地址""" + +log_burst_limit_window_time.desc: +"""参考 max_count。""" + +log_burst_limit_window_time.label: +"""Window Time""" + +cluster_dns_record_type.desc: +"""DNS 记录类型。""" + +cluster_dns_record_type.label: +"""DNS记录类型""" + +cluster_autoclean.desc: +"""指定多久之后从集群中删除离线节点。""" + +cluster_autoclean.label: +"""自动删除离线节点时间""" + +process_limit.desc: +"""Erlang系统同时存在的最大进程数。 +实际选择的最大值可能比设置的数字大得多。 +参考: https://www.erlang.org/doc/man/erl.html""" + +process_limit.label: +"""Erlang 最大进程数""" + +max_ports.desc: +"""Erlang系统同时存在的最大端口数。 +实际选择的最大值可能比设置的数字大得多。 +参考: https://www.erlang.org/doc/man/erl.html""" + +max_ports.label: +"""Erlang 最大端口数""" + +desc_log_rotation.desc: +"""默认情况下，日志存储在 `./log` 目录（用于从 zip 文件安装）或 `/var/log/emqx`（用于二进制安装）。
+这部分配置，控制每个日志处理进程保留的文件数量。""" + +desc_log_rotation.label: +"""日志轮换""" + +desc_log_overload_kill.desc: +"""日志过载终止，具有过载保护功能。当日志处理进程使用过多内存，或者缓存的日志消息过多时该功能被激活。
+检测到过载时，日志处理进程将终止，并在冷却期后重新启动。""" + +desc_log_overload_kill.label: +"""日志过载保护""" + +authorization.desc: +"""授权（ACL）。EMQX 支持完整的客户端访问控制（ACL）。""" + +authorization.label: +"""授权""" + +rpc_socket_keepalive_idle.desc: +"""broker 之间的连接在最后一条消息发送后保持打开的时间。""" + +rpc_socket_keepalive_idle.label: +"""RPC Socket Keepalive Idle""" + +desc_cluster_call.desc: +"""集群调用功能的选项。""" + +desc_cluster_call.label: +"""集群调用""" + +cluster_mcast_ports.desc: +"""指定多播端口。如有多个端口使用逗号 , 分隔。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_ports.label: +"""多播端口""" + +log_overload_kill_mem_size.desc: +"""日志处理进程允许使用的最大内存。""" + +log_overload_kill_mem_size.label: +"""日志处理进程允许使用的最大内存""" + +rpc_connect_timeout.desc: +"""建立 RPC 连接的超时时间。""" + +rpc_connect_timeout.label: +"""RPC 连接超时时间""" + +cluster_etcd_node_ttl.desc: +"""指定 etcd 中节点信息的过期时间。 +当 cluster.discovery_strategy 为 etcd 时，此配置项才有效。""" + +cluster_etcd_node_ttl.label: +"""Etcd 节点过期时间""" + +rpc_call_receive_timeout.desc: +"""同步 RPC 的回复超时时间。""" + +rpc_call_receive_timeout.label: +"""RPC 调用接收超时时间""" + +rpc_socket_recbuf.desc: +"""TCP 调节参数。TCP 接收缓冲区大小。""" + +rpc_socket_recbuf.label: +"""RPC 套接字接收缓冲区大小""" + +db_tlog_push_mode.desc: +"""同步模式下，核心节点等待复制节点的确认信息，然后再发送下一条事务日志。""" + +db_tlog_push_mode.label: +"""Tlog推送模式""" + +node_crash_dump_bytes.desc: +"""限制崩溃文件的大小，当崩溃时节点内存太大， +如果为了保存现场，需要全部存到崩溃文件中，此处限制最多能保存多大的文件。 +如果超过此限制，崩溃转储将被截断。如果设置为0，系统不会尝试写入崩溃转储文件。""" + +node_crash_dump_bytes.label: +"""崩溃文件最大容量""" + +rpc_certfile.desc: +"""TLS 证书文件的路径，用于验证集群节点的身份。 +只有当 rpc.driver 设置为 ssl 时，此配置才会生效。""" + +rpc_certfile.label: +"""RPC 证书文件""" + +node_crash_dump_seconds.desc: +"""该配置给出了运行时系统允许花费的写入崩溃转储的秒数。当给定的秒数已经过去，运行时系统将被终止。
+- 如果设置为0秒，运行时会立即终止，不会尝试写入崩溃转储文件。
+- 如果设置为一个正数 S，节点会等待 S 秒来完成崩溃转储文件，然后用SIGALRM信号终止运行时系统。
+- 如果设置为一个负值导致运行时系统的终止等待无限期地直到崩溃转储文件已经完全写入。""" + +node_crash_dump_seconds.label: +"""保存崩溃文件最长时间""" + +log_file_handlers.desc: +"""输出到文件的日志处理进程列表""" + +log_file_handlers.label: +"""File Handler""" + +node_global_gc_interval.desc: +"""系统调优参数，设置节点运行多久强制进行一次全局垃圾回收。禁用设置为 disabled。""" + +node_global_gc_interval.label: +"""全局垃圾回收""" + +common_handler_time_offset.desc: +"""日志中的时间戳使用的时间偏移量。 +可选值为： + - system: 本地系统使用的时区偏移量 + - utc: 0 时区的偏移量 + - +-[hh]:[mm]: 自定义偏移量，比如 "-02:00" 或者 "+00:00" +默认值为本地系统的时区偏移量：system。""" + +common_handler_time_offset.label: +"""时间偏移量""" + +rpc_mode.desc: +"""在 sync 模式下，发送端等待接收端的 ack信号。""" + +rpc_mode.label: +"""RPC 模式""" + +node_crash_dump_file.desc: +"""设置 Erlang crash_dump 文件的存储路径和文件名。""" + +node_crash_dump_file.label: +"""节点崩溃时的Dump文件""" + +cluster_mcast_loop.desc: +"""设置多播的报文是否投递到本地回环地址。 +当 cluster.discovery_strategy 为 mcast 时，此配置项才有效。""" + +cluster_mcast_loop.label: +"""多播回环开关""" + +rpc_socket_keepalive_interval.desc: +"""keepalive 消息的间隔。""" + +rpc_socket_keepalive_interval.label: +"""RPC Socket Keepalive 间隔""" + +common_handler_level.desc: +"""当前日志处理进程的日志级别。 +默认为 warning 级别。""" + +common_handler_level.label: +"""日志级别""" + +desc_rpc.desc: +"""EMQX 使用 gen_rpc 库来实现跨节点通信。
+大多数情况下，默认的配置应该可以工作，但如果你需要做一些性能优化或者实验，可以尝试调整这些参数。""" + +desc_rpc.label: +"""RPC""" + +rpc_ssl_server_port.desc: +"""RPC 本地服务使用的监听SSL端口。
+只有当 rpc.port_discovery 设置为 manual 且 dirver 设置为 ssl， +此配置才会生效。""" + +rpc_ssl_server_port.label: +"""RPC SSL 服务监听端口""" + +desc_cluster.desc: +"""EMQX 节点可以组成一个集群，以提高总容量。
这里指定了节点之间如何连接。""" + +desc_cluster.label: +"""集群""" + +common_handler_sync_mode_qlen.desc: +"""只要缓冲的日志事件的数量低于这个值，所有的日志事件都会被异步处理。 +这意味着，日志落地速度不会影响正常的业务进程，因为它们不需要等待日志处理进程的响应。 +如果消息队列的增长超过了这个值，处理程序开始同步处理日志事件。也就是说，发送事件的客户进程必须等待响应。 +当处理程序将消息队列减少到低于sync_mode_qlen阈值的水平时，异步操作就会恢复。 +默认为100条信息，当等待的日志事件大于100条时，就开始同步处理日志。""" + +common_handler_sync_mode_qlen.label: +"""进入异步模式的队列长度""" + +common_handler_formatter.desc: +"""选择日志格式类型。 text 用于纯文本，json 用于结构化日志记录。""" + +common_handler_formatter.label: +"""日志格式类型""" + +rpc_async_batch_size.desc: +"""异步模式下，发送的批量消息的最大数量。""" + +rpc_async_batch_size.label: +"""异步模式下的批量消息数量""" + +cluster_call_max_history.desc: +"""集群间调用最多保留的历史记录数。只用于排错时查看。""" + +cluster_call_max_history.label: +"""最大历史记录""" + +cluster_discovery_strategy.desc: +"""集群节点发现方式。可选值为: +- manual: 使用 emqx ctl cluster 命令管理集群。
+- static: 配置静态节点。配置几个固定的节点，新节点通过连接固定节点中的某一个来加入集群。
+- dns: 使用 DNS A 记录的方式发现节点。
+- etcd: 使用 etcd 发现节点。
+- k8s: 使用 Kubernetes API 发现节点。""" + +cluster_discovery_strategy.label: +"""集群服务发现策略""" + +rpc_send_timeout.desc: +"""发送 RPC 请求的超时时间。""" + +rpc_send_timeout.label: +"""RPC 发送超时时间""" + +common_handler_single_line.desc: +"""如果设置为 true，则单行打印日志。 否则，日志消息可能跨越多行。""" + +common_handler_single_line.label: +"""单行模式""" + +rpc_socket_buffer.desc: +"""TCP 调节参数。用户模式套接字缓冲区大小。""" + +rpc_socket_buffer.label: +"""RPC 套接字缓冲区大小""" + +db_shard_transports.desc: +"""允许为每个 shard 下的事务日志复制操作的传输方法进行调优。
+gen_rpc 使用 gen_rpc 库， +distr 使用 Erlang 自带的 rpc 库。
如果未指定， +默认是使用 db.default_shard_transport 中设置的值。""" + +db_shard_transports.label: +"""事务日志传输协议""" + +} diff --git a/rel/i18n/zh/emqx_connector_api.hocon b/rel/i18n/zh/emqx_connector_api.hocon new file mode 100644 index 000000000..ab651c102 --- /dev/null +++ b/rel/i18n/zh/emqx_connector_api.hocon @@ -0,0 +1,46 @@ +emqx_connector_api { + +conn_get.desc: +"""列出所有连接器""" + +conn_get.label: +"""列出所有连接器""" + +conn_id_delete.desc: +"""通过 ID 删除一个连接器""" + +conn_id_delete.label: +"""删除连接器""" + +conn_id_get.desc: +"""通过 ID 获取连接器""" + +conn_id_get.label: +"""获取连接器""" + +conn_id_put.desc: +"""通过 ID 更新一个连接器""" + +conn_id_put.label: +"""更新连接器""" + +conn_post.desc: +"""创建一个新的连接器""" + +conn_post.label: +"""创建连接器""" + +conn_test_post.desc: +"""通过给定的 ID 测试创建一个新的连接器
+ID 的格式必须为“{type}:{name}”""" + +conn_test_post.label: +"""创建测试连接器""" + +id.desc: +"""连接器 ID， 格式必须为 {type}:{name}""" + +id.label: +"""连接器 ID""" + +} diff --git a/rel/i18n/zh/emqx_connector_http.hocon b/rel/i18n/zh/emqx_connector_http.hocon new file mode 100644 index 000000000..5d6398b2e --- /dev/null +++ b/rel/i18n/zh/emqx_connector_http.hocon @@ -0,0 +1,77 @@ +emqx_connector_http { + +base_url.desc: +"""base URL 只包含host和port。
+发送HTTP请求时，真实的URL是由base URL 和 path parameter连接而成。
+示例：`http://localhost:9901/`""" + +base_url.label: +"""Base Url""" + +body.desc: +"""HTTP请求报文主体。""" + +body.label: +"""HTTP请求报文主体""" + +connect_timeout.desc: +"""连接HTTP服务器的超时时间。""" + +connect_timeout.label: +"""连接超时""" + +enable_pipelining.desc: +"""正整数，设置最大可发送的异步 HTTP 请求数量。当设置为 1 时，表示每次发送完成 HTTP 请求后都需要等待服务器返回，再继续发送下一个请求。""" + +enable_pipelining.label: +"""HTTP 管道""" + +headers.desc: +"""HTTP 头字段列表。""" + +headers.label: +"""HTTP 头字段列表""" + +max_retries.desc: +"""请求出错时的最大重试次数。""" + +max_retries.label: +"""最大重试次数""" + +method.desc: +"""HTTP 请求方法。""" + +method.label: +"""HTTP 请求方法""" + +path.desc: +"""HTTP请求路径。""" + +path.label: +"""HTTP请求路径""" + +pool_size.desc: +"""连接池大小。""" + +pool_size.label: +"""连接池大小""" + +pool_type.desc: +"""连接池的类型，可用类型有`random`, `hash`。""" + +pool_type.label: +"""连接池类型""" + +request.desc: +"""设置 HTTP 请求的参数。""" + +request.label: +"""HTTP 请求""" + +request_timeout.desc: +"""HTTP 请求超时。""" + +request_timeout.label: +"""HTTP 请求超时""" + +} diff --git a/rel/i18n/zh/emqx_connector_ldap.hocon b/rel/i18n/zh/emqx_connector_ldap.hocon new file mode 100644 index 000000000..6826af6e6 --- /dev/null +++ b/rel/i18n/zh/emqx_connector_ldap.hocon @@ -0,0 +1,21 @@ +emqx_connector_ldap { + +bind_dn.desc: +"""LDAP 绑定的 DN 的值""" + +bind_dn.label: +"""Bind DN""" + +port.desc: +"""LDAP 端口""" + +port.label: +"""端口""" + +timeout.desc: +"""LDAP 查询超时时间""" + +timeout.label: +"""超时时间""" + +} diff --git a/rel/i18n/zh/emqx_connector_mongo.hocon b/rel/i18n/zh/emqx_connector_mongo.hocon new file mode 100644 index 000000000..cd84a242e --- /dev/null +++ b/rel/i18n/zh/emqx_connector_mongo.hocon @@ -0,0 +1,152 @@ +emqx_connector_mongo { + +auth_source.desc: +"""与用户证书关联的数据库名称。""" + +auth_source.label: +"""认证源""" + +connect_timeout.desc: +"""超时重连的等待时间。""" + +connect_timeout.label: +"""连接超时""" + +desc_rs.desc: +"""配置 Replica Set""" + +desc_rs.label: +"""配置 Replica Set""" + +desc_sharded.desc: +"""配置 Sharded Cluster""" + +desc_sharded.label: +"""配置 Sharded Cluster""" + +desc_single.desc: +"""配置 Single 模式""" + +desc_single.label: +"""配置 Single 模式""" + +desc_topology.desc: +"""配置 Topology""" + +desc_topology.label: +"""配置 Topology""" + +heartbeat_period.desc: +"""控制驱动程序何时检查MongoDB部署的状态。指定检查的间隔时间，从上一次检查结束到下一次检查开始计算。如果连接数增加（例如，如果你增加池子的大小，就会发生这种情况），你可能也需要增加这个周期，以避免在MongoDB日志文件中创建太多的日志条目。""" + +heartbeat_period.label: +"""心跳期""" + +local_threshold.desc: +"""在多个合适的MongoDB实例中进行选择的延迟窗口的大小。""" + +local_threshold.label: +"""本地阈值""" + +max_overflow.desc: +"""最大溢出。""" + +max_overflow.label: +"""最大溢出""" + +min_heartbeat_period.desc: +"""心跳间的最小间隙""" + +min_heartbeat_period.label: +"""最小心跳周期""" + +overflow_check_period.desc: +"""检查是否有超过配置的工人的周期（"溢出"）。""" + +overflow_check_period.label: +"""溢出检查周期""" + +overflow_ttl.desc: +"""当池内工人太多时，等待多久清除多余工人。""" + +overflow_ttl.label: +"""溢出TTL""" + +r_mode.desc: +"""读模式。""" + +r_mode.label: +"""读模式""" + +replica_set_name.desc: +"""副本集的名称。""" + +replica_set_name.label: +"""副本集名称""" + +rs_mongo_type.desc: +"""Replica set模式。当 MongoDB 服务运行在 replica-set 模式下，该配置必须设置为 'rs'。""" + +rs_mongo_type.label: +"""Replica set 模式""" + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 MongoDB 默认端口 27017。""" + +server.label: +"""服务器地址""" + +server_selection_timeout.desc: +"""指定在抛出异常之前为服务器选择阻断多长时间。""" + +server_selection_timeout.label: +"""服务器选择超时""" + +servers.desc: +"""集群将要连接的节点列表。 节点之间用逗号分隔，如：`Node[,Node].` +每个节点的配置为：将要连接的 IPv4 或 IPv6 地址或主机名。 +主机名具有以下形式：`Host[:Port]`。 +如果未指定 `[:Port]`，则使用 MongoDB 默认端口 27017。""" + +servers.label: +"""服务器列表""" + +sharded_mongo_type.desc: +"""Sharded cluster模式。当 MongoDB 服务运行在 sharded 模式下，该配置必须设置为 'sharded'。""" + +sharded_mongo_type.label: +"""Sharded cluster 模式""" + +single_mongo_type.desc: +"""Standalone 模式。当 MongoDB 服务运行在 standalone 模式下，该配置必须设置为 'single'。""" + +single_mongo_type.label: +"""Standalone 模式""" + +socket_timeout.desc: +"""在尝试超时之前，在套接字上尝试发送或接收的持续时间。""" + +socket_timeout.label: +"""套接字操作超时""" + +srv_record.desc: +"""使用 DNS SRV 记录。""" + +srv_record.label: +"""SRV 记录""" + +w_mode.desc: +"""写模式。""" + +w_mode.label: +"""写模式""" + +wait_queue_timeout.desc: +"""工作者等待连接可用的最长时间。""" + +wait_queue_timeout.label: +"""等待队列超时""" + +} diff --git a/rel/i18n/zh/emqx_connector_mqtt.hocon b/rel/i18n/zh/emqx_connector_mqtt.hocon new file mode 100644 index 000000000..1df6a89b6 --- /dev/null +++ b/rel/i18n/zh/emqx_connector_mqtt.hocon @@ -0,0 +1,21 @@ +emqx_connector_mqtt { + +name.desc: +"""连接器名称，人类可读的连接器描述。""" + +name.label: +"""连接器名称""" + +num_of_bridges.desc: +"""当前使用此连接器的网桥数量。""" + +num_of_bridges.label: +"""网桥数量""" + +type.desc: +"""连接器类型。""" + +type.label: +"""连接器类型""" + +} diff --git a/rel/i18n/zh/emqx_connector_mqtt_schema.hocon b/rel/i18n/zh/emqx_connector_mqtt_schema.hocon new file mode 100644 index 000000000..66ba2accf --- /dev/null +++ b/rel/i18n/zh/emqx_connector_mqtt_schema.hocon @@ -0,0 +1,170 @@ +emqx_connector_mqtt_schema { + +bridge_mode.desc: +"""是否启用 Bridge Mode。 +注意：此设置只针对 MQTT 协议版本 < 5.0 有效，并且需要远程 MQTT Broker 支持 Bridge Mode。 +如果设置为 true ，桥接会告诉远端服务器当前连接是一个桥接而不是一个普通的客户端。 +这意味着消息回环检测会更加高效，并且远端服务器收到的保留消息的标志位会透传给本地。""" + +bridge_mode.label: +"""Bridge 模式""" + +clean_start.desc: +"""与 ingress MQTT 桥的远程服务器重连时是否清除老的 MQTT 会话。""" + +clean_start.label: +"""清除会话""" + +clientid_prefix.desc: +"""可选的前缀，用于在出口网桥使用的clientid前加上前缀。""" + +clientid_prefix.label: +"""客户ID前缀""" + +egress_desc.desc: +"""出口配置定义了该桥接如何将消息从本地 Broker 转发到远程 Broker。 +以下字段中允许使用带有变量的模板：'remote.topic', 'local.qos', 'local.retain', 'local.payload'。
+注意：如果此桥接被用作规则的动作，并且配置了 'local.topic'，则从规则输出的数据以及匹配到 'local.topic' 的 MQTT 消息都会被转发。""" + +egress_desc.label: +"""出方向配置""" + +egress_local.desc: +"""如何从本地 Broker 接收消息相关的配置。""" + +egress_local.label: +"""本地配置""" + +egress_local_topic.desc: +"""要转发到远程broker的本地主题""" + +egress_local_topic.label: +"""本地主题""" + +egress_remote.desc: +"""发送消息到远程 Broker 相关的配置。""" + +egress_remote.label: +"""远程配置""" + +egress_remote_qos.desc: +"""待发送 MQTT 消息的 QoS。
+允许使用带有变量的模板。""" + +egress_remote_qos.label: +"""远程 QoS""" + +egress_remote_topic.desc: +"""转发到远程broker的哪个topic。
+允许使用带有变量的模板。""" + +egress_remote_topic.label: +"""远程主题""" + +ingress_desc.desc: +"""入口配置定义了该桥接如何从远程 MQTT Broker 接收消息，然后将消息发送到本地 Broker。
+ 以下字段中允许使用带有变量的模板：'remote.qos', 'local.topic', 'local.qos', 'local.retain', 'local.payload'。
+ 注意：如果此桥接被用作规则的输入，并且配置了 'local.topic'，则从远程代理获取的消息将同时被发送到 'local.topic' 和规则。""" + +ingress_desc.label: +"""入方向配置""" + +ingress_local.desc: +"""发送消息到本地 Broker 相关的配置。""" + +ingress_local.label: +"""本地配置""" + +ingress_local_qos.desc: +"""待发送 MQTT 消息的 QoS。
+允许使用带有变量的模板。""" + +ingress_local_qos.label: +"""本地 QoS""" + +ingress_local_topic.desc: +"""向本地broker的哪个topic发送消息。
+允许使用带有变量的模板。""" + +ingress_local_topic.label: +"""本地主题""" + +ingress_remote.desc: +"""订阅远程 Broker 相关的配置。""" + +ingress_remote.label: +"""远程配置""" + +ingress_remote_qos.desc: +"""订阅远程borker时要使用的 QoS 级别""" + +ingress_remote_qos.label: +"""远程 QoS""" + +ingress_remote_topic.desc: +"""从远程broker的哪个topic接收消息""" + +ingress_remote_topic.label: +"""远程主题""" + +max_inflight.desc: +"""MQTT 协议的最大飞行（已发送但未确认）消息""" + +max_inflight.label: +"""最大飞行消息""" + +mode.desc: +"""MQTT 桥的模式。
+- cluster_shareload：在 emqx 集群的每个节点上创建一个 MQTT 连接。
+在“cluster_shareload”模式下，来自远程代理的传入负载通过共享订阅的方式接收。
+请注意，clientid 以节点名称为后缀，这是为了避免不同节点之间的 clientid 冲突。 +而且对于入口连接的 remote.topic，我们只能使用共享订阅主题过滤器。""" + +mode.label: +"""MQTT 桥接模式""" + +password.desc: +"""MQTT 协议的密码""" + +password.label: +"""密码""" + +payload.desc: +"""要发送的 MQTT 消息的负载。
+允许使用带有变量的模板。""" + +payload.label: +"""消息负载""" + +proto_ver.desc: +"""MQTT 协议版本""" + +proto_ver.label: +"""协议版本""" + +retain.desc: +"""要发送的 MQTT 消息的“保留”标志。
+允许使用带有变量的模板。""" + +retain.label: +"""保留消息标志""" + +server.desc: +"""远程 MQTT Broker的主机和端口。""" + +server.label: +"""Broker主机和端口""" + +server_configs.desc: +"""服务器相关的配置。""" + +server_configs.label: +"""服务配置。""" + +username.desc: +"""MQTT 协议的用户名""" + +username.label: +"""用户名""" + +} diff --git a/rel/i18n/zh/emqx_connector_mysql.hocon b/rel/i18n/zh/emqx_connector_mysql.hocon new file mode 100644 index 000000000..a6900056f --- /dev/null +++ b/rel/i18n/zh/emqx_connector_mysql.hocon @@ -0,0 +1,11 @@ +emqx_connector_mysql { + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 MySQL 默认端口 3306。""" + +server.label: +"""服务器地址""" + +} diff --git a/rel/i18n/zh/emqx_connector_pgsql.hocon b/rel/i18n/zh/emqx_connector_pgsql.hocon new file mode 100644 index 000000000..c391034fd --- /dev/null +++ b/rel/i18n/zh/emqx_connector_pgsql.hocon @@ -0,0 +1,11 @@ +emqx_connector_pgsql { + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 PostgreSQL 默认端口 5432。""" + +server.label: +"""服务器地址""" + +} diff --git a/rel/i18n/zh/emqx_connector_redis.hocon b/rel/i18n/zh/emqx_connector_redis.hocon new file mode 100644 index 000000000..a0819876e --- /dev/null +++ b/rel/i18n/zh/emqx_connector_redis.hocon @@ -0,0 +1,50 @@ +emqx_connector_redis { + +cluster.desc: +"""集群模式。当 Redis 服务运行在集群模式下，该配置必须设置为 'cluster'。""" + +cluster.label: +"""集群模式""" + +database.desc: +"""Redis 数据库 ID。""" + +database.label: +"""数据库 ID""" + +sentinel.desc: +"""哨兵模式。当 Redis 服务运行在哨兵模式下，该配置必须设置为 'sentinel'。""" + +sentinel.label: +"""哨兵模式""" + +sentinel_desc.desc: +"""Redis 哨兵模式下的集群名称。""" + +sentinel_desc.label: +"""集群名称""" + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 Redis 默认端口 6379。""" + +server.label: +"""服务器地址""" + +servers.desc: +"""集群将要连接的节点列表。 节点之间用逗号分隔，如：`Node[,Node].` +每个节点的配置为：将要连接的 IPv4 或 IPv6 地址或主机名。 +主机名具有以下形式：`Host[:Port]`。 +如果未指定 `[:Port]`，则使用 Redis 默认端口 6379。""" + +servers.label: +"""服务器列表""" + +single.desc: +"""单机模式。当 Redis 服务运行在单机模式下，该配置必须设置为 'single'。""" + +single.label: +"""单机模式""" + +} diff --git a/rel/i18n/zh/emqx_connector_schema_lib.hocon b/rel/i18n/zh/emqx_connector_schema_lib.hocon new file mode 100644 index 000000000..318b5c31c --- /dev/null +++ b/rel/i18n/zh/emqx_connector_schema_lib.hocon @@ -0,0 +1,45 @@ +emqx_connector_schema_lib { + +auto_reconnect.desc: +"""已弃用。自动重连数据库。""" + +auto_reconnect.label: +"""已弃用。自动重连数据库""" + +database_desc.desc: +"""数据库名字。""" + +database_desc.label: +"""数据库名字""" + +password.desc: +"""内部数据库密码。""" + +password.label: +"""密码""" + +pool_size.desc: +"""桥接远端服务时使用的连接池大小。""" + +pool_size.label: +"""连接池大小""" + +prepare_statement.desc: +"""SQL 预处理语句列表。""" + +prepare_statement.label: +"""SQL 预处理语句列表""" + +ssl.desc: +"""启用 SSL 连接。""" + +ssl.label: +"""启用SSL""" + +username.desc: +"""内部数据库的用户名。""" + +username.label: +"""用户名""" + +} diff --git a/rel/i18n/zh/emqx_dashboard_api.hocon b/rel/i18n/zh/emqx_dashboard_api.hocon new file mode 100644 index 000000000..8292d8bba --- /dev/null +++ b/rel/i18n/zh/emqx_dashboard_api.hocon @@ -0,0 +1,66 @@ +emqx_dashboard_api { + +change_pwd_api.desc: +"""更改 Dashboard 用户密码""" + +create_user_api.desc: +"""创建 Dashboard 用户""" + +create_user_api_success.desc: +"""创建 Dashboard 用户成功""" + +delete_user_api.desc: +"""删除 Dashboard 用户""" + +license.desc: +"""EMQX 许可类型。可为 opensource 或 enterprise""" + +list_users_api.desc: +"""Dashboard 用户列表""" + +login_api.desc: +"""获取 Dashboard 认证 Token。""" + +login_failed401.desc: +"""登录失败。用户名或密码错误""" + +login_failed_response400.desc: +"""登录失败。用户名或密码错误""" + +login_success.desc: +"""Dashboard 认证成功""" + +logout_api.desc: +"""Dashboard 用户登出""" + +new_pwd.desc: +"""新密码""" + +old_pwd.desc: +"""旧密码""" + +password.desc: +"""Dashboard 密码""" + +token.desc: +"""Dashboard 认证 Token""" + +update_user_api.desc: +"""更新 Dashboard 用户描述""" + +update_user_api200.desc: +"""更新 Dashboard 用户成功""" + +user_description.desc: +"""Dashboard 用户描述""" + +username.desc: +"""Dashboard 用户名""" + +users_api404.desc: +"""Dashboard 用户不存在""" + +version.desc: +"""EMQX 版本""" + +} diff --git a/rel/i18n/zh/emqx_dashboard_schema.hocon b/rel/i18n/zh/emqx_dashboard_schema.hocon new file mode 100644 index 000000000..a07f3e6d8 --- /dev/null +++ b/rel/i18n/zh/emqx_dashboard_schema.hocon @@ -0,0 +1,130 @@ +emqx_dashboard_schema { + +backlog.desc: +"""排队等待连接的队列的最大长度。""" + +backlog.label: +"""排队长度""" + +bind.desc: +"""监听地址和端口，热更新此配置时，会重启 Dashboard 服务。""" + +bind.label: +"""绑定端口""" + +bootstrap_users_file.desc: +"""已废弃，请使用 api_key.bootstrap_file。""" + +bootstrap_users_file.label: +"""已废弃""" + +cors.desc: +"""支持跨域资源共享(CORS)， +允许服务器指示任何来源(域名、协议或端口)，除了本服务器之外的任何浏览器应允许加载资源。""" + +cors.label: +"""跨域资源共享""" + +default_password.desc: +"""Dashboard 的默认密码，为了安全，应该尽快修改密码。 +当通过网页首次登录 Dashboard 并按提示修改成复杂密码后，此值就会失效。""" + +default_password.label: +"""默认密码""" + +default_username.desc: +"""Dashboard 的默认用户名。""" + +default_username.label: +"""默认用户名""" + +desc_dashboard.desc: +"""EMQX Dashboard 配置。""" + +desc_dashboard.label: +"""Dashboard""" + +desc_http.desc: +"""Dashboard 监听器(HTTP)配置。""" + +desc_http.label: +"""HTTP""" + +desc_https.desc: +"""Dashboard 监听器(HTTPS)配置。""" + +desc_https.label: +"""HTTPS""" + +desc_listeners.desc: +"""Dashboard 监听器配置。""" + +desc_listeners.label: +"""监听器""" + +i18n_lang.desc: +"""设置 Swagger 多语言的版本，可为 en 或 zh。""" + +i18n_lang.label: +"""多语言支持""" + +inet6.desc: +"""启用IPv6， 如果机器不支持IPv6，请关闭此选项，否则会导致 Dashboard 无法使用。""" + +inet6.label: +"""IPv6""" + +ipv6_v6only.desc: +"""当开启 inet6 功能的同时禁用 IPv4-to-IPv6 映射。该配置仅在 inet6 功能开启时有效。""" + +ipv6_v6only.label: +"""IPv6 only""" + +listener_enable.desc: +"""忽略或启用该监听器。""" + +listener_enable.label: +"""启用""" + +listeners.desc: +"""Dashboard 监听器设置。监听器必须有唯一的端口号和IP地址的组合。 +例如，可以通过指定IP地址 0.0.0.0 来监听机器上给定端口上的所有配置的IP地址。 +或者，可以为每个监听器指定唯一的IP地址，但使用相同的端口。""" + +listeners.label: +"""监听器""" + +max_connections.desc: +"""同时处理的最大连接数。""" + +max_connections.label: +"""最大连接数""" + +num_acceptors.desc: +"""TCP协议的Socket acceptor池大小, 默认设置在线的调度器数量（通常为 CPU 核数）""" + +num_acceptors.label: +"""Acceptor 数量""" + +proxy_header.desc: +"""开启对 `HAProxy` 的支持，注意：一旦开启了这个功能，就无法再处理普通的 HTTP 请求了。""" + +proxy_header.label: +"""开启对 `HAProxy` 的支持""" + +sample_interval.desc: +"""Dashboard 中图表指标的时间间隔。必须小于60，且被60的整除，默认设置 10s。""" + +send_timeout.desc: +"""Socket发送超时时间。""" + +send_timeout.label: +"""发送超时时间""" + +token_expired_time.desc: +"""JWT token 过期时间。默认设置为 60 分钟。""" + +token_expired_time.label: +"""JWT 过期时间""" + +} diff --git a/rel/i18n/zh/emqx_delayed_api.hocon b/rel/i18n/zh/emqx_delayed_api.hocon new file mode 100644 index 000000000..d783a17c3 --- /dev/null +++ b/rel/i18n/zh/emqx_delayed_api.hocon @@ -0,0 +1,72 @@ +emqx_delayed_api { + +bad_msgid_format.desc: +"""消息 ID 格式错误""" + +count.desc: +"""延迟消息总数""" + +delayed_interval.desc: +"""延迟时间(秒)""" + +delayed_remaining.desc: +"""剩余时间(秒)""" + +delete_api.desc: +"""删除延迟消息""" + +expected_at.desc: +"""期望的发送时间, RFC 3339 格式""" + +from_clientid.desc: +"""消息的 ClientID""" + +from_username.desc: +"""消息的 Username""" + +get_message_api.desc: +"""查看延迟消息""" + +illegality_limit.desc: +"""数量上限不合法""" + +list_api.desc: +"""查看延迟消息列表""" + +msgid.desc: +"""延迟消息 ID""" + +msgid_not_found.desc: +"""未找到对应消息""" + +node.desc: +"""消息的来源节点""" + +payload.desc: +"""消息内容, base64 格式。如果消息的大小超过 2048 字节，则消息内容会被设置为 'PAYLOAD_TO_LARGE'""" + +publish_at.desc: +"""客户端发送时间, RFC 3339 格式""" + +qos.desc: +"""QoS""" + +topic.desc: +"""主题""" + +update_api.desc: +"""开启或者关闭功能，或者设置延迟消息数量上限""" + +update_success.desc: +"""开启或者关闭功能操作成功""" + +view_limit.desc: +"""每页数量""" + +view_page.desc: +"""查看的页数""" + +view_status_api.desc: +"""查看慢订阅状态""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_cassa.hocon b/rel/i18n/zh/emqx_ee_bridge_cassa.hocon new file mode 100644 index 000000000..2d1125a6b --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_cassa.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_cassa { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +cql_template.desc: +"""CQL 模板""" + +cql_template.label: +"""CQL 模板""" + +desc_config.desc: +"""Cassandra 桥接配置""" + +desc_config.label: +"""Cassandra 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 Cassandra。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon b/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon new file mode 100644 index 000000000..a3ede08ba --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon @@ -0,0 +1,46 @@ +emqx_ee_bridge_clickhouse { + +batch_value_separator.desc: +"""默认为逗号 ','，适用于 VALUE 格式。您也可以使用其他分隔符， 请参考 [INSERT INTO 语句](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)。""" + +batch_value_separator.label: +"""分隔符""" + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""Clickhouse 桥接配置""" + +desc_config.label: +"""Clickhouse 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 Clickhouse。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +sql_template.desc: +"""可以使用 ${field} 占位符来引用消息与客户端上下文中的变量，请确保对应字段存在且数据格式符合预期。此处不支持 [SQL 预处理](https://docs.emqx.com/zh/enterprise/v5.0/data-integration/data-bridges.html#sql-预处理)。""" + +sql_template.label: +"""SQL 模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_dynamo.hocon b/rel/i18n/zh/emqx_ee_bridge_dynamo.hocon new file mode 100644 index 000000000..adf33b9e8 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_dynamo.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_dynamo { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""DynamoDB 桥接配置""" + +desc_config.label: +"""DynamoDB 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 DynamoDB。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +template.desc: +"""模板, 默认为空，为空时将会将整个消息存入数据库""" + +template.label: +"""模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_gcp_pubsub.hocon b/rel/i18n/zh/emqx_ee_bridge_gcp_pubsub.hocon new file mode 100644 index 000000000..4318211c9 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_gcp_pubsub.hocon @@ -0,0 +1,77 @@ +emqx_ee_bridge_gcp_pubsub { + +connect_timeout.desc: +"""连接 HTTP 服务器的超时时间。""" + +connect_timeout.label: +"""连接超时""" + +desc_config.desc: +"""GCP PubSub 桥接配置""" + +desc_config.label: +"""GCP PubSub 桥接配置""" + +desc_name.desc: +"""桥接名字，可读描述""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""桥接类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 GCP PubSub。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 GCP PubSub。""" + +local_topic.label: +"""本地 Topic""" + +max_retries.desc: +"""请求出错时的最大重试次数。""" + +max_retries.label: +"""最大重试次数""" + +payload_template.desc: +"""用于格式化外发信息的模板。 如果未定义，将以JSON格式发送所有可用的上下文。""" + +payload_template.label: +"""HTTP 请求消息体模板""" + +pipelining.desc: +"""正整数，设置最大可发送的异步 HTTP 请求数量。当设置为 1 时，表示每次发送完成 HTTP 请求后都需要等待服务器返回，再继续发送下一个请求。""" + +pipelining.label: +"""HTTP 流水线""" + +pool_size.desc: +"""连接池大小。""" + +pool_size.label: +"""连接池大小""" + +pubsub_topic.desc: +"""要发布消息的GCP PubSub主题。""" + +pubsub_topic.label: +"""GCP PubSub 主题""" + +request_timeout.desc: +"""废弃的。在缓冲区设置中配置请求超时。""" + +request_timeout.label: +"""HTTP 请求超时""" + +service_account_json.desc: +"""包含将与 PubSub 一起使用的 GCP 服务账户凭证的 JSON。 +当创建GCP服务账户时（如https://developers.google.com/identity/protocols/oauth2/service-account#creatinganaccount），可以选择下载 JSON 形式的凭证，然后在该配置项中使用。""" + +service_account_json.label: +"""GCP 服务账户凭证""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_hstreamdb.hocon b/rel/i18n/zh/emqx_ee_bridge_hstreamdb.hocon new file mode 100644 index 000000000..55cdebe3c --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_hstreamdb.hocon @@ -0,0 +1,52 @@ +emqx_ee_bridge_hstreamdb { + +config_direction.desc: +"""桥接的方向， 必须是 egress""" + +config_direction.label: +"""桥接方向""" + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""HStreamDB 桥接配置""" + +desc_config.label: +"""HStreamDB 桥接配置""" + +desc_connector.desc: +"""连接器的通用配置。""" + +desc_connector.label: +"""连接器通用配置。""" + +desc_name.desc: +"""桥接名字，可读描述""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 HStreamDB。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 HStreamDB。""" + +local_topic.label: +"""本地 Topic""" + +payload.desc: +"""要转发到 HStreamDB 的数据内容，支持占位符""" + +payload.label: +"""消息内容""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_influxdb.hocon b/rel/i18n/zh/emqx_ee_bridge_influxdb.hocon new file mode 100644 index 000000000..c9c7c6a54 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_influxdb.hocon @@ -0,0 +1,47 @@ +emqx_ee_bridge_influxdb { + +config_enable.desc: +"""启用/禁用桥接。""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""InfluxDB 桥接配置。""" + +desc_config.label: +"""InfluxDB 桥接配置""" + +desc_name.desc: +"""桥接名称。""" + +desc_name.label: +"""桥接名称""" + +desc_type.desc: +"""桥接类型。""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 InfluxDB。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 InfluxDB。""" + +local_topic.label: +"""本地 Topic""" + +write_syntax.desc: +"""使用 InfluxDB API Line Protocol 写入 InfluxDB 的数据，支持占位符
+参考 [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) 及 +[InfluxDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
+TLDR:
+``` +[,=[,=]] =[,=] [] +``` +注意，整形数值占位符后需要添加一个字符 `i` 类型标识。例如 `${payload.int_value}i`""" + +write_syntax.label: +"""写语句""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_mongodb.hocon b/rel/i18n/zh/emqx_ee_bridge_mongodb.hocon new file mode 100644 index 000000000..71703c662 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_mongodb.hocon @@ -0,0 +1,57 @@ +emqx_ee_bridge_mongodb { + +collection.desc: +"""数据将被存储到的集合""" + +collection.label: +"""将要使用的集合（Collection）""" + +desc_config.desc: +"""为MongoDB桥配置""" + +desc_config.label: +"""MongoDB桥配置""" + +desc_name.desc: +"""桥接名称。""" + +desc_name.label: +"""桥接名称""" + +desc_type.desc: +"""桥接类型。""" + +desc_type.label: +"""桥接类型""" + +enable.desc: +"""启用或停用该MongoDB桥""" + +enable.label: +"""启用或禁用""" + +mongodb_rs_conf.desc: +"""MongoDB（Replica Set）配置""" + +mongodb_rs_conf.label: +"""MongoDB（Replica Set）配置""" + +mongodb_sharded_conf.desc: +"""MongoDB (Sharded)配置""" + +mongodb_sharded_conf.label: +"""MongoDB (Sharded)配置""" + +mongodb_single_conf.desc: +"""MongoDB（独立）配置""" + +mongodb_single_conf.label: +"""MongoDB（独立）配置""" + +payload_template.desc: +"""用于格式化写入 MongoDB 的消息模板。 如果未定义，规则引擎会使用 JSON 格式序列化所有的可见输入，例如 clientid, topic, payload 等。""" + +payload_template.label: +"""有效载荷模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_mysql.hocon b/rel/i18n/zh/emqx_ee_bridge_mysql.hocon new file mode 100644 index 000000000..e03b147b0 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_mysql.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_mysql { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""HStreamDB 桥接配置""" + +desc_config.label: +"""HStreamDB 桥接配置""" + +desc_name.desc: +"""桥接名字，可读描述""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 MySQL。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +sql_template.desc: +"""SQL 模板""" + +sql_template.label: +"""SQL 模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_pgsql.hocon b/rel/i18n/zh/emqx_ee_bridge_pgsql.hocon new file mode 100644 index 000000000..ebf7f331a --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_pgsql.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_pgsql { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""PostgreSQL 桥接配置""" + +desc_config.label: +"""PostgreSQL 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 PostgreSQL。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +sql_template.desc: +"""SQL 模板""" + +sql_template.label: +"""SQL 模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_redis.hocon b/rel/i18n/zh/emqx_ee_bridge_redis.hocon new file mode 100644 index 000000000..378df508f --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_redis.hocon @@ -0,0 +1,41 @@ +emqx_ee_bridge_redis { + +command_template.desc: +"""用于推送数据的 Redis 命令模板。 每个列表元素代表一个命令名称或其参数。 +例如，要通过键值 `msgs` 将消息体推送到 Redis 列表中，数组元素应该是： `rpush`, `msgs`, `${payload}`。""" + +command_template.label: +"""Redis Command 模板""" + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""Resis 桥接配置""" + +desc_config.label: +"""Redis 桥接配置""" + +desc_name.desc: +"""桥接名字，可读描述""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 Redis。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发到 Redis。""" + +local_topic.label: +"""本地 Topic""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon b/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon new file mode 100644 index 000000000..924004361 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_rocketmq.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_rocketmq { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""RocketMQ 桥接配置""" + +desc_config.label: +"""RocketMQ 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 RocketMQ。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +template.desc: +"""模板, 默认为空，为空时将会将整个消息转发给 RocketMQ""" + +template.label: +"""模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_sqlserver.hocon b/rel/i18n/zh/emqx_ee_bridge_sqlserver.hocon new file mode 100644 index 000000000..0958d4b7a --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_sqlserver.hocon @@ -0,0 +1,46 @@ +emqx_ee_bridge_sqlserver { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""Microsoft SQL Server 桥接配置""" + +desc_config.label: +"""Microsoft SQL Server 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +driver.desc: +"""SQL Server Driver 名称""" + +driver.label: +"""SQL Server Driver 名称""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 Microsoft SQL Server。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +sql_template.desc: +"""SQL 模板""" + +sql_template.label: +"""SQL 模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_bridge_tdengine.hocon b/rel/i18n/zh/emqx_ee_bridge_tdengine.hocon new file mode 100644 index 000000000..5e417a1c7 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_bridge_tdengine.hocon @@ -0,0 +1,40 @@ +emqx_ee_bridge_tdengine { + +config_enable.desc: +"""启用/禁用桥接""" + +config_enable.label: +"""启用/禁用桥接""" + +desc_config.desc: +"""TDengine 桥接配置""" + +desc_config.label: +"""TDengine 桥接配置""" + +desc_name.desc: +"""桥接名字""" + +desc_name.label: +"""桥接名字""" + +desc_type.desc: +"""Bridge 类型""" + +desc_type.label: +"""桥接类型""" + +local_topic.desc: +"""发送到 'local_topic' 的消息都会转发到 TDengine。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。""" + +local_topic.label: +"""本地 Topic""" + +sql_template.desc: +"""SQL 模板""" + +sql_template.label: +"""SQL 模板""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_cassa.hocon b/rel/i18n/zh/emqx_ee_connector_cassa.hocon new file mode 100644 index 000000000..ffbadc7de --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_cassa.hocon @@ -0,0 +1,17 @@ +emqx_ee_connector_cassa { + +keyspace.desc: +"""要连接到的 Keyspace 名称。""" + +keyspace.label: +"""Keyspace""" + +servers.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port][,Host2:Port]`。
+如果未指定 `[:Port]`，则使用 Cassandra 默认端口 9042。""" + +servers.label: +"""Servers""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon b/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon new file mode 100644 index 000000000..f1457a1f6 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_clickhouse.hocon @@ -0,0 +1,15 @@ +emqx_ee_connector_clickhouse { + +base_url.desc: +"""你想连接到的Clickhouse服务器的HTTP URL（例如http://myhostname:8123）。""" + +base_url.label: +"""服务器 URL""" + +connect_timeout.desc: +"""连接HTTP服务器的超时时间。""" + +connect_timeout.label: +"""连接超时""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_dynamo.hocon b/rel/i18n/zh/emqx_ee_connector_dynamo.hocon new file mode 100644 index 000000000..540d79dd0 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_dynamo.hocon @@ -0,0 +1,9 @@ +emqx_ee_connector_dynamo { + +url.desc: +"""DynamoDB 的地址。""" + +url.label: +"""DynamoDB 地址""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_hstreamdb.hocon b/rel/i18n/zh/emqx_ee_connector_hstreamdb.hocon new file mode 100644 index 000000000..d9d8fb3ed --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_hstreamdb.hocon @@ -0,0 +1,45 @@ +emqx_ee_connector_hstreamdb { + +config.desc: +"""HStreamDB 连接配置。""" + +config.label: +"""连接配置""" + +name.desc: +"""连接器名称，人类可读的连接器描述。""" + +name.label: +"""连接器名称""" + +ordering_key.desc: +"""HStreamDB 分区键""" + +ordering_key.label: +"""HStreamDB 分区键""" + +pool_size.desc: +"""HStreamDB 连接池大小""" + +pool_size.label: +"""HStreamDB 连接池大小""" + +stream_name.desc: +"""HStreamDB 流名称""" + +stream_name.label: +"""HStreamDB 流名称""" + +type.desc: +"""连接器类型。""" + +type.label: +"""连接器类型""" + +url.desc: +"""HStreamDB 服务器 URL""" + +url.label: +"""HStreamDB 服务器 URL""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_influxdb.hocon b/rel/i18n/zh/emqx_ee_connector_influxdb.hocon new file mode 100644 index 000000000..6148b400a --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_influxdb.hocon @@ -0,0 +1,71 @@ +emqx_ee_connector_influxdb { + +bucket.desc: +"""InfluxDB bucket 名称。""" + +bucket.label: +"""Bucket""" + +database.desc: +"""InfluxDB 数据库。""" + +database.label: +"""数据库""" + +influxdb_api_v1.desc: +"""InfluxDB HTTP API 协议。支持 Influxdb v1.8 以及之前的版本。""" + +influxdb_api_v1.label: +"""HTTP API 协议""" + +influxdb_api_v2.desc: +"""InfluxDB HTTP API V2 协议。支持 Influxdb v2.0 以及之后的版本。""" + +influxdb_api_v2.label: +"""HTTP API V2 协议""" + +org.desc: +"""InfluxDB 组织名称。""" + +org.label: +"""组织""" + +password.desc: +"""InfluxDB 密码。""" + +password.label: +"""密码""" + +precision.desc: +"""InfluxDB 时间精度。""" + +precision.label: +"""时间精度""" + +protocol.desc: +"""InfluxDB 协议。HTTP API 或 HTTP API V2。""" + +protocol.label: +"""协议""" + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 InfluxDB 默认端口 8086。""" + +server.label: +"""服务器地址""" + +token.desc: +"""InfluxDB token。""" + +token.label: +"""Token""" + +username.desc: +"""InfluxDB 用户名。""" + +username.label: +"""用户名""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon b/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon new file mode 100644 index 000000000..d32e6ea01 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_rocketmq.hocon @@ -0,0 +1,35 @@ +emqx_ee_connector_rocketmq { + +refresh_interval.desc: +"""RocketMQ 主题路由更新间隔。""" + +refresh_interval.label: +"""主题路由更新间隔""" + +security_token.desc: +"""RocketMQ 服务器安全令牌""" + +security_token.label: +"""安全令牌""" + +send_buffer.desc: +"""RocketMQ 驱动的套字节发送消息的缓冲区大小""" + +send_buffer.label: +"""发送消息的缓冲区大小""" + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 RocketMQ 默认端口 9876。""" + +server.label: +"""服务器地址""" + +topic.desc: +"""RocketMQ 主题""" + +topic.label: +"""RocketMQ 主题""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_sqlserver.hocon b/rel/i18n/zh/emqx_ee_connector_sqlserver.hocon new file mode 100644 index 000000000..44377c86d --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_sqlserver.hocon @@ -0,0 +1,11 @@ +emqx_ee_connector_sqlserver { + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 SQL Server 默认端口 1433。""" + +server.label: +"""服务器地址""" + +} diff --git a/rel/i18n/zh/emqx_ee_connector_tdengine.hocon b/rel/i18n/zh/emqx_ee_connector_tdengine.hocon new file mode 100644 index 000000000..f3064aeb5 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_connector_tdengine.hocon @@ -0,0 +1,11 @@ +emqx_ee_connector_tdengine { + +server.desc: +"""将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 TDengine 默认端口 6041。""" + +server.label: +"""服务器地址""" + +} diff --git a/rel/i18n/zh/emqx_ee_schema_registry_http_api.hocon b/rel/i18n/zh/emqx_ee_schema_registry_http_api.hocon new file mode 100644 index 000000000..1dab50a3f --- /dev/null +++ b/rel/i18n/zh/emqx_ee_schema_registry_http_api.hocon @@ -0,0 +1,39 @@ +emqx_ee_schema_registry_http_api { + +desc_param_path_schema_name.desc: +"""模式名称""" + +desc_param_path_schema_name.label: +"""模式名称""" + +desc_schema_registry_api_delete.desc: +"""删除一个模式""" + +desc_schema_registry_api_delete.label: +"""删除模式""" + +desc_schema_registry_api_get.desc: +"""通过名称获取模式""" + +desc_schema_registry_api_get.label: +"""获取模式""" + +desc_schema_registry_api_list.desc: +"""列出所有注册的模式""" + +desc_schema_registry_api_list.label: +"""列表模式""" + +desc_schema_registry_api_post.desc: +"""注册一个新的模式""" + +desc_schema_registry_api_post.label: +"""注册模式""" + +desc_schema_registry_api_put.desc: +"""更新一个现有的模式""" + +desc_schema_registry_api_put.label: +"""更新模式""" + +} diff --git a/rel/i18n/zh/emqx_ee_schema_registry_schema.hocon b/rel/i18n/zh/emqx_ee_schema_registry_schema.hocon new file mode 100644 index 000000000..3bf0a7dc8 --- /dev/null +++ b/rel/i18n/zh/emqx_ee_schema_registry_schema.hocon @@ -0,0 +1,45 @@ +emqx_ee_schema_registry_schema { + +avro_type.desc: +"""[阿帕奇-阿夫罗](https://avro.apache.org/) 序列化格式。""" + +avro_type.label: +"""阿帕奇-阿夫罗""" + +schema_description.desc: +"""对该模式的描述。""" + +schema_description.label: +"""模式描述""" + +schema_name.desc: +"""模式的一个名称，将作为其标识符。""" + +schema_name.label: +"""模式名称""" + +schema_registry_root.desc: +"""模式注册表的配置。""" + +schema_registry_root.label: +"""模式注册表""" + +schema_registry_schemas.desc: +"""注册的模式。""" + +schema_registry_schemas.label: +"""注册的模式""" + +schema_source.desc: +"""模式的源文本。""" + +schema_source.label: +"""模式来源""" + +schema_type.desc: +"""模式类型。""" + +schema_type.label: +"""模式类型""" + +} diff --git a/rel/i18n/zh/emqx_exhook_api.hocon b/rel/i18n/zh/emqx_exhook_api.hocon new file mode 100644 index 000000000..aba0e3e58 --- /dev/null +++ b/rel/i18n/zh/emqx_exhook_api.hocon @@ -0,0 +1,81 @@ +emqx_exhook_api { + +add_server.desc: +"""添加 ExHook 服务器""" + +delete_server.desc: +"""删除 Exhook 服务器""" + +get_detail.desc: +"""查看 Exhook 服务器详细信息""" + +get_hooks.desc: +"""获取 Exhook 服务器的钩子信息""" + +hook_metrics.desc: +"""当前节点中该钩子的指标信息""" + +hook_name.desc: +"""钩子的名称""" + +hook_params.desc: +"""钩子注册时使用的参数""" + +list_all_servers.desc: +"""查看ExHook 服务器列表""" + +metric_failed.desc: +"""钩子执行失败的次数""" + +metric_max_rate.desc: +"""钩子的最大调用速率""" + +metric_rate.desc: +"""钩子的调用速率""" + +metric_succeed.desc: +"""钩子执行成功的次数""" + +metrics.desc: +"""指标信息""" + +move_api.desc: +"""移动 Exhook 服务器顺序。 +注意: 移动的参数只能是：front | rear | before:{name} | after:{name}""" + +move_api.label: +"""改变已注册的Exhook服务器的执行顺序""" + +move_position.desc: +"""移动的方向""" + +node.desc: +"""节点名称""" + +node_hook_metrics.desc: +"""所有节点中该钩子的指标信息""" + +node_metrics.desc: +"""所有节点中该服务器的指标信息""" + +node_status.desc: +"""所有节点中该服务器的状态信息""" + +server_metrics.desc: +"""当前节点中该服务器的指标信息""" + +server_name.desc: +"""Exhook 服务器的名称""" + +status.desc: +"""Exhook 服务器的状态。
+connected: 连接成功
+connecting: 连接失败，重连中
+disconnected: 连接失败，且未设置自动重连
+disabled: 该服务器未开启
+error: 查看该服务器状态时发生错误""" + +update_server.desc: +"""更新 Exhook 服务器""" + +} diff --git a/rel/i18n/zh/emqx_exhook_schema.hocon b/rel/i18n/zh/emqx_exhook_schema.hocon new file mode 100644 index 000000000..54d86169b --- /dev/null +++ b/rel/i18n/zh/emqx_exhook_schema.hocon @@ -0,0 +1,43 @@ +emqx_exhook_schema { + +auto_reconnect.desc: +"""自动重连到 gRPC 服务器的设置。 +当 gRPC 服务器不可用时，Exhook 将会按照这里设置的间隔时间进行重连，并重新初始化注册的钩子""" + +enable.desc: +"""开启这个 Exhook 服务器""" + +failed_action.desc: +"""当 gRPC 请求失败后的操作""" + +keepalive.desc: +"""当没有其他数据交换时，是否向连接的对端套接字定期的发送探测包。如果另一端没有响应，则认为连接断开，并向控制进程发送错误消息""" + +name.desc: +"""ExHook 服务器名称""" + +nodelay.desc: +"""如果为 true，则为套接字设置 TCP_NODELAY 选项，这意味着会立即发送数据包""" + +pool_size.desc: +"""gRPC 客户端进程池大小""" + +recbuf.desc: +"""套接字的最小接收缓冲区大小""" + +request_timeout.desc: +"""gRPC 服务器请求超时时间""" + +servers.desc: +"""ExHook 服务器列表""" + +sndbuf.desc: +"""套接字的最小发送缓冲区大小""" + +socket_options.desc: +"""连接套接字设置""" + +url.desc: +"""gRPC 服务器地址""" + +} diff --git a/rel/i18n/zh/emqx_exproto_schema.hocon b/rel/i18n/zh/emqx_exproto_schema.hocon new file mode 100644 index 000000000..794a563f3 --- /dev/null +++ b/rel/i18n/zh/emqx_exproto_schema.hocon @@ -0,0 +1,26 @@ +emqx_exproto_schema { + +exproto.desc: +"""ExProto 网关""" + +exproto_grpc_handler_address.desc: +"""对端 gRPC 服务器地址。""" + +exproto_grpc_handler_ssl.desc: +"""gRPC 客户端的 SSL 配置。""" + +exproto_grpc_server_bind.desc: +"""服务监听地址和端口。""" + +exproto_grpc_server_ssl.desc: +"""服务 SSL 配置。""" + +exproto_handler.desc: +"""配置 ExProto 网关需要请求的 ConnectionHandler 服务地址。 +该服务用于给 ExProto 提供客户端的 Socket 事件处理、字节解码、订阅消息接收等功能。""" + +exproto_server.desc: +"""配置 ExProto 网关需要启动的 ConnectionAdapter 服务。 +该服务用于提供客户端的认证、发布、订阅和数据下行等功能。""" + +} diff --git a/rel/i18n/zh/emqx_gateway_api.hocon b/rel/i18n/zh/emqx_gateway_api.hocon new file mode 100644 index 000000000..c9ea582a3 --- /dev/null +++ b/rel/i18n/zh/emqx_gateway_api.hocon @@ -0,0 +1,73 @@ +emqx_gateway_api { + +delete_gateway.desc: +"""停用指定网关""" + +enable_gateway.desc: +"""使用配置启动某一网关。""" + +gateway_created_at.desc: +"""网关创建时间""" + +gateway_current_connections.desc: +"""当前连接数""" + +gateway_enable_in_path.desc: +"""是否开启此网关""" + +gateway_listener_id.desc: +"""监听器 ID""" + +gateway_listener_name.desc: +"""监听器名称""" + +gateway_listener_running.desc: +"""监听器运行状态""" + +gateway_listener_type.desc: +"""监听器类型""" + +gateway_listeners.desc: +"""网关监听器列表""" + +gateway_max_connections.desc: +"""最大连接数""" + +gateway_name.desc: +"""网关名称""" + +gateway_name_in_qs.desc: +"""网关名称.
+可取值为 `stomp`、`mqttsn`、`coap`、`lwm2m`、`exproto`""" + +gateway_node_status.desc: +"""网关在集群中每个节点上的状态""" + +gateway_started_at.desc: +"""网关启用时间""" + +gateway_status.desc: +"""网关启用状态""" + +gateway_status_in_qs.desc: +"""通过网关状态筛选
+可选值为 `running`、`stopped`、`unloaded`""" + +gateway_stopped_at.desc: +"""网关停用时间""" + +get_gateway.desc: +"""获取网关配置详情""" + +list_gateway.desc: +"""该接口会返回指定或所有网关的概览状态， +包括当前状态、连接数、监听器状态等。""" + +node.desc: +"""节点名称""" + +update_gateway.desc: +"""更新指定网关的基础配置、和启用的状态。
+注：认证、和监听器的配置更新需参考对应的 API 接口。""" + +} diff --git a/rel/i18n/zh/emqx_gateway_api_authn.hocon b/rel/i18n/zh/emqx_gateway_api_authn.hocon new file mode 100644 index 000000000..662789551 --- /dev/null +++ b/rel/i18n/zh/emqx_gateway_api_authn.hocon @@ -0,0 +1,45 @@ +emqx_gateway_api_authn { + +add_authn.desc: +"""为指定网关开启认证器实现客户端认证的功能。
+当未配置认证器或关闭认证器时，则认为允许所有客户端的连接。
+注：在网关中仅支持添加一个认证器，而不是像 MQTT 一样允许配置多个认证器构成认证链。""" + +add_user.desc: +"""添加用户（仅支持 built_in_database 类型的认证器）""" + +delete_authn.desc: +"""删除指定网关的认证器。""" + +delete_user.desc: +"""删除用户（仅支持 built_in_database 类型的认证器）""" + +get_authn.desc: +"""获取指定网关认证器的配置 +当网关或认证未启用时，返回 404。""" + +get_user.desc: +"""获取用户信息（仅支持 built_in_database 类型的认证器）""" + +import_users.desc: +"""导入用户（仅支持 built_in_database 类型的认证器）""" + +is_superuser.desc: +"""是否是超级用户""" + +like_user_id.desc: +"""使用用户 ID （username 或 clientid）模糊搜索，仅支持按子串的方式进行搜索。""" + +list_users.desc: +"""获取用户列表（仅支持 built_in_database 类型的认证器）""" + +update_authn.desc: +"""更新指定网关认证器的配置，或停用认证器。""" + +update_user.desc: +"""更新用户信息（仅支持 built_in_database 类型的认证器）""" + +user_id.desc: +"""用户 ID""" + +} diff --git a/rel/i18n/zh/emqx_gateway_api_clients.hocon b/rel/i18n/zh/emqx_gateway_api_clients.hocon new file mode 100644 index 000000000..5dc7ff22a --- /dev/null +++ b/rel/i18n/zh/emqx_gateway_api_clients.hocon @@ -0,0 +1,207 @@ +emqx_gateway_api_clients { + +disconnected_at.desc: +"""客户端连接断开时间""" + +heap_size.desc: +"""进程堆内存大小，单位：字节""" + +send_oct.desc: +"""已发送字节数""" + +get_client.desc: +"""获取客户端信息""" + +param_gte_created_at.desc: +"""匹配会话创建时间大于等于指定值的客户端""" + +param_conn_state.desc: +"""匹配客户端连接状态""" + +send_pkt.desc: +"""已发送应用层协议控制报文数""" + +clean_start.desc: +"""标识客户端是否以 clean_start 的标志连接到网关""" + +inflight_cnt.desc: +"""客户端当前飞行窗口大小""" + +delete_subscription.desc: +"""为某客户端删除某订阅关系""" + +param_lte_connected_at.desc: +"""匹配连接创建时间小于等于指定值的客户端""" + +node.desc: +"""客户端连接到的节点名称""" + +awaiting_rel_cnt.desc: +"""客户端当前等待 PUBREL 确认的 PUBREC 消息的条数""" + +rap.desc: +"""Retain as Published 选项，枚举：0，1""" + +inflight_max.desc: +"""客户端允许的飞行窗口最大值""" + +param_username.desc: +"""匹配客户端 Username""" + +param_like_endpoint_name.desc: +"""子串匹配 LwM2M 客户端 Endpoint Name""" + +created_at.desc: +"""会话创建时间""" + +sub_props.desc: +"""订阅属性""" + +list_clients.desc: +"""获取指定网关的客户端列表""" + +subscriptions_cnt.desc: +"""客户端已订阅主题数""" + +mailbox_len.desc: +"""进程邮箱大小""" + +send_cnt.desc: +"""已发送 Socket 报文次数""" + +rh.desc: +"""Retain Handling 选项，枚举：0，1，2""" + +connected.desc: +"""标识客户端是否已连接到网关""" + +qos.desc: +"""QoS 等级，枚举：0，1，2""" + +ip_address.desc: +"""客户端 IP 地址""" + +param_gte_connected_at.desc: +"""匹配连接创建时间大于等于指定值的客户端""" + +awaiting_rel_max.desc: +"""客户端允许的最大 PUBREC 等待队列长度""" + +param_like_username.desc: +"""子串匹配 客户端 Username""" + +nl.desc: +"""No Local 选项，枚举：0，1""" + +param_like_clientid.desc: +"""子串匹配客户端 ID""" + +param_lte_created_at.desc: +"""匹配会话创建时间小于等于指定值的客户端""" + +topic.desc: +"""主题过滤器或主题名称""" + +proto_ver.desc: +"""客户端使用的协议版本""" + +mountpoint.desc: +"""主题固定前缀""" + +proto_name.desc: +"""客户端使用的协议名称""" + +param_lte_lifetime.desc: +"""匹配心跳时间小于等于指定值的 LwM2M 客户端""" + +port.desc: +"""客户端端口""" + +connected_at.desc: +"""客户端连接时间""" + +expiry_interval.desc: +"""会话超期时间，单位：秒""" + +username.desc: +"""客户端连接的用户名""" + +param_clean_start.desc: +"""匹配客户端 `clean_start` 标记""" + +recv_msg.desc: +"""已接收上行的消息条数""" + +list_subscriptions.desc: +"""获取某客户端的主题订阅列表""" + +recv_oct.desc: +"""已接收的字节数""" + +keepalive.desc: +"""Keepalive 时间，单位：秒""" + +param_clientid.desc: +"""匹配客户端 ID""" + +subscriptions_max.desc: +"""客户端允许订阅的最大主题数""" + +param_ip_address.desc: +"""匹配客户端 IP 地址""" + +mqueue_max.desc: +"""客户端允许的最大消息队列长度""" + +mqueue_dropped.desc: +"""由于消息队列过程，客户端消息队列丢弃消息条数""" + +subid.desc: +"""订阅ID，仅用于 Stomp 网关。用于创建订阅关系时指定订阅 ID。取值范围 1-65535。""" + +clientid.desc: +"""客户端 ID""" + +kick_client.desc: +"""踢出指定客户端""" + +is_bridge.desc: +"""标识客户端是否通过 is_bridge 标志连接""" + +lifetime.desc: +"""LwM2M 客户端心跳周期""" + +send_msg.desc: +"""已发送下行消息数条数""" + +add_subscription.desc: +"""为某客户端新增订阅关系""" + +param_endpoint_name.desc: +"""匹配 LwM2M 客户端 Endpoint Name""" + +param_node.desc: +"""匹配客户端的节点名称""" + +recv_cnt.desc: +"""已接收 Socket 报文次数""" + +recv_pkt.desc: +"""已接收应用层协议控制报文数""" + +endpoint_name.desc: +"""LwM2M 客户端 Endpoint Name""" + +param_proto_ver.desc: +"""匹配客户端协议版本""" + +reductions.desc: +"""进程已消耗 Reduction 数""" + +param_gte_lifetime.desc: +"""匹配心跳时间大于等于指定值的 LwM2M 客户端""" + +mqueue_len.desc: +"""客户端当前消息队列长度""" + +} diff --git a/rel/i18n/zh/emqx_gateway_api_listeners.hocon b/rel/i18n/zh/emqx_gateway_api_listeners.hocon new file mode 100644 index 000000000..731b14b74 --- /dev/null +++ b/rel/i18n/zh/emqx_gateway_api_listeners.hocon @@ -0,0 +1,65 @@ +emqx_gateway_api_listeners { + +add_listener.desc: +"""为指定网关添加监听器。
+注：对于某网关不支持的监听器类型，该接口会返回 `400: BAD_REQUEST`。""" + +add_listener_authn.desc: +"""为指定监听器开启认证器以实现客户端认证的能力。
+当某一监听器开启认证后，所有连接到该监听器的客户端会使用该认证器进行认证。""" + +add_user.desc: +"""添加用户（仅支持 built_in_database 类型的认证器）""" + +current_connections.desc: +"""当前连接数""" + +delete_listener.desc: +"""删除指定监听器。被删除的监听器下所有已连接的客户端都会离线。""" + +delete_listener_authn.desc: +"""移除指定监听器的认证器。""" + +delete_user.desc: +"""删除用户（仅支持 built_in_database 类型的认证器）""" + +get_listener.desc: +"""获取指定网关监听器的配置。""" + +get_listener_authn.desc: +"""获取监听器的认证器配置。""" + +get_user.desc: +"""获取用户信息（仅支持 built_in_database 类型的认证器）""" + +import_users.desc: +"""导入用户（仅支持 built_in_database 类型的认证器）""" + +list_listeners.desc: +"""获取网关监听器列表。该接口会返回监听器所有的配置（包括该监听器上的认证器），同时也会返回该监听器在集群中运行的状态。""" + +list_users.desc: +"""获取用户列表（仅支持 built_in_database 类型的认证器）""" + +listener_id.desc: +"""监听器 ID""" + +listener_node_status.desc: +"""监听器在集群中每个节点上的状态""" + +listener_status.desc: +"""监听器状态""" + +node.desc: +"""节点名称""" + +update_listener.desc: +"""更新某网关监听器的配置。被更新的监听器会执行重启，所有已连接到该监听器上的客户端都会被断开。""" + +update_listener_authn.desc: +"""更新指定监听器的认证器配置，或停用/启用该认证器。""" + +update_user.desc: +"""更新用户信息（仅支持 built_in_database 类型的认证器）""" + +} diff --git a/rel/i18n/zh/emqx_gateway_schema.hocon b/rel/i18n/zh/emqx_gateway_schema.hocon new file mode 100644 index 000000000..40cee4efb --- /dev/null +++ b/rel/i18n/zh/emqx_gateway_schema.hocon @@ -0,0 +1,112 @@ +emqx_gateway_schema { + +dtls_listener_acceptors.desc: +"""Acceptor 进程池大小。""" + +dtls_listener_dtls_opts.desc: +"""DTLS Socket 配置""" + +gateway_common_authentication.desc: +"""网关的认证器配置，对该网关下所以的监听器生效。如果每个监听器需要配置不同的认证器，需要配置监听器下的 authentication 字段。""" + +gateway_common_clientinfo_override.desc: +"""ClientInfo 重写。""" + +gateway_common_clientinfo_override_clientid.desc: +"""clientid 重写模板""" + +gateway_common_clientinfo_override_password.desc: +"""password 重写模板""" + +gateway_common_clientinfo_override_username.desc: +"""username 重写模板""" + +gateway_common_enable.desc: +"""是否启用该网关""" + +gateway_common_enable_stats.desc: +"""是否开启客户端统计""" + +gateway_common_idle_timeout.desc: +"""客户端连接过程的空闲时间。该配置用于： + 1. 一个新创建的客户端进程如果在该时间间隔内没有收到任何客户端请求，将被直接关闭。 + 2. 一个正在运行的客户进程如果在这段时间后没有收到任何客户请求，将进入休眠状态以节省资源。""" + +gateway_common_listener_access_rules.desc: +"""配置监听器的访问控制规则。 +见：https://github.com/emqtt/esockd#allowdeny""" + +gateway_common_listener_bind.desc: +"""监听器绑定的 IP 地址或端口。""" + +gateway_common_listener_enable.desc: +"""是否启用该监听器。""" + +gateway_common_listener_enable_authn.desc: +"""配置 true （默认值）启用客户端进行身份认证。 +配置 false 时，将不对客户端做任何认证。""" + +gateway_common_listener_max_conn_rate.desc: +"""监听器支持的最大连接速率。""" + +gateway_common_listener_max_connections.desc: +"""监听器支持的最大连接数。""" + +gateway_mountpoint.desc: +"""发布或订阅时，在所有主题前增加前缀字符串。 +当消息投递给订阅者时，前缀字符串将从主题名称中删除。挂载点是用户可以用来实现不同监听器之间的消息路由隔离的一种方式。 +例如，如果客户端 A 在 `listeners.tcp.\.mountpoint` 设置为 `some_tenant` 的情况下订阅 `t`， +则客户端实际上订阅了 `some_tenant/t` 主题。 +类似地，如果另一个客户端 B（连接到与客户端 A 相同的侦听器）向主题 `t` 发送消息， +则该消息被路由到所有订阅了 `some_tenant/t` 的客户端，因此客户端 A 将收到该消息，带有 主题名称`t`。 设置为 `""` 以禁用该功能。 +挂载点字符串中可用的变量：
+ - ${clientid}：clientid
+ - ${username}：用户名""" + +listener_name_to_settings_map.desc: +"""从监听器名称到配置参数的映射。""" + +ssl_listener_options.desc: +"""SSL Socket 配置。""" + +tcp_listener_acceptors.desc: +"""Acceptor 进程池大小。""" + +tcp_listener_proxy_protocol.desc: +"""是否开启 Proxy Protocol V1/2。当 EMQX 集群部署在 HAProxy 或 Nginx 后需要获取客户端真实 IP 时常用到该选项。参考：https://www.haproxy.com/blog/haproxy/proxy-protocol/""" + +tcp_listener_proxy_protocol_timeout.desc: +"""接收 Proxy Protocol 报文头的超时时间。如果在超时内没有收到 Proxy Protocol 包，EMQX 将关闭 TCP 连接。""" + +tcp_listener_tcp_opts.desc: +"""TCP Socket 配置。""" + +tcp_listeners.desc: +"""配置 TCP 类型的监听器。""" + +tcp_udp_listeners.desc: +"""监听器配置。""" + +udp_listener_active_n.desc: +"""为 Socket 指定 {active, N} 选项。 +参见：https://erlang.org/doc/man/inet.html#setopts-2""" + +udp_listener_buffer.desc: +"""Socket 在用户空间的缓冲区大小。""" + +udp_listener_recbuf.desc: +"""Socket 在内核空间接收缓冲区的大小。""" + +udp_listener_reuseaddr.desc: +"""允许重用本地处于 TIME_WAIT 的端口号。""" + +udp_listener_sndbuf.desc: +"""Socket 在内核空间发送缓冲区的大小。""" + +udp_listener_udp_opts.desc: +"""UDP Socket 配置。""" + +udp_listeners.desc: +"""配置 UDP 类型的监听器。""" + +} diff --git a/rel/i18n/zh/emqx_license_http_api.hocon b/rel/i18n/zh/emqx_license_http_api.hocon new file mode 100644 index 000000000..4ad471684 --- /dev/null +++ b/rel/i18n/zh/emqx_license_http_api.hocon @@ -0,0 +1,15 @@ +emqx_license_http_api { + +desc_license_info_api.desc: +"""获取许可证信息""" + +desc_license_info_api.label: +"""许可证信息""" + +desc_license_key_api.desc: +"""更新一个许可证密钥""" + +desc_license_key_api.label: +"""更新许可证""" + +} diff --git a/rel/i18n/zh/emqx_license_schema.hocon b/rel/i18n/zh/emqx_license_schema.hocon new file mode 100644 index 000000000..0bf5256e8 --- /dev/null +++ b/rel/i18n/zh/emqx_license_schema.hocon @@ -0,0 +1,29 @@ +emqx_license_schema { + +connection_high_watermark_field.desc: +"""高水位线，连接数超过这个水位线时，系统会触发许可证连接配额使用告警""" + +connection_high_watermark_field.label: +"""连接高水位""" + +connection_low_watermark_field.desc: +"""低水位限制，低于此水位线时系统会清除连接配额使用告警""" + +connection_low_watermark_field.label: +"""连接低水位线""" + +key_field.desc: +"""许可证字符串""" + +key_field.label: +"""许可证字符串""" + +license_root.desc: +"""EMQX企业许可证。 +EMQX 自带一个默认的试用许可证，默认试用许可允许最多接入 100 个连接，签发时间是 2023年1月9日，有效期是 5 年（1825 天）。若需要在生产环境部署， +请访问 https://www.emqx.com/apply-licenses/emqx 来申请。""" + +license_root.label: +"""许可证""" + +} diff --git a/rel/i18n/zh/emqx_limiter_schema.hocon b/rel/i18n/zh/emqx_limiter_schema.hocon new file mode 100644 index 000000000..4f5a0ce2f --- /dev/null +++ b/rel/i18n/zh/emqx_limiter_schema.hocon @@ -0,0 +1,89 @@ +emqx_limiter_schema { + +bucket_cfg.desc: +"""桶的配置""" + +bucket_cfg.label: +"""桶的配置""" + +burst.desc: +"""突发速率。 +突发速率允许短时间内速率超过设置的速率值，突发速率 + 速率 = 当前桶能达到的最大速率值""" + +burst.label: +"""突发速率""" + +bytes.desc: +"""流入字节率控制器。 +这个是用来控制当前节点上的数据流入的字节率，每条消息将会消耗和其二进制大小等量的令牌，当达到最大速率后，会话将会被限速甚至被强制挂起一小段时间""" + +bytes.label: +"""流入字节率""" + +client.desc: +"""对桶的每个使用者的速率控制设置""" + +client.label: +"""每个使用者的限制""" + +connection.desc: +"""连接速率控制器。 +这个用来控制当前节点上的连接速率，当达到最大速率后，新的连接将会被拒绝""" + +connection.label: +"""连接速率""" + +divisible.desc: +"""申请的令牌数是否可以被分割""" + +divisible.label: +"""是否可分割""" + +failure_strategy.desc: +"""当所有的重试都失败后的处理策略""" + +failure_strategy.label: +"""失败策略""" + +initial.desc: +"""桶中的初始令牌数""" + +initial.label: +"""初始令牌数""" + +internal.desc: +"""EMQX 内部功能所用限制器。""" + +low_watermark.desc: +"""当桶中剩余的令牌数低于这个值，即使令牌申请成功了，也会被强制暂停一会儿""" + +low_watermark.label: +"""低水位线""" + +max_retry_time.desc: +"""申请失败后，尝试重新申请的时长最大值""" + +max_retry_time.label: +"""最大重试时间""" + +message_routing.desc: +"""消息派发速率控制器。 +这个用来控制当前节点内的消息派发速率，当达到最大速率后，新的推送将会被拒绝""" + +message_routing.label: +"""消息派发""" + +messages.desc: +"""流入速率控制器。 +这个用来控制当前节点上的消息流入速率，当达到最大速率后，会话将会被限速甚至被强制挂起一小段时间""" + +messages.label: +"""消息流入速率""" + +rate.desc: +"""桶的令牌生成速率""" + +rate.label: +"""速率""" + +} diff --git a/rel/i18n/zh/emqx_lwm2m_api.hocon b/rel/i18n/zh/emqx_lwm2m_api.hocon new file mode 100644 index 000000000..7cf53060a --- /dev/null +++ b/rel/i18n/zh/emqx_lwm2m_api.hocon @@ -0,0 +1,27 @@ +emqx_lwm2m_api { + +dataType.desc: +"""数据类型""" + +lookup_resource.desc: +"""查看指定资源状态""" + +name.desc: +"""资源名称""" + +observe_resource.desc: +"""Observe/Un-Observe 指定资源""" + +operations.desc: +"""资源可用操作列表""" + +path.desc: +"""资源路径""" + +read_resource.desc: +"""发送读指令到某资源""" + +write_resource.desc: +"""发送写指令到某资源""" + +} diff --git a/rel/i18n/zh/emqx_lwm2m_schema.hocon b/rel/i18n/zh/emqx_lwm2m_schema.hocon new file mode 100644 index 000000000..3dea6a0c6 --- /dev/null +++ b/rel/i18n/zh/emqx_lwm2m_schema.hocon @@ -0,0 +1,56 @@ +emqx_lwm2m_schema { + +lwm2m.desc: +"""LwM2M 网关配置。仅支持 v1.0.1 协议。""" + +lwm2m_auto_observe.desc: +"""自动 Observe REGISTER 数据包的 Object 列表。""" + +lwm2m_lifetime_max.desc: +"""允许 LwM2M 客户端允许设置的心跳最大值。""" + +lwm2m_lifetime_min.desc: +"""允许 LwM2M 客户端允许设置的心跳最小值。""" + +lwm2m_qmode_time_window.desc: +"""在QMode模式下，LwM2M网关认为网络链接有效的时间窗口的值。 +例如，在收到客户端的更新信息后，在这个时间窗口内的任何信息都会直接发送到LwM2M客户端，而超过这个时间窗口的所有信息都会暂时储存在内存中。""" + +lwm2m_translators.desc: +"""LwM2M 网关订阅/发布消息的主题映射配置。""" + +lwm2m_translators_command.desc: +"""下行命令主题。 +对于每个成功上线的新 LwM2M 客户端，网关会创建一个订阅关系来接收下行消息并将其发送给客户端。""" + +lwm2m_translators_notify.desc: +"""用于发布来自 LwM2M 客户端的通知事件的主题。 +在成功 Observe 到 LwM2M 客户端的资源后，如果客户端报告任何资源状态的变化，网关将通过该主题发送通知事件。""" + +lwm2m_translators_register.desc: +"""用于发布来自 LwM2M 客户端的注册事件的主题。""" + +lwm2m_translators_response.desc: +"""用于网关发布来自 LwM2M 客户端的确认事件的主题。""" + +lwm2m_translators_update.desc: +"""用于发布来自LwM2M客户端的更新事件的主题。""" + +lwm2m_update_msg_publish_condition.desc: +"""发布UPDATE事件消息的策略。
+ - always: 只要收到 UPDATE 请求，就发送更新事件。
+ - contains_object_list: 仅当 UPDATE 请求携带 Object 列表时才发送更新事件。""" + +lwm2m_xml_dir.desc: +"""LwM2M Resource 定义的 XML 文件目录路径。""" + +translator.desc: +"""配置某网关客户端对于发布消息或订阅的主题和 QoS 等级。""" + +translator_qos.desc: +"""QoS 等级""" + +translator_topic.desc: +"""主题名称""" + +} diff --git a/rel/i18n/zh/emqx_mgmt_api_alarms.hocon b/rel/i18n/zh/emqx_mgmt_api_alarms.hocon new file mode 100644 index 000000000..d9dafd867 --- /dev/null +++ b/rel/i18n/zh/emqx_mgmt_api_alarms.hocon @@ -0,0 +1,37 @@ +emqx_mgmt_api_alarms { + +activate_at.desc: +"""告警开始时间，使用 rfc3339 标准时间格式。""" + +deactivate_at.desc: +"""告警结束时间，使用 rfc3339 标准时间格式。""" + +delete_alarms_api.desc: +"""删除所有历史告警。""" + +delete_alarms_api_response204.desc: +"""历史告警已成功清除。""" + +details.desc: +"""告警详情，提供了更多的告警信息，主要提供给程序处理。""" + +duration.desc: +"""表明告警已经持续了多久，单位：毫秒。""" + +get_alarms_qs_activated.desc: +"""用于指定查询的告警类型， +为 true 时返回当前激活的告警，为 false 时返回历史告警，默认为 false。""" + +list_alarms_api.desc: +"""列出当前激活的告警或历史告警，由查询参数决定。""" + +message.desc: +"""告警消息，以人类可读的方式描述告警内容。""" + +name.desc: +"""告警名称，用于区分不同的告警。""" + +node.desc: +"""触发此告警的节点名称。""" + +} diff --git a/rel/i18n/zh/emqx_mgmt_api_banned.hocon b/rel/i18n/zh/emqx_mgmt_api_banned.hocon new file mode 100644 index 000000000..cee3ba288 --- /dev/null +++ b/rel/i18n/zh/emqx_mgmt_api_banned.hocon @@ -0,0 +1,54 @@ +emqx_mgmt_api_banned { + +as.desc: +"""封禁方式，可以通过客户端 ID、用户名或者 IP 地址等方式进行封禁。""" + +as.label: +"""封禁方式""" + +at.desc: +"""封禁的起始时间，格式为 rfc3339，默认为发起操作的时间。""" + +at.label: +"""封禁时间""" + +by.desc: +"""封禁的发起者。""" + +by.label: +"""封禁发起者""" + +create_banned_api.desc: +"""添加一个客户端 ID、用户名或者 IP 地址到黑名单。""" + +create_banned_api_response400.desc: +"""错误的请求，可能是参数错误或封禁对象已存在等原因。""" + +delete_banned_api.desc: +"""将一个客户端 ID、用户名或者 IP 地址从黑名单中删除。""" + +delete_banned_api_response404.desc: +"""未在黑名单中找到该封禁对象。""" + +list_banned_api.desc: +"""列出目前所有被封禁的客户端 ID、用户名和 IP 地址。""" + +reason.desc: +"""封禁原因，记录当前对象被封禁的原因。""" + +reason.label: +"""封禁原因""" + +until.desc: +"""封禁的结束时间，格式为 rfc3339，默认值为发起操作的时间 + 1 年。""" + +until.label: +"""封禁结束时间""" + +who.desc: +"""封禁对象，具体的客户端 ID、用户名或者 IP 地址。""" + +who.label: +"""封禁对象""" + +} diff --git a/rel/i18n/zh/emqx_mgmt_api_key_schema.hocon b/rel/i18n/zh/emqx_mgmt_api_key_schema.hocon new file mode 100644 index 000000000..9a0536fe6 --- /dev/null +++ b/rel/i18n/zh/emqx_mgmt_api_key_schema.hocon @@ -0,0 +1,19 @@ +emqx_mgmt_api_key_schema { + +api_key.desc: +"""API 密钥， 可用于请求除管理 API 密钥及 Dashboard 用户管理 API 的其它接口""" + +api_key.label: +"""API 密钥""" + +bootstrap_file.desc: +"""用于在启动 emqx 时，添加 API 密钥，其格式为： + ``` + 7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK + ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL + ```""" + +bootstrap_file.label: +"""API 密钥初始化文件""" + +} diff --git a/rel/i18n/zh/emqx_mgmt_api_publish.hocon b/rel/i18n/zh/emqx_mgmt_api_publish.hocon new file mode 100644 index 000000000..2a532fbdd --- /dev/null +++ b/rel/i18n/zh/emqx_mgmt_api_publish.hocon @@ -0,0 +1,81 @@ +emqx_mgmt_api_publish { + +error_message.desc: +"""失败的详细原因。""" + +message_id.desc: +"""全局唯一的一个消息 ID，方便用于关联和追踪。""" + +message_properties.desc: +"""PUBLISH 消息里的 Property 字段。""" + +msg_content_type.desc: +"""内容类型标识符，以 UTF-8 格式编码的字符串，用来描述应用消息的内容，服务端必须把收到的应用消息中的内容类型原封不动的发送给所有的订阅者。""" + +msg_correlation_data.desc: +"""对比数据标识符，服务端在收到应用消息时必须原封不动的把对比数据发送给所有的订阅者。对比数据只对请求消息(Request Message)的发送端和响应消息(Response Message)的接收端有意义。""" + +msg_message_expiry_interval.desc: +"""消息过期间隔标识符，以秒为单位。当消失已经过期时，如果服务端还没有开始向匹配的订阅者投递该消息，则服务端会删除该订阅者的消息副本。如果不设置，则消息永远不会过期""" + +msg_payload_format_indicator.desc: +"""载荷格式指示标识符，0 表示载荷是未指定格式的数据，相当于没有发送载荷格式指示；1 表示载荷是 UTF-8 编码的字符数据，载荷中的 UTF-8 数据必须是按照 Unicode 的规范和 RFC 3629 的标准要求进行编码的。""" + +msg_response_topic.desc: +"""响应主题标识符， UTF-8 编码的字符串，用作响应消息的主题名。响应主题不能包含通配符，也不能包含多个主题，否则将造成协议错误。当存在响应主题时，消息将被视作请求报文。服务端在收到应用消息时必须将响应主题原封不动的发送给所有的订阅者。""" + +msg_user_properties.desc: +"""指定 MQTT 消息的 User Property 键值对。注意，如果出现重复的键，只有最后一个会保留。""" + +payload.desc: +"""MQTT 消息体。""" + +payload_encoding.desc: +"""MQTT 消息体的编码方式，可以是 base64 或 plain。当设置为 base64 时，消息在发布前会先被解码。""" + +publish_api.desc: +"""发布一个消息。
+可能的 HTTP 状态码如下：
+200: 消息被成功发送到至少一个订阅。
+202: 没有匹配到任何订阅。
+400: 消息编码错误，如非法主题，或 QoS 超出范围等。
+503: 服务重启等过程中导致转发失败。""" + +publish_api.label: +"""发布一条信息""" + +publish_bulk_api.desc: +"""批量发布一组消息。
+可能的 HTTP 状态码如下：
+200: 所有的消息都被成功发送到至少一个订阅。
+202: 至少有一个消息没有匹配到任何订阅。
+400: 至少有一个消息编码错误，如非法主题，或 QoS 超出范围等。
+503: 至少有一个小因为服务重启的原因导致转发失败。
+ +请求的 Body 或者 Body 中包含的某个消息无法通过 API 规范的类型检查时，HTTP 响应的消息与发布单个消息的 API + /publish 是一样的。 +如果所有的消息都是合法的，那么 HTTP 返回的内容是一个 JSON 数组，每个元素代表了该消息转发的状态。""" + +publish_bulk_api.label: +"""发布一批信息""" + +qos.desc: +"""MQTT 消息的 QoS""" + +reason_code.desc: +"""MQTT 消息发布的错误码，这些错误码也是 MQTT 规范中 PUBACK 消息可能携带的错误码。
+当前支持如下错误码：
+ +16(0x10)：没能匹配到任何订阅；
+131(0x81)：消息转发时发生错误，例如 EMQX 服务重启；
+144(0x90)：主题名称非法；
+151(0x97)：受到了速率限制，或者消息尺寸过大。全局消息大小限制可以通过配置项 mqtt.max_packet_size 来进行修改。
+注意：消息尺寸的是通过主题和消息体的字节数进行估算的。具体发布时所占用的字节数可能会稍大于这个估算的值。""" + +retain.desc: +"""布尔型字段，用于表示该消息是否保留消息。""" + +topic_name.desc: +"""主题名称""" + +} diff --git a/rel/i18n/zh/emqx_mgmt_api_status.hocon b/rel/i18n/zh/emqx_mgmt_api_status.hocon new file mode 100644 index 000000000..3625db967 --- /dev/null +++ b/rel/i18n/zh/emqx_mgmt_api_status.hocon @@ -0,0 +1,22 @@ +emqx_mgmt_api_status { + +get_status_api.desc: +"""作为节点的健康检查。 返回一个纯文本的响应，描述节点的状态。 + +如果 EMQX 应用程序已经启动并运行，返回状态代码 200，否则返回 503。 + +这个API是在v5.0.10中引入的。 +GET `/status`端点（没有`/api/...`前缀）也是这个端点的一个别名，工作方式相同。 这个别名从v5.0.0开始就有了。""" + +get_status_api.label: +"""服务健康检查""" + +get_status_response200.desc: +"""Node emqx@127.0.0.1 is started +emqx is running""" + +get_status_response503.desc: +"""Node emqx@127.0.0.1 is stopped +emqx is not_running""" + +} diff --git a/rel/i18n/zh/emqx_modules_schema.hocon b/rel/i18n/zh/emqx_modules_schema.hocon new file mode 100644 index 000000000..e1c2ca913 --- /dev/null +++ b/rel/i18n/zh/emqx_modules_schema.hocon @@ -0,0 +1,45 @@ +emqx_modules_schema { + +enable.desc: +"""是否开启该功能""" + +max_delayed_messages.desc: +"""延迟消息的数量上限(0 代表无限)""" + +rewrite.desc: +"""EMQX 的主题重写功能支持根据用户配置的规则在客户端订阅主题、发布消息、取消订阅的时候将 A 主题重写为 B 主题。 +重写规则分为 Pub 规则和 Sub 规则，Pub 规则匹配 PUSHLISH 报文携带的主题，Sub 规则匹配 SUBSCRIBE、UNSUBSCRIBE 报文携带的主题。 +每条重写规则都由主题过滤器、正则表达式、目标表达式三部分组成。 +在主题重写功能开启的前提下，EMQX 在收到诸如 PUBLISH 报文等带有主题的 MQTT 报文时，将使用报文中的主题去依次匹配配置文件中规则的主题过滤器部分，一旦成功匹配，则使用正则表达式提取主题中的信息，然后替换至目标表达式以构成新的主题。 +目标表达式中可以使用 `$N` 这种格式的变量匹配正则表达中提取出来的元素，`$N` 的值为正则表达式中提取出来的第 N 个元素，比如 `$1` 即为正则表达式提取的第一个元素。 +需要注意的是，EMQX 使用倒序读取配置文件中的重写规则，当一条主题可以同时匹配多条主题重写规则的主题过滤器时，EMQX 仅会使用它匹配到的第一条规则进行重写，如果该条规则中的正则表达式与 MQTT 报文主题不匹配，则重写失败，不会再尝试使用其他的规则进行重写。 +因此用户在使用时需要谨慎的设计 MQTT 报文主题以及主题重写规则。""" + +rewrite.label: +"""主题重写""" + +tr_action.desc: +"""主题重写在哪种操作上生效： + - `subscribe`：订阅时重写主题； + - `publish`：发布时重写主题； + -`all`：全部重写主题""" + +tr_action.label: +"""Action""" + +tr_dest_topic.desc: +"""目标主题。""" + +tr_dest_topic.label: +"""目标主题""" + +tr_re.desc: +"""正则表达式""" + +tr_source_topic.desc: +"""源主题，客户端业务指定的主题""" + +tr_source_topic.label: +"""源主题""" + +} diff --git a/rel/i18n/zh/emqx_mqttsn_schema.hocon b/rel/i18n/zh/emqx_mqttsn_schema.hocon new file mode 100644 index 000000000..c6d3a98a6 --- /dev/null +++ b/rel/i18n/zh/emqx_mqttsn_schema.hocon @@ -0,0 +1,30 @@ +emqx_mqttsn_schema { + +mqttsn.desc: +"""MQTT-SN 网关配置。当前实现仅支持 v1.2 版本""" + +mqttsn_broadcast.desc: +"""是否周期性广播 ADVERTISE 消息""" + +mqttsn_enable_qos3.desc: +"""是否允许无连接的客户端发送 QoS 等于 -1 的消息。 +该功能主要用于支持轻量的 MQTT-SN 客户端实现，它不会向网关建立连接，注册主题，也不会发起订阅；它只使用 QoS 为 -1 来发布消息""" + +mqttsn_gateway_id.desc: +"""MQTT-SN 网关 ID。 +当 broadcast 打开时，MQTT-SN 网关会使用该 ID 来广播 ADVERTISE 消息""" + +mqttsn_predefined.desc: +"""预定义主题列表。 +预定义的主题列表，是一组 主题 ID 和 主题名称 的映射关系。使用预先定义的主题列表，可以减少 MQTT-SN 客户端和网关对于固定主题的注册请求""" + +mqttsn_predefined_id.desc: +"""主题 ID。范围：1-65535""" + +mqttsn_predefined_topic.desc: +"""主题名称。注：不支持通配符""" + +mqttsn_subs_resume.desc: +"""在会话被重用后，网关是否主动向客户端注册对已订阅主题名称""" + +} diff --git a/rel/i18n/zh/emqx_plugins_schema.hocon b/rel/i18n/zh/emqx_plugins_schema.hocon new file mode 100644 index 000000000..4da5c5a7b --- /dev/null +++ b/rel/i18n/zh/emqx_plugins_schema.hocon @@ -0,0 +1,46 @@ +emqx_plugins_schema { + +check_interval.desc: +"""检查间隔：检查集群中插件的状态是否一致，
+如果连续3次检查结果不一致，则报警。""" + +enable.desc: +"""设置为“true”以启用此插件""" + +enable.label: +"""启用""" + +install_dir.desc: +"""插件安装包的目录，出于安全考虑，该目录应该值允许 emqx，或用于运行 EMQX 服务的用户拥有写入权限。""" + +install_dir.label: +"""安装目录""" + +name_vsn.desc: +"""插件的名称{name}-{version}。
+它应该与插件的发布包名称一致，如my_plugin-0.1.0。""" + +name_vsn.label: +"""名称-版本""" + +plugins.desc: +"""管理EMQX插件。
+插件可以是EMQX安装包中的一部分，也可以是一个独立的安装包。
+独立安装的插件称为“外部插件”。""" + +plugins.label: +"""插件""" + +state.desc: +"""描述插件的状态""" + +state.label: +"""插件状态""" + +states.desc: +"""一组插件的状态。插件将按照定义的顺序启动""" + +states.label: +"""插件启动顺序及状态""" + +} diff --git a/rel/i18n/zh/emqx_prometheus_schema.hocon b/rel/i18n/zh/emqx_prometheus_schema.hocon new file mode 100644 index 000000000..a1e59e517 --- /dev/null +++ b/rel/i18n/zh/emqx_prometheus_schema.hocon @@ -0,0 +1,47 @@ +emqx_prometheus_schema { + +enable.desc: +"""开启或关闭 Prometheus 数据推送""" + +headers.desc: +"""推送到 Push Gateway 的 HTTP Headers 列表。
+例如， { Authorization = "some-authz-tokens"}""" + +interval.desc: +"""数据推送间隔""" + +job_name.desc: +"""推送到 Push Gateway 的 Job 名称。可用变量为：
+- ${name}: EMQX 节点的名称。 +- ${host}: EMQX 节点主机名。 +例如，当 EMQX 节点名为 emqx@127.0.0.1 则 name 变量的值为 emqx，host 变量的值为 127.0.0.1。
+默认值为: ${name}/instance/${name}~${host}""" + +mnesia_collector.desc: +"""开启或关闭 Mnesia 采集器, 使用 mnesia:system_info/1 收集 Mnesia 相关指标""" + +prometheus.desc: +"""Prometheus 监控数据推送""" + +prometheus.label: +"""Prometheus""" + +push_gateway_server.desc: +"""Prometheus 服务器地址""" + +vm_dist_collector.desc: +"""开启或关闭 VM 分布采集器，收集 Erlang 分布机制中涉及的套接字和进程的信息。""" + +vm_memory_collector.desc: +"""开启或关闭 VM 内存采集器, 使用 erlang:memory/0 收集 Erlang 虚拟机动态分配的内存信息，同时提供基本的 (D)ETS 统计信息""" + +vm_msacc_collector.desc: +"""开启或关闭 VM msacc 采集器, 使用 erlang:statistics(microstate_accounting) 收集微状态计数指标""" + +vm_statistics_collector.desc: +"""开启或关闭 VM 统计采集器, 使用 erlang:statistics/1 收集 Erlang VM 相关指标""" + +vm_system_info_collector.desc: +"""开启或关闭 VM 系统信息采集器, 使用 erlang:system_info/1 收集 Erlang VM 相关指标""" + +} diff --git a/rel/i18n/zh/emqx_psk_schema.hocon b/rel/i18n/zh/emqx_psk_schema.hocon new file mode 100644 index 000000000..0fc14e730 --- /dev/null +++ b/rel/i18n/zh/emqx_psk_schema.hocon @@ -0,0 +1,28 @@ +emqx_psk_schema { + +chunk_size.desc: +"""将 PSK 文件导入到内建数据时每个块的大小""" + +enable.desc: +"""是否开启 TLS PSK 支持""" + +init_file.desc: +"""如果设置了初始化文件，EMQX 将在启动时从初始化文件中导入 PSK 信息到内建数据库中。 +这个文件需要按行进行组织，每一行必须遵守如下格式: PSKIdentity:SharedSecret +例如: mydevice1:c2VjcmV0""" + +psk_authentication.desc: +"""此配置用于启用 TLS-PSK 身份验证。 + +PSK 是 “Pre-Shared-Keys” 的缩写。 + +注意: 确保 SSL 监听器仅启用了 'tlsv1.2'，并且配置了PSK 密码套件，例如 'RSA-PSK-AES256-GCM-SHA384'。 + +可以通过查看监听器中的 SSL 选项，了解更多详细信息。 + +可以通过配置 'init_file' 来设置初始化的 ID 和 密钥""" + +separator.desc: +"""PSK 文件中 PSKIdentity 和 SharedSecret 之间的分隔符""" + +} diff --git a/rel/i18n/zh/emqx_resource_schema.hocon b/rel/i18n/zh/emqx_resource_schema.hocon new file mode 100644 index 000000000..9365b1026 --- /dev/null +++ b/rel/i18n/zh/emqx_resource_schema.hocon @@ -0,0 +1,112 @@ +emqx_resource_schema { + +auto_restart_interval.desc: +"""资源断开以后，自动重连的时间间隔。""" + +auto_restart_interval.label: +"""自动重连间隔""" + +batch_size.desc: +"""最大批量请求大小。如果设为1，则无批处理。""" + +batch_size.label: +"""最大批量请求大小""" + +batch_time.desc: +"""在较低消息率情况下尝试累积批量输出时的最大等待间隔，以提高资源的利用率。""" + +batch_time.label: +"""批量等待最大间隔""" + +buffer_mode.desc: +"""队列操作模式。 +memory_only: 所有的消息都缓存在内存里。volatile_offload: 先将消息缓存在内存中，当内存中的消息堆积超过一定限制（配置项 buffer_seg_bytes 指定该限制）后， 消息会开始缓存到磁盘上。""" + +buffer_mode.label: +"""缓存模式""" + +buffer_seg_bytes.desc: +"""当缓存模式是 volatile_offload 时适用。该配置用于指定缓存到磁盘上的文件的大小。""" + +buffer_seg_bytes.label: +"""缓存文件大小""" + +creation_opts.desc: +"""资源启动相关的选项。""" + +creation_opts.label: +"""资源启动选项""" + +enable_batch.desc: +"""启用批量模式。""" + +enable_batch.label: +"""启用批量模式""" + +enable_queue.desc: +"""启用磁盘缓存队列（仅对 egress 方向桥接有用）。""" + +enable_queue.label: +"""启用磁盘缓存队列""" + +health_check_interval.desc: +"""健康检查间隔。""" + +health_check_interval.label: +"""健康检查间隔""" + +inflight_window.desc: +"""请求飞行队列窗口大小。当请求模式为异步时，如果需要严格保证来自同一 MQTT 客户端的消息有序，则必须将此值设为 1。""" + +inflight_window.label: +"""请求飞行队列窗口""" + +max_buffer_bytes.desc: +"""每个缓存 worker 允许使用的最大字节数。""" + +max_buffer_bytes.label: +"""缓存队列最大长度""" + +query_mode.desc: +"""请求模式。可选 '同步/异步'，默认为'异步'模式。""" + +query_mode.label: +"""请求模式""" + +request_timeout.desc: +"""从请求进入缓冲区开始计时，如果请求在规定的时间内仍停留在缓冲区内或者已发送但未能及时收到响应或确认，该请求将被视为过期。""" + +request_timeout.label: +"""请求超期""" + +resource_opts.desc: +"""资源相关的选项。""" + +resource_opts.label: +"""资源选项""" + +resume_interval.desc: +"""在发送失败后尝试重传飞行窗口中的请求的时间间隔。""" + +resume_interval.label: +"""重试时间间隔""" + +start_after_created.desc: +"""是否在创建资源后立即启动资源。""" + +start_after_created.label: +"""创建后立即启动""" + +start_timeout.desc: +"""在回复资源创建请求前等待资源进入健康状态的时间。""" + +start_timeout.label: +"""启动超时时间""" + +worker_pool_size.desc: +"""缓存队列 worker 数量。仅对 egress 类型的桥接有意义。当桥接仅有 ingress 方向时，可设置为 0，否则必须大于 0。""" + +worker_pool_size.label: +"""缓存池大小""" + +} diff --git a/rel/i18n/zh/emqx_retainer_api.hocon b/rel/i18n/zh/emqx_retainer_api.hocon new file mode 100644 index 000000000..f8107f8ce --- /dev/null +++ b/rel/i18n/zh/emqx_retainer_api.hocon @@ -0,0 +1,63 @@ +emqx_retainer_api { + +config_content.desc: +"""配置内容""" + +config_not_found.desc: +"""配置不存在""" + +delete_matching_api.desc: +"""删除对应的消息""" + +from_clientid.desc: +"""发布者的 ClientID""" + +from_username.desc: +"""发布者的用户名""" + +get_config_api.desc: +"""查看配置内容""" + +list_retained_api.desc: +"""查看保留消息列表""" + +lookup_api.desc: +"""通过不带通配符的主题查看对应的保留消息""" + +message_detail.desc: +"""消息详情""" + +message_not_exist.desc: +"""消息不存在""" + +msgid.desc: +"""消息 ID""" + +payload.desc: +"""消息内容""" + +publish_at.desc: +"""消息发送时间, RFC 3339 格式""" + +qos.desc: +"""QoS""" + +retained_list.desc: +"""保留消息列表""" + +topic.desc: +"""主题""" + +unsupported_backend.desc: +"""不支持的后端""" + +update_config_failed.desc: +"""配置更新失败""" + +update_config_success.desc: +"""配置更新成功""" + +update_retainer_api.desc: +"""更新配置""" + +} diff --git a/rel/i18n/zh/emqx_retainer_schema.hocon b/rel/i18n/zh/emqx_retainer_schema.hocon new file mode 100644 index 000000000..1e8630007 --- /dev/null +++ b/rel/i18n/zh/emqx_retainer_schema.hocon @@ -0,0 +1,46 @@ +emqx_retainer_schema { + +backend.desc: +"""保留消息的存储后端""" + +batch_deliver_limiter.desc: +"""批量发送的限流器的名称。 +限流器可以用来防止短时间内向客户端发送太多的消息，从而避免过多的消息导致客户端队列堵塞甚至崩溃。 +这个名称需要是指向 `limiter.batch` 下的一个真实存在的限流器。 +如果这个字段为空，则不使用限流器。""" + +batch_deliver_number.desc: +"""批量派发时每批的数量。0 代表一次性全部派发""" + +batch_read_number.desc: +"""从存储后端批量加载时的每批数量上限，0 代表一次性读取""" + +enable.desc: +"""是否开启消息保留功能""" + +flow_control.desc: +"""流控设置""" + +max_payload_size.desc: +"""消息大小最大值""" + +max_retained_messages.desc: +"""消息保留的数量上限。0 表示无限""" + +mnesia_config_storage_type.desc: +"""选择消息是存放在磁盘还是内存中""" + +mnesia_config_type.desc: +"""后端类型""" + +msg_clear_interval.desc: +"""消息清理间隔。0 代表不进行清理""" + +msg_expiry_interval.desc: +"""消息保留时间。0 代表永久保留""" + +stop_publish_clear_msg.desc: +"""是否不发送保留消息的清理消息，在 MQTT 5.0 中如果一条保留消息的消息体为空，则会清除掉之前存储 +的对应的保留消息，通过这个值控制是否停止发送清理消息""" + +} diff --git a/rel/i18n/zh/emqx_rewrite_api.hocon b/rel/i18n/zh/emqx_rewrite_api.hocon new file mode 100644 index 000000000..2be95d38b --- /dev/null +++ b/rel/i18n/zh/emqx_rewrite_api.hocon @@ -0,0 +1,12 @@ +emqx_rewrite_api { + +list_topic_rewrite_api.desc: +"""列出全部主题重写规则""" + +update_topic_rewrite_api.desc: +"""更新全部主题重写规则""" + +update_topic_rewrite_api_response413.desc: +"""超出主题重写规则数量上限""" + +} diff --git a/rel/i18n/zh/emqx_rule_api_schema.hocon b/rel/i18n/zh/emqx_rule_api_schema.hocon new file mode 100644 index 000000000..854f7707f --- /dev/null +++ b/rel/i18n/zh/emqx_rule_api_schema.hocon @@ -0,0 +1,381 @@ +emqx_rule_api_schema { + +event_action.desc: +"""订阅或发布""" + +event_action.label: +"""订阅或发布""" + +event_payload.desc: +"""消息负载""" + +event_payload.label: +"""消息负载""" + +metrics_actions_failed_out_of_service.desc: +"""由于输出停止服务而导致规则调用输出失败的次数。 例如，桥接被禁用或停止。""" + +metrics_actions_failed_out_of_service.label: +"""调用输出失败次数""" + +metrics_actions_failed_unknown.desc: +"""由于未知错误，规则调用输出失败的次数。""" + +metrics_actions_failed_unknown.label: +"""调用输出失败次数""" + +event_server.desc: +"""MQTT broker的 IP 地址（或主机名）和端口，采用 IP:Port 格式""" + +event_server.label: +"""服务器 IP 地址和端口""" + +metrics_actions_total.desc: +"""规则调用输出的次数。 该值可能是“sql.matched”的几倍，具体取决于规则输出的数量。""" + +metrics_actions_total.label: +"""调用输出次数""" + +event_ctx_disconnected_da.desc: +"""客户端断开连接的时刻""" + +event_ctx_disconnected_da.label: +"""客户端断开连接时刻""" + +event_topic.desc: +"""消息主题""" + +event_topic.label: +"""消息主题""" + +event_peername.desc: +"""对等客户端的 IP 地址和端口""" + +event_peername.label: +"""IP 地址和端口""" + +metrics_sql_passed.desc: +"""SQL 通过的次数""" + +metrics_sql_passed.label: +"""SQL 通过""" + +test_context.desc: +"""测试事件的上下文""" + +test_context.label: +"""事件上下文""" + +node_node.desc: +"""节点名字""" + +node_node.label: +"""节点名字""" + +event_from_clientid.desc: +"""事件来源客户端的 ID""" + +event_from_clientid.label: +"""客户端 ID""" + +event_keepalive.desc: +"""保持连接""" + +event_keepalive.label: +"""保持连接""" + +event_connected_at.desc: +"""客户端连接完成时的时刻""" + +event_connected_at.label: +"""连接完成时的时刻""" + +metrics_sql_failed_exception.desc: +"""SQL 由于执行异常而失败的次数。 这可能是因为调用 SQL 函数时崩溃，或者试图对未定义的变量进行算术运算""" + +metrics_sql_failed_exception.label: +"""SQL 执行异常""" + +event_from_username.desc: +"""事件来源客户端的用户名""" + +event_from_username.label: +"""用户名""" + +event_ctx_connack_reason_code.desc: +"""错误码""" + +event_ctx_connack_reason_code.label: +"""错误码""" + +rs_description.desc: +"""描述""" + +rs_description.label: +"""描述""" + +rule_id.desc: +"""规则的 ID""" + +rule_id.label: +"""规则 ID""" + +rs_event.desc: +"""事件主题""" + +rs_event.label: +"""事件主题""" + +root_rule_engine.desc: +"""规则引擎配置。该 API 可用于查看和修改规则引擎相关的一些设置。但不可用于规则，如需查看或修改规则，请调用 '/rules' API 进行操作。""" + +root_rule_engine.label: +"""规则引擎配置""" + +event_sockname.desc: +"""本地监听的 IP 地址和端口""" + +event_sockname.label: +"""IP 地址和端口""" + +event_qos.desc: +"""消息的 QoS""" + +event_qos.label: +"""消息 QoS""" + +event_mountpoint.desc: +"""挂载点""" + +event_mountpoint.label: +"""挂载点""" + +rs_title.desc: +"""标题""" + +rs_title.label: +"""标题""" + +ri_metrics.desc: +"""规则的计数器""" + +ri_metrics.label: +"""规则计数器""" + +event_retain.desc: +"""是否是保留消息""" + +event_retain.label: +"""保留消息""" + +event_event_type.desc: +"""事件类型""" + +event_event_type.label: +"""事件类型""" + +event_expiry_interval.desc: +"""到期间隔""" + +event_expiry_interval.label: +"""到期间隔""" + +metrics_sql_matched.desc: +"""SQL 的 FROM 子句匹配的次数。""" + +metrics_sql_matched.label: +"""命中数""" + +event_clientid.desc: +"""客户端 ID""" + +event_clientid.label: +"""客户端 ID""" + +metrics_actions_success.desc: +"""规则成功调用输出的次数。""" + +metrics_actions_success.label: +"""成功调用输出次数""" + +metrics_actions_failed.desc: +"""规则调用输出失败的次数。""" + +metrics_actions_failed.label: +"""调用输出失败次数""" + +metrics_sql_matched_rate.desc: +"""命中速率，次/秒""" + +metrics_sql_matched_rate.label: +"""Matched Rate""" + +event_proto_ver.desc: +"""协议版本""" + +event_proto_ver.label: +"""协议版本""" + +event_publish_received_at.desc: +"""消息被接受的时间""" + +event_publish_received_at.label: +"""消息被接受的时间""" + +metrics_sql_matched_rate_last5m.desc: +"""5分钟平均命中速率，次/秒""" + +metrics_sql_matched_rate_last5m.label: +"""平均命中速率""" + +event_is_bridge.desc: +"""是否桥接""" + +event_is_bridge.label: +"""是否桥接""" + +event_authz_source.desc: +"""缓存，插件或者默认值""" + +event_authz_source.label: +"""认证源""" + +metrics_sql_failed_unknown.desc: +"""由于未知错误导致 SQL 失败的次数。""" + +metrics_sql_failed_unknown.label: +"""SQL 未知错误""" + +metrics_sql_failed.desc: +"""SQL 失败的次数""" + +metrics_sql_failed.label: +"""SQL 失败""" + +event_ctx_dropped.desc: +"""消息被丢弃的原因""" + +event_ctx_dropped.label: +"""丢弃原因""" + +root_rule_test.desc: +"""用于规则测试的 Schema""" + +root_rule_test.label: +"""用于规则测试的 Schema""" + +rs_test_columns.desc: +"""测试列""" + +rs_test_columns.label: +"""测试列""" + +event_peerhost.desc: +"""对等客户端的 IP 地址""" + +event_peerhost.label: +"""对等客户端的 IP""" + +event_proto_name.desc: +"""协议名称""" + +event_proto_name.label: +"""协议名称""" + +root_rule_events.desc: +"""用于事件的 Schema""" + +root_rule_events.label: +"""用于规则事件的 Schema""" + +rs_sql_example.desc: +"""SQL 例子""" + +rs_sql_example.label: +"""SQL 例子""" + +metrics_sql_matched_rate_max.desc: +"""最大命中速率，次/秒""" + +metrics_sql_matched_rate_max.label: +"""最大命中速率""" + +event_clean_start.desc: +"""清除会话""" + +event_clean_start.label: +"""清除会话""" + +ri_created_at.desc: +"""规则创建时间""" + +ri_created_at.label: +"""规则创建时间""" + +event_dup.desc: +"""MQTT 消息的 DUP 标志""" + +event_dup.label: +"""DUP 标志""" + +ri_from.desc: +"""规则指定的主题""" + +ri_from.label: +"""规则指定的主题""" + +ri_node_metrics.desc: +"""每个节点的规则计数器""" + +ri_node_metrics.label: +"""每个节点规则计数器""" + +root_rule_creation.desc: +"""用于创建规则的 Schema""" + +root_rule_creation.label: +"""用于创建规则的 Schema""" + +event_result.desc: +"""允许或禁止""" + +event_result.label: +"""认证结果""" + +event_id.desc: +"""消息 ID""" + +event_id.label: +"""消息 ID""" + +event_username.desc: +"""用户名""" + +event_username.label: +"""用户名""" + +root_rule_info.desc: +"""用于规则信息的 Schema""" + +root_rule_info.label: +"""用于规则信息的 Schema""" + +rs_columns.desc: +"""列""" + +rs_columns.label: +"""列""" + +test_sql.desc: +"""测试的 SQL""" + +test_sql.label: +"""测试 SQL""" + +event_ctx_disconnected_reason.desc: +"""断开连接的原因""" + +event_ctx_disconnected_reason.label: +"""断开连接原因""" + +} diff --git a/rel/i18n/zh/emqx_rule_engine_api.hocon b/rel/i18n/zh/emqx_rule_engine_api.hocon new file mode 100644 index 000000000..eb1be4e73 --- /dev/null +++ b/rel/i18n/zh/emqx_rule_engine_api.hocon @@ -0,0 +1,93 @@ +emqx_rule_engine_api { + +api1.desc: +"""列出所有规则""" + +api1.label: +"""列出所有规则""" + +api10.desc: +"""更新规则引擎配置。""" + +api10.label: +"""更新配置""" + +api1_enable.desc: +"""根据规则是否开启条件过滤""" + +api1_from.desc: +"""根据规则来源 Topic 过滤, 需要完全匹配""" + +api1_like_description.desc: +"""根据规则描述过滤, 使用子串模糊匹配""" + +api1_like_from.desc: +"""根据规则来源 Topic 过滤, 使用子串模糊匹配""" + +api1_like_id.desc: +"""根据规则 id 过滤, 使用子串模糊匹配""" + +api1_match_from.desc: +"""根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配""" + +api1_resp.desc: +"""规则列表""" + +api1_resp.label: +"""列出所有规则""" + +api2.desc: +"""通过指定 ID 创建规则""" + +api2.label: +"""通过指定 ID 创建规则""" + +api3.desc: +"""列出所有能被规则使用的事件""" + +api3.label: +"""列出所有能被规则使用的事件""" + +api4.desc: +"""通过 ID 查询规则""" + +api4.label: +"""查询规则""" + +api4_1.desc: +"""通过给定的 Id 获得规则的指标数据""" + +api4_1.label: +"""获得指标数据""" + +api5.desc: +"""通过 ID 更新集群里所有节点上的规则""" + +api5.label: +"""更新集群规则""" + +api6.desc: +"""通过 ID 删除集群里所有节点上的规则""" + +api6.label: +"""基于给定 ID 新建一条规则""" + +api7.desc: +"""重置规则计数""" + +api7.label: +"""重置规则计数""" + +api8.desc: +"""测试一个规则""" + +api8.label: +"""测试规则""" + +api9.desc: +"""获取规则引擎配置。""" + +api9.label: +"""获取配置""" + +} diff --git a/rel/i18n/zh/emqx_rule_engine_schema.hocon b/rel/i18n/zh/emqx_rule_engine_schema.hocon new file mode 100644 index 000000000..26858e10f --- /dev/null +++ b/rel/i18n/zh/emqx_rule_engine_schema.hocon @@ -0,0 +1,184 @@ +emqx_rule_engine_schema { + +console_function.desc: +"""将输出打印到控制台""" + +console_function.label: +"""控制台函数""" + +desc_builtin_action_console.desc: +"""配置打印到控制台""" + +desc_builtin_action_console.label: +"""配置打印到控制台""" + +desc_builtin_action_republish.desc: +"""配置重新发布。""" + +desc_builtin_action_republish.label: +"""配置重新发布""" + +desc_republish_args.desc: +"""内置 'republish' 动作的参数。 +可以在参数中使用变量。 +变量是规则中选择的字段。 例如规则 SQL 定义如下： + + SELECT clientid, qos, payload FROM "t/1" + +然后有 3 个变量可用：clientid、qos 和 payload。 如果我们将参数设置为： + + { + topic = "t/${clientid}" + qos = "${qos}" + payload = "msg: ${payload}" + } + +当收到一条消息 payload = `hello`, qos = 1, clientid = `Steve` 时，将重新发布一条新的 MQTT 消息到主题 `t/Steve` +消息内容为 payload = `msg: hello`, and `qos = 1""" + +desc_republish_args.label: +"""重新发布参数""" + +desc_rule_engine.desc: +"""配置 EMQX 规则引擎。""" + +desc_rule_engine.label: +"""配置规则引擎""" + +desc_rules.desc: +"""配置规则""" + +desc_rules.label: +"""配置规则""" + +desc_user_provided_function.desc: +"""配置用户函数""" + +desc_user_provided_function.label: +"""配置用户函数""" + +republish_args_payload.desc: +"""要重新发布的消息的有效负载。允许使用带有变量的模板，请参阅“republish_args”的描述。 +默认为 ${payload}。 如果从所选结果中未找到变量 ${payload}，则使用字符串 "undefined"。""" + +republish_args_payload.label: +"""消息负载""" + +republish_args_qos.desc: +"""要重新发布的消息的 qos。允许使用带有变量的模板，请参阅“republish_args”的描述。 +默认为 ${qos}。 如果从规则的选择结果中没有找到变量 ${qos}，则使用 0。""" + +republish_args_qos.label: +"""消息 QoS 等级""" + +republish_args_retain.desc: +"""要重新发布的消息的“保留”标志。允许使用带有变量的模板，请参阅“republish_args”的描述。 +默认为 ${retain}。 如果从所选结果中未找到变量 ${retain}，则使用 false。""" + +republish_args_retain.label: +"""保留消息标志""" + +republish_args_topic.desc: +"""重新发布消息的目标主题。 +允许使用带有变量的模板，请参阅“republish_args”的描述。""" + +republish_args_topic.label: +"""目标主题""" + +republish_args_user_properties.desc: +"""指定使用哪个变量来填充 MQTT 消息的 User-Property 列表。这个变量的值必须是一个 map 类型。 +可以设置成 ${pub_props.'User-Property'} 或者 +使用 SELECT *,pub_props.'User-Property' as user_properties 来把源 MQTT 消息 +的 User-Property 列表用于填充。 +也可以使用 map_put 函数来添加新的 User-Property， +map_put('my-prop-name', 'my-prop-value', user_properties) as user_properties +注意：MQTT 协议允许一个消息中出现多次同一个 property 名，但是 EMQX 的规则引擎不允许。""" + +republish_function.desc: +"""将消息重新发布为新的 MQTT 消息""" + +republish_function.label: +"""重新发布函数""" + +rule_engine_ignore_sys_message.desc: +"""当设置为“true”（默认）时，规则引擎将忽略发布到 $SYS 主题的消息。""" + +rule_engine_ignore_sys_message.label: +"""忽略系统消息""" + +rule_engine_jq_function_default_timeout.desc: +"""规则引擎内建函数 `jq` 默认时间限制""" + +rule_engine_jq_function_default_timeout.label: +"""规则引擎 jq 函数时间限制""" + +rule_engine_jq_implementation_module.desc: +"""jq 规则引擎功能的实现模块。可用的两个选项是 jq_nif 和 jq_port。jq_nif 使用 Erlang NIF 库访问 jq 库，而 jq_port 使用基于 Erlang Port 的实现。jq_nif 方式（默认选项）是这两个选项中最快的实现，但 jq_port 方式更安全，因为这种情况下 jq 程序不会在 Erlang VM 进程中执行。""" + +rule_engine_jq_implementation_module.label: +"""JQ 实现模块""" + +rule_engine_rules.desc: +"""规则""" + +rule_engine_rules.label: +"""规则""" + +rules_actions.desc: +"""规则的动作列表。 +动作可以是指向 EMQX bridge 的引用，也可以是一个指向函数的对象。 +我们支持一些内置函数，如“republish”和“console”，我们还支持用户提供的函数，它的格式为：“{module}:{function}”。 +列表中的动作按顺序执行。这意味着如果其中一个动作执行缓慢，则以下所有动作都不会被执行直到它返回。 +如果其中一个动作崩溃，在它之后的所有动作仍然会被按照原始顺序执行。 +如果运行动作时出现任何错误，则会出现错误消息，并且相应的计数器会增加。""" + +rules_actions.label: +"""动作列表""" + +rules_description.desc: +"""规则的描述""" + +rules_description.label: +"""规则描述""" + +rules_enable.desc: +"""启用或禁用规则引擎""" + +rules_enable.label: +"""启用或禁用规则引擎""" + +rules_metadata.desc: +"""规则的元数据，不要手动修改""" + +rules_metadata.label: +"""规则的元数据""" + +rules_name.desc: +"""规则名字""" + +rules_name.label: +"""规则名字""" + +rules_sql.desc: +"""用于处理消息的 SQL 。 +示例：SELECT * FROM "test/topic" WHERE payload.x = 1""" + +rules_sql.label: +"""规则 SQL""" + +user_provided_function_args.desc: +"""用户提供的参数将作为函数 module:function/3 的第三个参数， +请检查源文件：apps/emqx_rule_engine/src/emqx_rule_actions.erl 中的示例函数 console 和republish 。""" + +user_provided_function_args.label: +"""用户提供函数的参数""" + +user_provided_function_function.desc: +"""用户提供的函数。 格式应为：'{module}:{function}'。 +其中 {module} 是 Erlang 回调模块， {function} 是 Erlang 函数。 +要编写自己的函数，请检查源文件：apps/emqx_rule_engine/src/emqx_rule_actions.erl 中的示例函数 console 和republish 。""" + +user_provided_function_function.label: +"""用户提供的函数""" + +} diff --git a/rel/i18n/zh/emqx_schema.hocon b/rel/i18n/zh/emqx_schema.hocon new file mode 100644 index 000000000..3616abe91 --- /dev/null +++ b/rel/i18n/zh/emqx_schema.hocon @@ -0,0 +1,1473 @@ +emqx_schema { + +fields_mqtt_quic_listener_peer_unidi_stream_count.desc: +"""允许对端打开的单向流的数量""" + +fields_mqtt_quic_listener_peer_unidi_stream_count.label: +"""对端单向流的数量""" + +fields_authorization_no_match.desc: +"""如果用户或客户端不匹配ACL规则，或者从可配置授权源(比如内置数据库、HTTP API 或 PostgreSQL 等。)内未找 +到此类用户或客户端时，模式的认访问控制操作。 +在“授权”中查找更多详细信息。""" + +fields_authorization_no_match.label: +"""未匹时的默认授权动作""" + +sysmon_top_db_hostname.desc: +"""收集数据点的 PostgreSQL 数据库的主机名。""" + +sysmon_top_db_hostname.label: +"""数据库主机名""" + +zones.desc: +"""zone 是按name 分组的一组配置。 +对于灵活的配置映射，可以将 name 设置为侦听器的 zone 配置。 +注：名为 default 的内置区域是自动创建的，无法删除。""" + +fields_mqtt_quic_listener_certfile.desc: +"""证书文件。在 5.1 中会被废弃，使用 .ssl_options.certfile 代替。""" + +fields_mqtt_quic_listener_certfile.label: +"""证书文件""" + +fields_rate_limit_conn_bytes_in.desc: +"""限制 MQTT 连接接收数据包的速率。 速率以每秒的数据包字节数计算。""" + +fields_rate_limit_conn_bytes_in.label: +"""数据包速率""" + +crl_cache_capacity.desc: +"""缓存中可容纳的 CRL URL 的最大数量。 如果缓存的容量已满，并且必须获取一个新的 URL，那么它将驱逐缓存中插入的最老的 URL。""" + +crl_cache_capacity.label: +"""CRL 缓存容量""" + +alarm_actions.desc: +"""警报激活时触发的动作。
目前，支持以下操作：log 和 publish. +log 将告警写入日志 (控制台或者文件). +publish 将告警作为 MQTT 消息发布到系统主题: +$SYS/brokers/emqx@xx.xx.xx.x/alarms/activate and +$SYS/brokers/emqx@xx.xx.xx.x/alarms/deactivate""" + +alarm_actions.label: +"""告警动作""" + +base_listener_max_connections.desc: +"""监听器允许的最大并发连接数。""" + +base_listener_max_connections.label: +"""最大并发连接数""" + +mqtt_peer_cert_as_username.desc: +"""使用对端证书中的 CN、DN 字段或整个证书内容来作为用户名；仅适用于 TLS 连接。 +目前支持： +- cn: 取证书的 CN 字段 +- dn: 取证书的 DN 字段 +- crt: 取 DER 或 PEM 的证书内容 +- pem: 将 DER 证书转换为 PEM 格式作为用户名 +- md5: 取 DER 或 PEM 证书内容的 MD5 值""" + +mqtt_peer_cert_as_username.label: +"""对端证书作为用户名""" + +fields_cache_enable.desc: +"""启用或禁用授权缓存。""" + +fields_cache_enable.label: +"""启用或禁用授权缓存""" + +fields_mqtt_quic_listener_disconnect_timeout_ms.desc: +"""在判定路径无效和断开连接之前，要等待多长时间的ACK。默认：16000""" + +fields_mqtt_quic_listener_disconnect_timeout_ms.label: +"""断开连接超时 毫秒""" + +mqtt_max_topic_alias.desc: +"""允许的最大主题别名数，0 表示不支持主题别名。""" + +mqtt_max_topic_alias.label: +"""最大主题别名""" + +common_ssl_opts_schema_user_lookup_fun.desc: +"""用于查找预共享密钥（PSK）标识的 EMQX 内部回调。""" + +common_ssl_opts_schema_user_lookup_fun.label: +"""SSL PSK 用户回调""" + +fields_listeners_wss.desc: +"""HTTPS websocket 监听器。""" + +fields_listeners_wss.label: +"""HTTPS websocket 监听器""" + +sysmon_top_max_procs.desc: +"""当 VM 中的进程数超过此值时，停止收集数据。""" + +sysmon_top_max_procs.label: +"""最大进程数""" + +mqtt_use_username_as_clientid.desc: +"""是否使用用户名作为客户端 ID。 +此设置的作用时间晚于 对端证书作为用户名 和 对端证书作为客户端 ID。""" + +mqtt_use_username_as_clientid.label: +"""用户名作为客户端 ID""" + +mqtt_max_qos_allowed.desc: +"""允许的最大 QoS 等级。""" + +mqtt_max_qos_allowed.label: +"""最大 QoS""" + +fields_mqtt_quic_listener_max_binding_stateless_operations.desc: +"""在任何时候可以在一个绑定上排队的无状态操作的最大数量。默认值：100""" + +fields_mqtt_quic_listener_max_binding_stateless_operations.label: +"""最大绑定无状态操作""" + +fields_mqtt_quic_listener_stream_recv_buffer_default.desc: +"""流的初始缓冲区大小。默认：4096""" + +fields_mqtt_quic_listener_stream_recv_buffer_default.label: +"""流媒体接收缓冲区默认值""" + +fields_mqtt_quic_listener_pacing_enabled.desc: +"""有节奏的发送，以避免路径上的缓冲区过度填充。默认值：1（已启用）""" + +fields_mqtt_quic_listener_pacing_enabled.label: +"""启用节奏发送""" + +mqtt_max_subscriptions.desc: +"""允许每个客户端建立的最大订阅数量。""" + +mqtt_max_subscriptions.label: +"""最大订阅数量""" + +persistent_session_builtin_messages_table.desc: +"""用于内建消息表的性能调优参数。""" + +persistent_session_builtin_messages_table.label: +"""持久化消息""" + +sysmon_os_cpu_low_watermark.desc: +"""在解除相应警报之前可以使用多少系统 CPU 的阈值，以系统CPU负载的百分比表示。""" + +sysmon_os_cpu_low_watermark.label: +"""CPU 低水位线""" + +fields_mqtt_quic_listener_tls_server_max_send_buffer.desc: +"""缓冲多少TLS数据。 默认值：8192""" + +fields_mqtt_quic_listener_tls_server_max_send_buffer.label: +"""TLS 服务器最大发送缓冲区""" + +base_listener_bind.desc: +"""监听套接字的 IP 地址和端口。""" + +base_listener_bind.label: +"""IP 地址和端口""" + +server_ssl_opts_schema_handshake_timeout.desc: +"""握手完成所允许的最长时间""" + +server_ssl_opts_schema_handshake_timeout.label: +"""握手超时时间""" + +fields_deflate_opts_server_context_takeover.desc: +"""接管意味着在服务器消息之间保留压缩状态。""" + +fields_deflate_opts_server_context_takeover.label: +"""服务上下文接管""" + +mqtt_session_expiry_interval.desc: +"""指定会话将在连接断开后多久过期，仅适用于非 MQTT 5.0 的连接。""" + +mqtt_session_expiry_interval.label: +"""会话过期间隔""" + +fields_listener_enabled.desc: +"""启停监听器。""" + +fields_listener_enabled.label: +"""启停监听器""" + +mqtt.desc: +"""全局的 MQTT 配置项。 +mqtt 下所有的配置作为全局的默认值存在，它可以被 zone 中的配置覆盖。""" + +crl_cache_refresh_http_timeout.desc: +"""获取 CRLs 时 HTTP 请求的超时。 该配置对所有启用 CRL 检查的监听器监听器有效。""" + +crl_cache_refresh_http_timeout.label: +"""CRL 缓存刷新 HTTP 超时""" + +fields_tcp_opts_backlog.desc: +"""TCP backlog 定义了挂起连接队列可以增长到的最大长度。""" + +fields_tcp_opts_backlog.label: +"""TCP 连接队列长度""" + +broker_route_batch_clean.desc: +"""是否开启批量清除路由。""" + +fields_mqtt_quic_listener_initial_window_packets.desc: +"""一个连接的初始拥堵窗口的大小（以包为单位）。默认值：10""" + +fields_mqtt_quic_listener_initial_window_packets.label: +"""初始窗口数据包""" + +flapping_detect_enable.desc: +"""启用抖动检测功能。""" + +flapping_detect_enable.label: +"""启用抖动检测""" + +sysmon_top_db_password.desc: +"""PostgreSQL 数据库的密码""" + +sysmon_top_db_password.label: +"""数据库密码""" + +fields_ws_opts_check_origins.desc: +"""允许的 origins 列表""" + +fields_ws_opts_check_origins.label: +"""允许的 origins""" + +fields_deflate_opts_client_context_takeover.desc: +"""接管意味着在客户端消息之间保留压缩状态。""" + +fields_deflate_opts_client_context_takeover.label: +"""客户端上下文接管""" + +base_listener_acceptors.desc: +"""监听器接收池的大小。""" + +base_listener_acceptors.label: +"""接收器数量""" + +common_ssl_opts_schema_cacertfile.desc: +"""受信任的PEM格式 CA 证书捆绑文件
+此文件中的证书用于验证TLS对等方的证书。 +如果要信任新 CA，请将新证书附加到文件中。 +无需重启EMQX即可加载更新的文件，因为系统会定期检查文件是否已更新（并重新加载）
+注意：从文件中失效（删除）证书不会影响已建立的连接。""" + +common_ssl_opts_schema_cacertfile.label: +"""CA 证书文件""" + +fields_ws_opts_mqtt_path.desc: +"""WebSocket 的 MQTT 协议路径。因此，EMQX Broker的WebSocket地址为： +ws://{ip}:{port}/mqtt""" + +fields_ws_opts_mqtt_path.label: +"""WS MQTT 路径""" + +sysmon_os_procmem_high_watermark.desc: +"""在发出相应警报之前，一个Erlang进程可以分配多少系统内存的阈值，以系统内存的百分比表示。""" + +sysmon_os_procmem_high_watermark.label: +"""进程内存高水位线""" + +fields_listeners_quic.desc: +"""QUIC 监听器。""" + +fields_listeners_quic.label: +"""QUIC 监听器""" + +fields_listeners_ws.desc: +"""HTTP websocket 监听器。""" + +fields_listeners_ws.label: +"""HTTP websocket 监听器""" + +mqtt_retry_interval.desc: +"""QoS 1/2 消息的重新投递间隔。""" + +mqtt_retry_interval.label: +"""重试间隔""" + +stats_enable.desc: +"""启用/禁用统计数据收集功能。""" + +stats_enable.label: +"""启用/禁用统计数据收集功能""" + +fields_authorization_deny_action.desc: +"""授权检查拒绝操作时的操作。""" + +fields_authorization_deny_action.label: +"""授权检查拒绝操作时的操作""" + +fields_deflate_opts_server_max_window_bits.desc: +"""指定服务器压缩上下文的大小。""" + +fields_deflate_opts_server_max_window_bits.label: +"""服务器压缩窗口大小""" + +client_ssl_opts_schema_server_name_indication.desc: +"""指定要在 TLS 服务器名称指示扩展中使用的主机名。
+例如，当连接到 "server.example.net" 时，接受连接并执行 TLS 握手的真正服务器可能与 TLS 客户端最初连接到的主机不同， +例如，当连接到 IP 地址时，或者当主机具有多个可解析的 DNS 记录时
+如果未指定，它将默认为使用的主机名字符串 +建立连接，除非使用 IP 地址
+然后，主机名也用于对等机的主机名验证证书
+特殊值 disable 阻止发送服务器名称指示扩展，并禁用主机名验证检查。""" + +client_ssl_opts_schema_server_name_indication.label: +"""服务器名称指示""" + +fields_mqtt_quic_listener_retry_memory_limit.desc: +"""在使用无状态重试之前，可用于握手连接的可用内存的百分比。计算为`N/65535`。默认值：65""" + +fields_mqtt_quic_listener_retry_memory_limit.label: +"""重试内存限制""" + +force_shutdown_max_message_queue_len.desc: +"""消息队列的最大长度。""" + +force_shutdown_max_message_queue_len.label: +"""进程邮箱消息队列的最大长度""" + +sys_heartbeat_interval.desc: +"""发送心跳系统消息的间隔时间，它包括： + - `$SYS/brokers//uptime` + - `$SYS/brokers//datetime`""" + +flapping_detect_ban_time.desc: +"""抖动的客户端将会被禁止登录多长时间。""" + +flapping_detect_ban_time.label: +"""禁止登录时长""" + +sysmon_top_num_items.desc: +"""每个监视组的顶级进程数。""" + +sysmon_top_num_items.label: +"""顶级进程数""" + +persistent_session_builtin_session_table.desc: +"""用于内建会话表的性能调优参数。""" + +persistent_session_builtin_session_table.label: +"""持久会话""" + +mqtt_upgrade_qos.desc: +"""投递消息时，是否根据订阅主题时的 QoS 等级来强制提升派发的消息的 QoS 等级。""" + +mqtt_upgrade_qos.label: +"""升级 QoS""" + +mqtt_shared_subscription.desc: +"""是否启用对 MQTT 共享订阅的支持。""" + +mqtt_shared_subscription.label: +"""共享订阅可用""" + +fields_tcp_opts_sndbuf.desc: +"""连接的 TCP 发送缓冲区（OS 内核）。""" + +fields_tcp_opts_sndbuf.label: +"""TCP 发送缓冲区""" + +sysmon_os_mem_check_interval.desc: +"""定期内存检查的时间间隔。""" + +sysmon_os_mem_check_interval.label: +"""内存检查间隔""" + +server_ssl_opts_schema_gc_after_handshake.desc: +"""内存使用调优。如果启用，将在TLS/SSL握手完成后立即执行垃圾回收。TLS/SSL握手建立后立即进行GC。""" + +server_ssl_opts_schema_gc_after_handshake.label: +"""握手后执行GC""" + +fields_mqtt_quic_listener_ssl_options.desc: +"""QUIC 传输层的 TLS 选项""" + +fields_mqtt_quic_listener_ssl_options.label: +"""TLS 选项""" + +fields_ws_opts_mqtt_piggyback.desc: +"""WebSocket消息是否允许包含多个 MQTT 数据包。""" + +fields_ws_opts_mqtt_piggyback.label: +"""MQTT Piggyback""" + +base_listener_mountpoint.desc: +"""发布或订阅时，请在所有主题前面加上 mountpoint 字符串。 + +将消息传递给订阅者时，将从主题名称中删除带前缀的字符串。挂载点是一种用户可以用来实现不同侦听器之间消息路由隔离的方法。 + +例如，如果客户机 A 使用 listeners.tcp.\.mountpoint 设置为'some_tenant'，那么客户端实际上订阅了主题'some_tenant/t'。
+类似地，如果另一个客户端B（与客户端A连接到同一个侦听器）向主题 't' 发送消息，该消息将路由到所有订阅了'some_租户/t'的客户端，因此客户端 A 将接收主题名为't'的消息
+ +设置为"" 以禁用该功能
+ +mountpoint 字符串中的变量： +- ${clientid}: clientid +- ${username}: username""" + +base_listener_mountpoint.label: +"""mountpoint""" + +mqtt_max_awaiting_rel.desc: +"""每个发布者的会话中，都存在一个队列来处理客户端发送的 QoS 2 消息。该队列会存储 QoS 2 消息的报文 ID 直到收到客户端的 PUBREL 或超时，达到队列长度的限制后，新的 QoS 2 消息发布会被拒绝，并返回 `147(0x93)` 错误。""" + +mqtt_max_awaiting_rel.label: +"""PUBREL 等待队列长度""" + +ciphers_schema_quic.desc: +"""此配置保存由逗号分隔的 TLS 密码套件名称，或作为字符串数组。例如 +"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256"或 +["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]。 +
+密码（及其顺序）定义了客户端和服务器通过网络连接加密信息的方式。 +选择一个好的密码套件对于应用程序的数据安全性、机密性和性能至关重要。 + +名称应为 OpenSSL 字符串格式（而不是 RFC 格式）。 +EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
+注意：某些密码套件仅与特定的 TLS 版本兼容（'tlsv1.1'、'tlsv1.2'或'tlsv1.3'）。 +不兼容的密码套件将被自动删除。 + +例如，如果只有 versions 仅配置为 tlsv1.3。为其他版本配置密码套件将无效。 + +
+注：PSK 的 Ciphers 不支持 tlsv1.3
+如果打算使用PSK密码套件，tlsv1.3。应在ssl.versions中禁用。 + +
+PSK 密码套件： +"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, +RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, +RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, +RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"
+ +注：QUIC 监听器不支持 tlsv1.3 的 ciphers""" + +ciphers_schema_quic.label: +"""""" + +fields_mqtt_quic_listener_max_bytes_per_key.desc: +"""在启动密钥更新之前，用单个 1-RTT 加密密钥加密的最大字节数。默认值：274877906944""" + +fields_mqtt_quic_listener_max_bytes_per_key.label: +"""每个密钥的最大字节数""" + +fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us.desc: +"""如果没有达到 max ，在重新尝试 MTU 探测之前要等待的时间，单位是微秒。默认值：600000000""" + +fields_mqtt_quic_listener_mtu_discovery_search_complete_timeout_us.label: +"""""" + +fields_ws_opts_check_origin_enable.desc: +"""如果true，originHTTP 头将根据check_origins参数中配置的允许来源列表进行验证。""" + +fields_ws_opts_check_origin_enable.label: +"""检查 origin""" + +sysmon_vm_busy_dist_port.desc: +"""启用后，当用于集群接点之间 RPC 的连接过忙时，会触发一条带有 busy_dist_port 关键字的 warning 级别日志。 +同时还会发布一条主题为 $SYS/sysmon/busy_dist_port 的 MQTT 系统消息。""" + +sysmon_vm_busy_dist_port.label: +"""启用分布式端口过忙监控""" + +mqtt_max_mqueue_len.desc: +"""消息队列最大长度。持久客户端断开连接或飞行窗口已满时排队的消息长度。""" + +mqtt_max_mqueue_len.label: +"""最大消息队列长度""" + +mqtt_max_inflight.desc: +"""允许在完成应答前同时投递的 QoS 1 和 QoS 2 消息的最大数量。""" + +mqtt_max_inflight.label: +"""最大飞行窗口""" + +persistent_session_store_enabled.desc: +"""使用数据库存储有关持久会话的信息。 +这使得在节点停止时，可以将客户端连接迁移到另一个群集节点。""" + +persistent_session_store_enabled.label: +"""启用持久会话保存""" + +fields_deflate_opts_level.desc: +"""压缩级别""" + +fields_deflate_opts_level.label: +"""压缩级别""" + +mqtt_server_keepalive.desc: +"""EMQX 要求的保活时间，如设为 disabled，则将使用客户端指定的保持连接时间；仅适用于 MQTT 5.0 客户端。""" + +mqtt_server_keepalive.label: +"""服务端保活时间""" + +global_authentication.desc: +"""全局 MQTT 监听器的默认认证配置。 为每个监听器配置认证参考监听器器配置中的authentication 配置。 + +该配置可以被配置为： +
    +
  • []: 默认值，允许所有的登录请求 +
  • 配置为单认证器，例如 {enable:true,backend:"built_in_database",mechanism="password_based"}
    • +
  • 配置为认证器数组
    • +
+ +当配置为认证链后，登录凭证会按照配置的顺序进行检查，直到做出allow 或 deny的结果。 + +如果在所有的认证器都执行完后，还是没有结果，登录将被拒绝。""" + +fields_mqtt_quic_listener_load_balancing_mode.desc: +"""0: 禁用, 1: SERVER_ID_IP, 2: SERVER_ID_FIXED. 默认: 0""" + +fields_mqtt_quic_listener_load_balancing_mode.label: +"""负载平衡模式""" + +persistent_session_store_session_message_gc_interval.desc: +"""持久会话消息的临时数据垃圾收集的开始间隔。 +这不会影响持久会话消息的生命周期长度。""" + +persistent_session_store_session_message_gc_interval.label: +"""会话消息清理间隔""" + +server_ssl_opts_schema_ocsp_refresh_http_timeout.desc: +"""检查 OCSP 响应时，HTTP 请求的超时。""" + +server_ssl_opts_schema_ocsp_refresh_http_timeout.label: +"""OCSP 刷新 HTTP 超时""" + +fields_tcp_opts_send_timeout.desc: +"""连接的 TCP 发送超时。""" + +fields_tcp_opts_send_timeout.label: +"""TCP 发送超时""" + +sysmon_vm_process_high_watermark.desc: +"""在发出相应警报之前，本地节点上可以同时存在多少进程的阈值（以进程百分比表示）。""" + +sysmon_vm_process_high_watermark.label: +"""进程数高水位线""" + +fields_tcp_opts_buffer.desc: +"""驱动程序使用的用户空间缓冲区的大小。""" + +fields_tcp_opts_buffer.label: +"""TCP 用户态缓冲区""" + +server_ssl_opts_schema_honor_cipher_order.desc: +"""一个重要的安全设置，它强制根据服务器指定的顺序而不是客户机指定的顺序设置密码，从而强制服务器管理员执行（通常配置得更正确）安全顺序。""" + +server_ssl_opts_schema_honor_cipher_order.label: +"""SSL honor cipher order""" + +conn_congestion_min_alarm_sustain_duration.desc: +"""清除警报前的最短时间。
只有当队列中没有挂起的数据，并且连接至少被堵塞了 min_alarm_sustain_duration 毫秒时，
报警才会被清除。这是为了避免太频繁地清除和再次发出警报。""" + +conn_congestion_min_alarm_sustain_duration.label: +"""告警维持时间""" + +fields_mqtt_quic_listener_keep_alive_interval_ms.desc: +"""多长时间发送一次PING帧以保活连接。""" + +fields_mqtt_quic_listener_keep_alive_interval_ms.label: +"""保持活着的时间间隔 毫秒""" + +fields_mqtt_quic_listener_handshake_idle_timeout_ms.desc: +"""一个握手在被丢弃之前可以空闲多长时间""" + +fields_mqtt_quic_listener_handshake_idle_timeout_ms.label: +"""握手空闲超时 毫秒""" + +broker_session_locking_strategy.desc: +"""Session 在集群中的锁策略。 + - `loca`：仅锁本节点的 Session； + - `one`：任选一个其它节点加锁； + - `quorum`：选择集群中半数以上的节点加锁； + - `all`：选择所有节点加锁。""" + +persistent_store_ram_cache.desc: +"""在内存中保持一份数据的副本，以便更快地访问。""" + +persistent_store_ram_cache.label: +"""内存缓存""" + +fields_mqtt_quic_listener_stream_recv_window_default.desc: +"""初始流接收窗口大小。 默认值：32678""" + +fields_mqtt_quic_listener_stream_recv_window_default.label: +"""流接收窗口默认""" + +mqtt_mqueue_priorities.desc: +"""主题优先级。取值范围 [1-255] +默认优先级表为空，即所有的主题优先级相同。 + +注：优先主题名称中不支持使用逗号和等号。 +注：不在此列表中的主题，被视为最高/最低优先级，这取决于mqtt.mqueue_default_priority 的配置 + +示例： +配置 "topic/1" > "topic/2": +mqueue_priorities: {"topic/1": 10, "topic/2": 8}""" + +mqtt_mqueue_priorities.label: +"""主题优先级""" + +fields_rate_limit_conn_messages_in.desc: +"""外部 MQTT 连接的消息限制。""" + +fields_rate_limit_conn_messages_in.label: +"""外部 MQTT 连接的消息限制""" + +fields_rate_limit_max_conn_rate.desc: +"""每秒最大连接数。""" + +fields_rate_limit_max_conn_rate.label: +"""每秒最大连接数""" + +alarm_size_limit.desc: +"""要保留为历史记录的已停用报警的最大总数。当超过此限制时，将删除最旧的停用报警，以限制总数。""" + +alarm_size_limit.label: +"""告警总数限制""" + +fields_cache_max_size.desc: +"""缓存项的最大数量。""" + +fields_cache_max_size.label: +"""缓存项的最大数量""" + +fields_listeners_tcp.desc: +"""TCP 监听器。""" + +fields_listeners_tcp.label: +"""TCP 监听器""" + +conn_congestion_enable_alarm.desc: +"""启用或者禁用连接阻塞告警功能。""" + +conn_congestion_enable_alarm.label: +"""启用/禁用阻塞告警""" + +fields_ws_opts_proxy_port_header.desc: +"""HTTP 头，用于传递有关客户端端口的信息。当 EMQX 集群部署在负载平衡器后面时，这一点非常重要。""" + +fields_ws_opts_proxy_port_header.label: +"""客户端端口头""" + +overload_protection_enable.desc: +"""是否对系统过载做出反应。""" + +overload_protection_enable.label: +"""是否对系统过载做出反应""" + +fields_mqtt_quic_listener_minimum_mtu.desc: +"""一个连接所支持的最小MTU。这将被作为起始MTU使用。默认值：1248""" + +fields_mqtt_quic_listener_minimum_mtu.label: +"""最小 MTU""" + +sys_msg_interval.desc: +"""发送 `$SYS` 主题的间隔时间。""" + +mqtt_await_rel_timeout.desc: +"""客户端发布 QoS 2 消息时，服务器等待 `PUBREL` 的最长时延。超过该时长后服务器会放弃等待，该PACKET ID 会被释放，从而允许后续新的 PUBLISH 消息使用。如果超时后收到 PUBREL，服务器将会产生一条告警日志。注意，向订阅客户端转发消息的动作发生在进入等待之前。""" + +mqtt_await_rel_timeout.label: +"""PUBREL 最大等待时间""" + +common_ssl_opts_schema_verify.desc: +"""启用或禁用对等验证。""" + +common_ssl_opts_schema_verify.label: +"""对等验证""" + +fields_listeners_ssl.desc: +"""SSL 监听器。""" + +fields_listeners_ssl.label: +"""SSL 监听器""" + +fields_deflate_opts_client_max_window_bits.desc: +"""指定客户端压缩上下文的大小。""" + +fields_deflate_opts_client_max_window_bits.label: +"""压缩窗口大小""" + +common_ssl_opts_schema_keyfile.desc: +"""PEM格式的私钥文件。""" + +common_ssl_opts_schema_keyfile.label: +"""私钥文件""" + +sysmon_os_cpu_high_watermark.desc: +"""在发出相应警报之前可以使用多少系统 CPU 的阈值，以系统CPU负载的百分比表示。""" + +sysmon_os_cpu_high_watermark.label: +"""CPU 高水位线""" + +flapping_detect_window_time.desc: +"""抖动检测的时间窗口。""" + +flapping_detect_window_time.label: +"""时间窗口""" + +mqtt_mqueue_default_priority.desc: +"""默认的主题优先级，不在 主题优先级（mqueue_priorities） 中的主题将会使用该优先级。""" + +mqtt_mqueue_default_priority.label: +"""默认主题优先级""" + +client_ssl_opts_schema_enable.desc: +"""启用 TLS。""" + +client_ssl_opts_schema_enable.label: +"""启用 TLS""" + +fields_mqtt_quic_listener_mtu_discovery_missing_probe_count.desc: +"""在任何时候都可以在一个绑定上排队的无状态操作的最大数量。默认值：3""" + +fields_mqtt_quic_listener_mtu_discovery_missing_probe_count.label: +"""MTU发现丢失的探针数量""" + +fields_tcp_opts_recbuf.desc: +"""连接的 TCP 接收缓冲区（OS 内核）。""" + +fields_tcp_opts_recbuf.label: +"""TCP 接收缓冲区""" + +sysmon_vm_process_check_interval.desc: +"""定期进程限制检查的时间间隔。""" + +sysmon_vm_process_check_interval.label: +"""进程限制检查时间""" + +fields_mqtt_quic_listener_server_resumption_level.desc: +"""连接恢复 和/或 0-RTT 服务器支持。默认值：0（无恢复功能）""" + +fields_mqtt_quic_listener_server_resumption_level.label: +"""服务端连接恢复支持""" + +fields_ws_opts_proxy_address_header.desc: +"""HTTP 头，用于传递有关客户端 IP 地址的信息。 +当 EMQX 集群部署在负载平衡器后面时，这一点非常重要。""" + +fields_ws_opts_proxy_address_header.label: +"""客户端地址头""" + +sysmon_os_sysmem_high_watermark.desc: +"""在发出相应报警之前可以分配多少系统内存的阈值，以系统内存的百分比表示。""" + +sysmon_os_sysmem_high_watermark.label: +"""系统内存高水位线""" + +fields_tcp_opts_high_watermark.desc: +"""当 VM 套接字实现内部排队的数据量达到此限制时，套接字将设置为忙碌状态。""" + +fields_tcp_opts_high_watermark.label: +"""""" + +fields_mqtt_quic_listener_stateless_operation_expiration_ms.desc: +"""同一个对端的操作之间的时间限制，单位是毫秒。 默认：100""" + +fields_mqtt_quic_listener_stateless_operation_expiration_ms.label: +"""无状态操作过期 毫秒""" + +server_ssl_opts_schema_dhfile.desc: +"""如果协商使用Diffie-Hellman密钥交换的密码套件，则服务器将使用包含PEM编码的Diffie-Hellman参数的文件的路径。如果未指定，则使用默认参数。
+注意：TLS 1.3不支持dhfile选项。""" + +server_ssl_opts_schema_dhfile.label: +"""SSL dhfile""" + +flapping_detect_max_count.desc: +"""MQTT 客户端在“窗口”时间内允许的最大断开次数。""" + +flapping_detect_max_count.label: +"""最大断开次数""" + +mqtt_max_topic_levels.desc: +"""允许的最大主题层级。""" + +mqtt_max_topic_levels.label: +"""最大主题层级""" + +force_shutdown_max_heap_size.desc: +"""Heap 的总大小。""" + +force_shutdown_max_heap_size.label: +"""Heap 的总大小""" + +persistent_store_on_disc.desc: +"""将持久会话数据保存在磁盘上。如果为 false 则存储在内存中。 +如开启， 持久会话数据可在集群重启后恢复。 +如关闭， 数据仅存储在内存中， 则在整个集群停止后丢失。""" + +persistent_store_on_disc.label: +"""持久化在磁盘上""" + +mqtt_ignore_loop_deliver.desc: +"""设置由 MQTT v3.1.1/v3.1.0 客户端发布的消息是否将转发给其本身；类似 MQTT 5.0 协议中的 No Local 选项。""" + +mqtt_ignore_loop_deliver.label: +"""忽略循环投递""" + +common_ssl_opts_schema_certfile.desc: +"""PEM格式证书链文件
+此文件中的证书应与证书颁发链的顺序相反。也就是说，主机的证书应该放在文件的开头， +然后是直接颁发者 CA 证书，依此类推，一直到根 CA 证书。 +根 CA 证书是可选的，如果想要添加，应加到文件到最末端。""" + +common_ssl_opts_schema_certfile.label: +"""证书文件""" + +mqtt_exclusive_subscription.desc: +"""是否启用对 MQTT 排它订阅的支持。""" + +mqtt_exclusive_subscription.label: +"""排它订阅""" + +mqtt_retain_available.desc: +"""是否启用对 MQTT 保留消息的支持。""" + +mqtt_retain_available.label: +"""保留消息可用""" + +fields_tcp_opts_reuseaddr.desc: +"""连接的 SO_REUSEADDR 标识。""" + +fields_tcp_opts_reuseaddr.label: +"""SO_REUSEADDR""" + +sysmon_vm_long_schedule.desc: +"""启用后，如果 Erlang VM 调度器出现某个任务占用时间过长时，会触发一条带有 'long_schedule' 关键字的日志。 +同时还会发布一条主题为 $SYS/sysmon/long_schedule 的 MQTT 系统消息。""" + +sysmon_vm_long_schedule.label: +"""启用长调度监控""" + +mqtt_keepalive_backoff.desc: +"""EMQX 判定客户端保活超时使用的阈值系数。计算公式为：Keep Alive * Backoff * 2""" + +mqtt_keepalive_backoff.label: +"""保活超时阈值系数""" + +force_gc_bytes.desc: +"""在进程处理过多少个字节之后，对此进程执行垃圾回收。""" + +force_gc_bytes.label: +"""垃圾回收字节数""" + +server_ssl_opts_schema_fail_if_no_peer_cert.desc: +"""TLS/DTLS 服务器与 {verify，verify_peer} 一起使用。 +如果设置为true，则如果客户端没有要发送的证书，即发送空证书，服务器将失败。 +如果设置为false，则仅当客户端发送无效证书（空证书被视为有效证书）时才会失败。""" + +server_ssl_opts_schema_fail_if_no_peer_cert.label: +"""没有证书则 SSL 失败""" + +fields_ws_opts_compress.desc: +"""如果 true，则使用zlib 压缩 WebSocket 消息
+deflate_opts 下的配置项属于压缩相关参数配置。""" + +fields_ws_opts_compress.label: +"""Ws 压缩""" + +fields_mqtt_quic_listener_keep_alive_interval.desc: +"""发送 PING 帧的频率，以保活连接. 设为 0 表示禁用。""" + +fields_mqtt_quic_listener_keep_alive_interval.label: +"""PING 保活频率""" + +fields_cache_ttl.desc: +"""缓存数据的生存时间。""" + +fields_cache_ttl.label: +"""缓存数据的生存时间。""" + +sys_topics.desc: +"""系统主题配置。""" + +sys_event_client_subscribed.desc: +"""是否开启客户端已成功订阅主题事件消息。""" + +sysmon_top_db_port.desc: +"""收集数据点的 PostgreSQL 数据库的端口。""" + +sysmon_top_db_port.label: +"""数据库端口""" + +fields_mqtt_quic_listener_max_operations_per_drain.desc: +"""每个连接操作的最大耗费操作数。默认：16""" + +fields_mqtt_quic_listener_max_operations_per_drain.label: +"""每次操作最大操作数""" + +fields_mqtt_quic_listener_datagram_receive_enabled.desc: +"""宣传对QUIC Datagram 扩展的支持。为将来保留。默认为0（FALSE）""" + +fields_mqtt_quic_listener_datagram_receive_enabled.label: +"""启用 Datagram 接收""" + +fields_mqtt_quic_listener_initial_rtt_ms.desc: +"""初始RTT估计""" + +fields_mqtt_quic_listener_initial_rtt_ms.label: +"""Initial RTT 毫秒""" + +overload_protection_backoff_gc.desc: +"""高负载时，跳过强制 GC。""" + +overload_protection_backoff_gc.label: +"""跳过GC""" + +broker_perf_route_lock_type.desc: +"""通配主题订阅/取消订阅性能调优。 +建议仅当通配符主题较多时才更改此参数。 + +注：当从/更改为 `global` 锁时，它要求集群中的所有节点在更改之前停止。 + - `key`：为 Mnesia 事务涉及到的每个 key 上锁，建议单节点时使用。 + - `tab`：为 Mnesia 事务涉及到的表上锁，建议在集群中使用。 + - `global`：所以更新操作都被全局的锁保护，仅建议在超大规模集群中使用。""" + +fields_tcp_opts_nodelay.desc: +"""连接的 TCP_NODELAY 标识""" + +fields_tcp_opts_nodelay.label: +"""TCP_NODELAY""" + +sysmon_top_db_username.desc: +"""PostgreSQL 数据库的用户名""" + +sysmon_top_db_username.label: +"""数据库用户名""" + +broker.desc: +"""Broker 相关配置项。""" + +force_gc_count.desc: +"""在进程收到多少消息之后，对此进程执行垃圾回收。""" + +force_gc_count.label: +"""垃圾回收消息数""" + +mqtt_max_clientid_len.desc: +"""允许的最大 MQTT Client ID 长度。""" + +mqtt_max_clientid_len.label: +"""最大 Client ID 长度""" + +fields_ws_opts_supported_subprotocols.desc: +"""逗号分隔的 subprotocols 支持列表。""" + +fields_ws_opts_supported_subprotocols.label: +"""Subprotocols 支持列表""" + +broker_shared_subscription_strategy.desc: +"""共享订阅消息派发策略。 + - `random`：随机挑选一个共享订阅者派发； + - `round_robin`：使用 round-robin 策略派发； + - `round_robin_per_group`：在共享组内循环选择下一个成员； + - `local`：选择随机的本地成员，否则选择随机的集群范围内成员; + - `sticky`：总是使用上次选中的订阅者派发，直到它断开连接； + - `hash_clientid`：通过对发送者的客户端 ID 进行 Hash 处理来选择订阅者; + - `hash_topic`：通过对源主题进行 Hash 处理来选择订阅者。""" + +fields_deflate_opts_mem_level.desc: +"""指定压缩状态的大小
+较低的值会减少每个连接的内存使用。""" + +fields_deflate_opts_mem_level.label: +"""压缩状态大小""" + +fields_mqtt_quic_listener_send_idle_timeout_ms.desc: +"""在闲置一定时间后重置拥堵控制。默认值：1000""" + +fields_mqtt_quic_listener_send_idle_timeout_ms.label: +"""发送空闲超时毫秒""" + +base_listener_limiter.desc: +"""速率限制类型""" + +base_listener_limiter.label: +"""速率限制类型""" + +persistent_session_store_backend.desc: +"""用于存储持久性会话和信息的数据库管理后端 +- `builtin`: 使用内置的数据库（mria）""" + +persistent_session_store_backend.label: +"""后端类型""" + +alarm_validity_period.desc: +"""停用报警的保留时间。报警在停用时不会立即删除，而是在保留时间之后删除。""" + +alarm_validity_period.label: +"""告警保留时间""" + +server_ssl_opts_schema_ocsp_issuer_pem.desc: +"""服务器证书的 OCSP 签发者的 PEM 编码证书。""" + +server_ssl_opts_schema_ocsp_issuer_pem.label: +"""OCSP 签发者证书""" + +fields_tcp_opts_active_n.desc: +"""为此套接字指定{active，N}选项
+See: https://erlang.org/doc/man/inet.html#setopts-2""" + +fields_tcp_opts_active_n.label: +"""active_n""" + +listener_authentication.desc: +"""监听器认证重载。 +认证配置可以是单个认证器实例，也可以是一个认证器数组组成的认证链。 +执行登录验证时（用户名、客户端 ID 等），将按配置的顺序执行。""" + +listener_authentication.label: +"""每个监听器的认证覆盖""" + +fields_trace_payload_encode.desc: +"""确定跟踪文件中有效负载格式的格式。
+`text`：基于文本的协议或纯文本协议。 +建议在有效负载为JSON编码时使用
+`hex`：二进制十六进制编码。当有效负载是自定义二进制协议时，建议使用此选项
+`hidden`：有效负载被模糊化为 `******`""" + +fields_trace_payload_encode.label: +"""有效负载编码""" + +mqtt_response_information.desc: +"""UTF-8 字符串，用于指定返回给客户端的响应主题，如 reqrsp/，此时请求和应答客户端都需要使用 reqrsp/ 前缀的主题来完成通讯。 +如希望禁用此功能，请在下方的文字框中输入""；仅适用于 MQTT 5.0 客户端。""" + +mqtt_response_information.label: +"""响应信息""" + +persistent_session_store_max_retain_undelivered.desc: +"""如果重新启动时处理上一个会话的节点已停止，则未传递到持久会话的消息在垃圾收集之前会被存储。""" + +persistent_session_store_max_retain_undelivered.label: +"""未投递的消息保留条数""" + +fields_mqtt_quic_listener_migration_enabled.desc: +"""开启客户端地址迁移功能。需要一个支持的负载平衡器，或者没有负载平衡器。默认值：1（已启用）""" + +fields_mqtt_quic_listener_migration_enabled.label: +"""启用地址迁移""" + +common_ssl_opts_schema_password.desc: +"""包含用户密码的字符串。仅在私钥文件受密码保护时使用。""" + +common_ssl_opts_schema_password.label: +"""秘钥文件密码""" + +common_ssl_opts_schema_hibernate_after.desc: +"""在闲置一定时间后休眠 SSL 进程，减少其内存占用。""" + +common_ssl_opts_schema_hibernate_after.label: +"""闲置多久后休眠""" + +fields_mqtt_quic_listener_send_buffering_enabled.desc: +"""缓冲发送数据，而不是保留应用缓冲区，直到发送数据被确认。默认值：1（启用）""" + +fields_mqtt_quic_listener_send_buffering_enabled.label: +"""启用发送缓冲功能""" + +sys_event_client_unsubscribed.desc: +"""是否开启客户端已成功取消订阅主题事件消息。""" + +overload_protection_backoff_new_conn.desc: +"""高负载时，拒绝新进来的客户端连接。""" + +overload_protection_backoff_new_conn.label: +"""关闭新连接""" + +server_ssl_opts_schema_ocsp_responder_url.desc: +"""用于检查服务器证书的 OCSP Responder 的 URL。""" + +server_ssl_opts_schema_ocsp_responder_url.label: +"""OCSP Responder 的 URL""" + +mqtt_idle_timeout.desc: +"""设置连接被断开或进入休眠状态前的等待时间，空闲超时后， + - 如暂未收到客户端的 CONNECT 报文，连接将断开； + - 如已收到客户端的 CONNECT 报文，连接将进入休眠模式以节省系统资源。 + +注意：请合理设置该参数值，如等待时间设置过长，可能造成系统资源的浪费。""" + +mqtt_idle_timeout.label: +"""空闲超时""" + +fields_mqtt_quic_listener_conn_flow_control_window.desc: +"""连接的流控窗口。默认：16777216""" + +fields_mqtt_quic_listener_conn_flow_control_window.label: +"""流控窗口""" + +fields_mqtt_quic_listener_maximum_mtu.desc: +"""一个连接所支持的最大MTU。这将是最大的探测值。默认值：1500""" + +fields_mqtt_quic_listener_maximum_mtu.label: +"""最大 MTU""" + +sysmon_top_db_name.desc: +"""PostgreSQL 数据库的数据库名""" + +sysmon_top_db_name.label: +"""数据库名""" + +mqtt_strict_mode.desc: +"""是否以严格模式解析 MQTT 消息。 +严格模式下，如客户端 ID、主题名称等中包含无效 utf8 字符串，连接将被断开。""" + +mqtt_strict_mode.label: +"""严格模式""" + +shared_subscription_group_strategy.desc: +"""设置共享订阅组为单位的分发策略。该配置是一个从组名到 +策略名的一个map，组名不得包含 `[A-Za-z0-9]` 之外的特殊字符。""" + +fields_deflate_opts_strategy.desc: +"""指定压缩策略。""" + +fields_deflate_opts_strategy.label: +"""指定压缩策略""" + +shared_subscription_strategy_enum.desc: +"""共享订阅的分发策略名称。 +- `random`：随机选择一个组内成员； +- `round_robin`：循环选择下一个成员； +- `round_robin_per_group`：在共享组内循环选择下一个成员； +- `sticky`：使用上一次选中的成员； +- `hash`：根据 ClientID 哈希映射到一个成员； +- `local`：随机分发到节点本地成成员，如果本地成员不存在，则随机分发到任意一个成员。""" + +persistent_session_builtin_sess_msg_table.desc: +"""优化内置的会话消息表的配置。""" + +persistent_session_builtin_sess_msg_table.label: +"""用于内建会话管理表的性能调优参数""" + +mqtt_mqueue_store_qos0.desc: +"""指定在连接断开但会话保持期间，是否需要在消息队列中存储 QoS 0 消息。""" + +mqtt_mqueue_store_qos0.label: +"""存储 QoS 0 消息""" + +server_ssl_opts_schema_client_renegotiation.desc: +"""在支持客户机发起的重新协商的协议中，这种操作的资源成本对于服务器来说高于客户机。 +这可能会成为拒绝服务攻击的载体。 +SSL 应用程序已经采取措施来反击此类尝试，但通过将此选项设置为 false，可以严格禁用客户端发起的重新协商。 +默认值为 true。请注意，由于基础密码套件可以加密的消息数量有限，禁用重新协商可能会导致长期连接变得不可用。""" + +server_ssl_opts_schema_client_renegotiation.label: +"""SSL 客户端冲协商""" + +server_ssl_opts_schema_enable_crl_check.desc: +"""是否为该监听器启用 CRL 检查。""" + +server_ssl_opts_schema_enable_crl_check.label: +"""启用 CRL 检查""" + +fields_mqtt_quic_listener_peer_bidi_stream_count.desc: +"""允许对端打开的双向流的数量""" + +fields_mqtt_quic_listener_peer_bidi_stream_count.label: +"""对端双向流的数量""" + +fields_mqtt_quic_listener_max_stateless_operations.desc: +"""无状态操作的最大数量，在任何时候都可以在一个工作者上排队。默认值：16""" + +fields_mqtt_quic_listener_max_stateless_operations.label: +"""最大无状态操作数""" + +fields_ws_opts_idle_timeout.desc: +"""关闭在此间隔内未发送 MQTT CONNECT 消息的客户端的传输层连接。""" + +fields_ws_opts_idle_timeout.label: +"""WS 空闲时间""" + +fields_mqtt_quic_listener_max_ack_delay_ms.desc: +"""在收到数据后要等待多长时间才能发送一个ACK。默认值：25""" + +fields_mqtt_quic_listener_max_ack_delay_ms.label: +"""最大应答延迟 毫秒""" + +base_listener_zone.desc: +"""监听器所属的配置组。""" + +base_listener_zone.label: +"""配置组""" + +fields_mqtt_quic_listener_handshake_idle_timeout.desc: +"""一个握手在被丢弃之前可以空闲多长时间。""" + +fields_mqtt_quic_listener_handshake_idle_timeout.label: +"""握手空闲超时时间""" + +force_gc_enable.desc: +"""启用强制垃圾回收。""" + +force_gc_enable.label: +"""启用强制垃圾回收""" + +fields_ws_opts_allow_origin_absence.desc: +"""If false and check_origin_enable is true, the server will reject requests that don't have origin HTTP header.""" + +fields_ws_opts_allow_origin_absence.label: +"""允许 origin 缺失""" + +common_ssl_opts_schema_versions.desc: +"""支持所有TLS/DTLS版本
+注：PSK 的 Ciphers 无法在 tlsv1.3 中使用，如果打算使用 PSK 密码套件，请确保这里配置为 ["tlsv1.2","tlsv1.1"]。""" + +common_ssl_opts_schema_versions.label: +"""SSL 版本""" + +mqtt_listener_proxy_protocol_timeout.desc: +"""代理协议超时。如果在超时时间内未收到代理协议数据包，EMQX将关闭TCP连接。""" + +mqtt_listener_proxy_protocol_timeout.label: +"""Proxy protocol 超时时间""" + +fields_mqtt_quic_listener_idle_timeout.desc: +"""一个连接在被关闭之前可以空闲多长时间。0表示禁用。""" + +fields_mqtt_quic_listener_idle_timeout.label: +"""空闲超时时间""" + +common_ssl_opts_schema_secure_renegotiate.desc: +"""SSL 参数重新协商是一种允许客户端和服务器动态重新协商 SSL 连接参数的功能。 +RFC 5746 定义了一种更安全的方法。通过启用安全的重新协商，您就失去了对不安全的重新协商的支持，从而容易受到 MitM 攻击。""" + +common_ssl_opts_schema_secure_renegotiate.label: +"""SSL 重新协商""" + +sysmon_vm_busy_port.desc: +"""当一个系统接口（例如 TCP socket）过忙，会触发一条带有 busy_port 关键字的 warning 级别的日志。 +同时还会发布一条主题为 $SYS/sysmon/busy_port 的 MQTT 系统消息。""" + +sysmon_vm_busy_port.label: +"""启用端口过忙监控""" + +sys_event_client_connected.desc: +"""是否开启客户端已连接事件消息。""" + +sysmon_vm_process_low_watermark.desc: +"""在清除相应警报之前，本地节点上可以同时存在多少进程的阈值（以进程百分比表示）。""" + +sysmon_vm_process_low_watermark.label: +"""进程数低水位线""" + +mqtt_max_packet_size.desc: +"""允许的最大 MQTT 报文大小。""" + +mqtt_max_packet_size.label: +"""最大报文大小""" + +common_ssl_opts_schema_reuse_sessions.desc: +"""启用 TLS 会话重用。""" + +common_ssl_opts_schema_reuse_sessions.label: +"""TLS 会话重用""" + +common_ssl_opts_schema_depth.desc: +"""在有效的证书路径中，可以跟随对等证书的非自颁发中间证书的最大数量。 +因此，如果深度为0，则对等方必须由受信任的根 CA 直接签名；
+如果是1，路径可以是 PEER、中间 CA、ROOT-CA；
+如果是2，则路径可以是PEER、中间 CA1、中间 CA2、ROOT-CA。""" + +common_ssl_opts_schema_depth.label: +"""CA 证书深度""" + +sysmon_vm_long_gc.desc: +"""当系统检测到某个 Erlang 进程垃圾回收占用过长时间，会触发一条带有 long_gc 关键字的日志。 +同时还会发布一条主题为 $SYS/sysmon/long_gc 的 MQTT 系统消息。""" + +sysmon_vm_long_gc.label: +"""启用长垃圾回收监控""" + +fields_mqtt_quic_listener_keyfile.desc: +"""私钥文件。在 5.1 中会被废弃，使用 .ssl_options.keyfile 代替。""" + +fields_mqtt_quic_listener_keyfile.label: +"""私钥文件""" + +mqtt_peer_cert_as_clientid.desc: +"""使用对端证书中的 CN、DN 字段或整个证书内容来作为客户端 ID。仅适用于 TLS 连接； +目前支持： +- cn: 取证书的 CN 字段 +- dn: 取证书的 DN 字段 +- crt: 取 DER 或 PEM 证书的内容 +- pem: 将 DER 证书内容转换为 PEM 格式作为客户端 ID +- md5: 取 DER 或 PEM 证书内容的 MD5 值""" + +mqtt_peer_cert_as_clientid.label: +"""对端证书作为客户端 ID""" + +persistent_session_store_message_gc_interval.desc: +"""将未送达的消息垃圾收集到持久会话的开始间隔。 +这会影响检查 "max_retain_undelivered"（最大保留未送达）的删除频率。""" + +persistent_session_store_message_gc_interval.label: +"""消息清理间隔""" + +broker_shared_dispatch_ack_enabled.desc: +"""该配置项已废弃，会在 5.1 中移除。 +启用/禁用 QoS 1 和 QoS 2 消息的共享派发确认。 +开启后，允许将消息从未及时回复 ACK 的订阅者 （例如，客户端离线） 重新派发给另外一个订阅者。""" + +base_listener_enable_authn.desc: +"""配置 true （默认值）启用客户端进行身份认证，通过检查认配置的认认证器链来决定是否允许接入。 +配置 false 时，将不对客户端做任何认证，任何客户端，不论是不是携带用户名等认证信息，都可以接入。 +配置 quick_deny_anonymous 时，行为跟 true 类似，但是会对匿名 +客户直接拒绝，不做使用任何认证器对客户端进行身份检查。""" + +base_listener_enable_authn.label: +"""启用身份认证""" + +force_shutdown_enable.desc: +"""启用 `force_shutdown` 功能。""" + +force_shutdown_enable.label: +"""启用 `force_shutdown` 功能""" + +broker_enable_session_registry.desc: +"""是否启用 Session Registry""" + +overload_protection_backoff_delay.desc: +"""高负载时，一些不重要的任务可能会延迟执行，在这里设置允许延迟的时间。""" + +overload_protection_backoff_delay.label: +"""延迟时间""" + +ciphers_schema_common.desc: +"""此配置保存由逗号分隔的 TLS 密码套件名称，或作为字符串数组。例如 +"TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256"或 +["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"]。 +
+密码（及其顺序）定义了客户端和服务器通过网络连接加密信息的方式。 +选择一个好的密码套件对于应用程序的数据安全性、机密性和性能至关重要。 + +名称应为 OpenSSL 字符串格式（而不是 RFC 格式）。 +EMQX 配置文档提供的所有默认值和示例都是 OpenSSL 格式
+注意：某些密码套件仅与特定的 TLS 版本兼容（'tlsv1.1'、'tlsv1.2'或'tlsv1.3'）。 +不兼容的密码套件将被自动删除。 + +例如，如果只有 versions 仅配置为 tlsv1.3。为其他版本配置密码套件将无效。 + +
+注：PSK 的 Ciphers 不支持 tlsv1.3
+如果打算使用PSK密码套件 tlsv1.3。应在ssl.versions中禁用。 + +
+PSK 密码套件： +"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384, +RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256, +RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA, +RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA"""" + +ciphers_schema_common.label: +"""""" + +sys_event_client_disconnected.desc: +"""是否开启客户端已断开连接事件消息。""" + +crl_cache_refresh_interval.desc: +"""从服务器刷新CRL的周期。 该配置对所有 URL 和监听器有效。""" + +crl_cache_refresh_interval.label: +"""CRL 缓存刷新间隔""" + +mqtt_listener_proxy_protocol.desc: +"""如果EMQX集群部署在 HAProxy 或 Nginx 之后，请启用代理协议 V1/2
+详情见: https://www.haproxy.com/blog/haproxy/proxy-protocol/""" + +mqtt_listener_proxy_protocol.label: +"""Proxy protocol""" + +mqtt_listener_access_rules.desc: +"""此监听器的访问控制规则。""" + +mqtt_listener_access_rules.label: +"""访问控制规则""" + +server_ssl_opts_schema_enable_ocsp_stapling.desc: +"""是否为监听器启用 OCSP Stapling 功能。 如果设置为 true，需要定义 OCSP Responder 的 URL 和证书签发者的 PEM 文件路径。""" + +server_ssl_opts_schema_enable_ocsp_stapling.label: +"""启用 OCSP Stapling""" + +fields_tcp_opts_send_timeout_close.desc: +"""如果发送超时，则关闭连接。""" + +fields_tcp_opts_send_timeout_close.label: +"""TCP 发送超时关闭连接""" + +sysmon_os_cpu_check_interval.desc: +"""定期 CPU 检查的时间间隔。""" + +sysmon_os_cpu_check_interval.label: +"""定期 CPU 检查的时间间隔""" + +sysmon_top_sample_interval.desc: +"""指定应收集进程顶部的频率。""" + +sysmon_top_sample_interval.label: +"""取样时间""" + +fields_mqtt_quic_listener_idle_timeout_ms.desc: +"""一个连接在被优雅地关闭之前可以空闲多长时间。0 表示禁用超时""" + +fields_mqtt_quic_listener_idle_timeout_ms.label: +"""空闲超时 毫秒""" + +fields_ws_opts_fail_if_no_subprotocol.desc: +"""如果true，当客户端未携带Sec WebSocket Protocol字段时，服务器将返回一个错误。 +
注意：微信小程序需要禁用此验证。""" + +fields_ws_opts_fail_if_no_subprotocol.label: +"""无 subprotocol 则失败""" + +mqtt_wildcard_subscription.desc: +"""是否启用对 MQTT 通配符订阅的支持。""" + +mqtt_wildcard_subscription.label: +"""通配符订阅可用""" + +server_ssl_opts_schema_ocsp_refresh_interval.desc: +"""为服务器刷新OCSP响应的周期。""" + +server_ssl_opts_schema_ocsp_refresh_interval.label: +"""OCSP 刷新间隔""" + +overload_protection_backoff_hibernation.desc: +"""高负载时，跳过进程休眠。""" + +overload_protection_backoff_hibernation.label: +"""跳过休眠""" + +fields_ws_opts_max_frame_size.desc: +"""单个 MQTT 数据包的最大长度。""" + +fields_ws_opts_max_frame_size.label: +"""最大数据包长度""" + +sys_event_messages.desc: +"""客户端事件消息。""" + +broker_perf_trie_compaction.desc: +"""是否开启主题表压缩存储。 +启用它会显着提高通配符主题订阅率，如果通配符主题具有唯一前缀，例如：'sensor/{{id}}/+/'，其中每个订阅者的 ID 是唯一的。 +如果消息主要发布到具有大量级别的主题，则主题匹配性能（发布时）可能会降低。 + +注意：这是一个集群范围的配置。 它要求在更改之前停止所有节点。""" + +sysmon_vm_large_heap.desc: +"""启用后，当一个 Erlang 进程申请了大量内存，系统会触发一条带有 large_heap 关键字的 +warning 级别日志。同时还会发布一条主题为 $SYS/sysmon/busy_dist_port 的 MQTT 系统消息。""" + +sysmon_vm_large_heap.label: +"""启用大 heap 监控""" + +} diff --git a/rel/i18n/zh/emqx_slow_subs_api.hocon b/rel/i18n/zh/emqx_slow_subs_api.hocon new file mode 100644 index 000000000..b0e801ca3 --- /dev/null +++ b/rel/i18n/zh/emqx_slow_subs_api.hocon @@ -0,0 +1,30 @@ +emqx_slow_subs_api { + +clear_records_api.desc: +"""清除当前记录，然后重新开始统计""" + +clientid.desc: +"""消息的客户端 ID""" + +get_records_api.desc: +"""查看慢订阅的统计数据""" + +get_setting_api.desc: +"""查看配置""" + +last_update_time.desc: +"""记录的更新时间戳""" + +node.desc: +"""消息的节点名称""" + +timespan.desc: +"""消息的传输耗时""" + +topic.desc: +"""消息的主题""" + +update_setting_api.desc: +"""更新配置""" + +} diff --git a/rel/i18n/zh/emqx_slow_subs_schema.hocon b/rel/i18n/zh/emqx_slow_subs_schema.hocon new file mode 100644 index 000000000..beadac1ea --- /dev/null +++ b/rel/i18n/zh/emqx_slow_subs_schema.hocon @@ -0,0 +1,18 @@ +emqx_slow_subs_schema { + +enable.desc: +"""开启慢订阅""" + +expire_interval.desc: +"""慢订阅记录的有效时间""" + +stats_type.desc: +"""慢订阅的统计类型""" + +threshold.desc: +"""慢订阅统计的阈值""" + +top_k_num.desc: +"""慢订阅统计表的记录数量上限""" + +} diff --git a/rel/i18n/zh/emqx_statsd_api.hocon b/rel/i18n/zh/emqx_statsd_api.hocon new file mode 100644 index 000000000..5761e2431 --- /dev/null +++ b/rel/i18n/zh/emqx_statsd_api.hocon @@ -0,0 +1,9 @@ +emqx_statsd_api { + +get_statsd_config_api.desc: +"""列出 StatsD 指标采集和推送服务的的配置。""" + +update_statsd_config_api.desc: +"""更新 StatsD 指标采集和推送服务的配置。""" + +} diff --git a/rel/i18n/zh/emqx_statsd_schema.hocon b/rel/i18n/zh/emqx_statsd_schema.hocon new file mode 100644 index 000000000..548ad33bc --- /dev/null +++ b/rel/i18n/zh/emqx_statsd_schema.hocon @@ -0,0 +1,30 @@ +emqx_statsd_schema { + +enable.desc: +"""启用或禁用 StatsD 指标采集和推送服务。""" + +flush_interval.desc: +"""指标的推送间隔。""" + +get_statsd_config_api.desc: +"""列出 StatsD 指标采集和推送服务的的配置。""" + +sample_interval.desc: +"""指标的采样间隔。""" + +server.desc: +"""StatsD 服务器地址。""" + +statsd.desc: +"""StatsD 指标采集与推送配置。""" + +statsd.label: +"""StatsD""" + +tags.desc: +"""指标的标签。""" + +update_statsd_config_api.desc: +"""更新 StatsD 指标采集和推送服务的配置。""" + +} diff --git a/rel/i18n/zh/emqx_stomp_schema.hocon b/rel/i18n/zh/emqx_stomp_schema.hocon new file mode 100644 index 000000000..13cfc0397 --- /dev/null +++ b/rel/i18n/zh/emqx_stomp_schema.hocon @@ -0,0 +1,15 @@ +emqx_stomp_schema { + +stom_frame_max_body_length.desc: +"""允许的 Stomp 报文 Body 的最大字节数""" + +stom_frame_max_headers.desc: +"""允许的 Header 最大数量""" + +stomp.desc: +"""Stomp 网关配置。当前实现支持 v1.2/1.1/1.0 协议版本""" + +stomp_frame_max_headers_length.desc: +"""允许的 Header 字符串的最大长度""" + +} diff --git a/rel/i18n/zh/emqx_telemetry_api.hocon b/rel/i18n/zh/emqx_telemetry_api.hocon new file mode 100644 index 000000000..4e445a56f --- /dev/null +++ b/rel/i18n/zh/emqx_telemetry_api.hocon @@ -0,0 +1,54 @@ +emqx_telemetry_api { + +active_modules.desc: +"""获取活跃模块""" + +active_plugins.desc: +"""获取活跃插件""" + +emqx_version.desc: +"""获取 emqx 版本""" + +enable.desc: +"""启用遥测""" + +get_telemetry_data_api.desc: +"""获取遥测数据""" + +get_telemetry_status_api.desc: +"""获取遥测状态""" + +license.desc: +"""获取 license 信息""" + +messages_received.desc: +"""获取接收到的消息数量""" + +messages_sent.desc: +"""获取发送的消息数量""" + +nodes_uuid.desc: +"""获取节点 UUID""" + +num_clients.desc: +"""获取客户端数量""" + +os_name.desc: +"""获取操作系统名称""" + +os_version.desc: +"""获取操作系统版本""" + +otp_version.desc: +"""获取 OTP 版本""" + +up_time.desc: +"""获取运行时间""" + +update_telemetry_status_api.desc: +"""更新遥测状态""" + +uuid.desc: +"""获取 UUID""" + +} diff --git a/rel/i18n/zh/emqx_topic_metrics_api.hocon b/rel/i18n/zh/emqx_topic_metrics_api.hocon new file mode 100644 index 000000000..23a5791b3 --- /dev/null +++ b/rel/i18n/zh/emqx_topic_metrics_api.hocon @@ -0,0 +1,105 @@ +emqx_topic_metrics_api { + +message_qos1_in_rate.desc: +"""QoS1 接收消息速率""" + +message_out_count.desc: +"""发送消息数量""" + +message_qos2_out_rate.desc: +"""QoS2 发送消息速率""" + +message_qos0_in_rate.desc: +"""QoS0 接收消息速率""" + +get_topic_metrics_api.desc: +"""获取主题监控数据""" + +reset_time.desc: +"""重置时间。标准 rfc3339 时间格式，例如：2018-01-01T12:00:00Z。如果从未重置则为空""" + +topic_metrics_api_response400.desc: +"""错误请求。已存在或错误的主题名称""" + +reset_topic_desc.desc: +"""主题名称。如果此参数不存在，则所有创建的主题监控数据都将重置。""" + +topic_metrics_api_response409.desc: +"""冲突。主题监控数据超过最大限制512""" + +post_topic_metrics_api.desc: +"""创建主题监控数据""" + +message_dropped_rate.desc: +"""丢弃消息速率""" + +message_qos2_in_rate.desc: +"""QoS2 接收消息速率""" + +message_in_rate.desc: +"""接收消息速率""" + +message_qos0_out_rate.desc: +"""QoS0 发送消息速率""" + +message_qos2_in_count.desc: +"""QoS2 接收消息数量""" + +message_dropped_count.desc: +"""丢弃消息数量""" + +topic_metrics_api_response404.desc: +"""未找到。主题监控数据未找到""" + +topic_in_path.desc: +"""主题字符串。注意：主题字符串在url路径中必须编码""" + +action.desc: +"""操作，仅支持 reset""" + +message_qos0_in_count.desc: +"""QoS0 接收消息数量""" + +message_qos1_out_rate.desc: +"""QoS1 发送消息速率""" + +topic.desc: +"""主题""" + +reset_topic_metrics_api.desc: +"""重置主题监控状态""" + +create_time.desc: +"""创建时间。标准 rfc3339 时间格式，例如：2018-01-01T12:00:00Z""" + +metrics.desc: +"""监控数据""" + +message_qos1_out_count.desc: +"""QoS1 发送消息数量""" + +gat_topic_metrics_data_api.desc: +"""获取主题监控数据""" + +message_qos1_in_count.desc: +"""QoS1 接收消息数量""" + +delete_topic_metrics_data_api.desc: +"""删除主题监控数据""" + +message_qos0_out_count.desc: +"""QoS0 发送消息数量""" + +topic_in_body.desc: +"""主题字符串""" + +message_in_count.desc: +"""接收消息数量""" + +message_qos2_out_count.desc: +"""QoS2 发送消息数量""" + +message_out_rate.desc: +"""发送消息速率""" + +} From b63b880116234b9022a41f0ef18811917a57d101 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 5 Apr 2023 16:20:51 +0200 Subject: [PATCH 176/279] refactor: update i18n style check script to work with new layout --- scripts/check-i18n-style.escript | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/scripts/check-i18n-style.escript b/scripts/check-i18n-style.escript index a76fce90e..b8b6bdac7 100755 --- a/scripts/check-i18n-style.escript +++ b/scripts/check-i18n-style.escript @@ -76,22 +76,16 @@ check_label(_Name, _) -> ok. check_desc(Name, #{<<"desc">> := Desc}) -> - do_check_desc(Name, Desc); + check_desc_string(Name, Desc); check_desc(Name, _) -> die("~s: no 'desc'~n", [Name]). -do_check_desc(Name, #{<<"zh">> := Zh, <<"en">> := En}) -> - ok = check_desc_string(Name, "zh", Zh), - ok = check_desc_string(Name, "en", En); -do_check_desc(Name, _) -> - die("~s: missing 'zh' or 'en'~n", [Name]). - -check_desc_string(Name, Tr, <<>>) -> - logerr("~s.~s: empty string~n", [Name, Tr]); -check_desc_string(Name, Tr, BinStr) -> +check_desc_string(Name, <<>>) -> + logerr("~s: empty string~n", [Name]); +check_desc_string(Name, BinStr) -> Str = unicode:characters_to_list(BinStr, utf8), Err = fun(Reason) -> - logerr("~s.~s: ~s~n", [Name, Tr, Reason]) + logerr("~s: ~s~n", [Name, Reason]) end, case Str of [$\s | _] -> From 9b7800aa8cd5e5ad60edad0fb01e0535ad6b6b63 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 5 Apr 2023 16:47:40 +0200 Subject: [PATCH 177/279] refactor: merge into per-language i18n files --- scripts/merge-i18n.escript | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript index b2501d10a..7ffd3aa8a 100755 --- a/scripts/merge-i18n.escript +++ b/scripts/merge-i18n.escript @@ -3,10 +3,14 @@ -mode(compile). main(_) -> + main_per_lang("en"), + main_per_lang("zh"). + +main_per_lang(Lang) -> BaseConf = <<"">>, - Cfgs0 = get_all_files(), + Cfgs0 = get_all_files(Lang), Conf = merge(BaseConf, Cfgs0), - OutputFile = "apps/emqx_dashboard/priv/i18n.conf", + OutputFile = "apps/emqx_dashboard/priv/i18n." ++ Lang ++ ".conf", ok = filelib:ensure_dir(OutputFile), ok = file:write_file(OutputFile, Conf). @@ -21,7 +25,11 @@ merge(BaseConf, Cfgs) -> end end, BaseConf, Cfgs). -get_all_files() -> - Dir = filename:join(["rel","i18n"]), +get_all_files(Lang) -> + Dir = + case Lang of + "en" -> filename:join(["rel", "i18n"]); + "zh" -> filename:join(["rel", "i18n", "zh"]) + end, Files = filelib:wildcard("*.hocon", Dir), lists:map(fun(Name) -> filename:join([Dir, Name]) end, Files). From f2fae16d3bb099cc107e28b5ac6077d9ecd36fb0 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 15 Apr 2023 19:41:07 +0800 Subject: [PATCH 178/279] fix(gw): load emqx applications before hocon configs checking --- apps/emqx_gateway/src/emqx_gateway_app.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_utils.erl | 6 +++--- bin/nodetool | 16 +++++++++++++++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 01a1aaddd..f0406bcaa 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -45,7 +45,7 @@ load_default_gateway_applications() -> fun(Def) -> load_gateway_application(Def) end, - emqx_gateway_utils:find_gateway_definations() + emqx_gateway_utils:find_gateway_definitions() ). load_gateway_application( diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index f0e65627f..8c80fc1fa 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -75,7 +75,7 @@ fields(gateway) -> } )} end, - emqx_gateway_utils:find_gateway_definations() + emqx_gateway_utils:find_gateway_definitions() ); fields(clientinfo_override) -> [ diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 07185ef42..7a0188387 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -47,7 +47,7 @@ listener_chain/3, make_deprecated_paths/1, make_compatible_schema/2, - find_gateway_definations/0 + find_gateway_definitions/0 ]). -export([stringfy/1]). @@ -564,8 +564,8 @@ make_compatible_schema2(Path, SchemaFun) -> Schema ). --spec find_gateway_definations() -> list(gateway_def()). -find_gateway_definations() -> +-spec find_gateway_definitions() -> list(gateway_def()). +find_gateway_definitions() -> lists:flatten( lists:map( fun(App) -> diff --git a/bin/nodetool b/bin/nodetool index 9a5d5e069..9f32f9b3b 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -362,6 +362,20 @@ add_libs_dir() -> add_lib_dir(RootDir, Name, Vsn) -> LibDir = filename:join([RootDir, lib, atom_to_list(Name) ++ "-" ++ Vsn, ebin]), case code:add_patha(LibDir) of - true -> ok; + true -> + %% load all applications into application controller, before performing + %% the configuration check of HOCON + %% + %% It helps to implement the feature of dynamically searching schema. + %% See `emqx_gateway_schema:fields(gateway)` + is_emqx_application(Name) andalso application:load(Name), + ok; {error, _} -> error(LibDir) end. + +is_emqx_application(Name) when is_atom(Name) -> + is_emqx_application(atom_to_list(Name)); +is_emqx_application("emqx_" ++ _Rest) -> + true; +is_emqx_application(_) -> + false. From 069afd042a59b8fcc0bee706d82cd8f7ca952d98 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sat, 15 Apr 2023 21:18:37 +0800 Subject: [PATCH 179/279] chore: update changes --- bin/nodetool | 11 +++++++++-- changes/ce/fix-10410.en.md | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 changes/ce/fix-10410.en.md diff --git a/bin/nodetool b/bin/nodetool index 9f32f9b3b..8170e68e2 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -272,7 +272,7 @@ chkconfig(File) -> end. check_license(Config) -> - ok = application:load(emqx_license), + ok = ensure_application_load(emqx_license), %% This checks formal license validity to ensure %% that the node can successfully start with the given license. @@ -368,7 +368,7 @@ add_lib_dir(RootDir, Name, Vsn) -> %% %% It helps to implement the feature of dynamically searching schema. %% See `emqx_gateway_schema:fields(gateway)` - is_emqx_application(Name) andalso application:load(Name), + is_emqx_application(Name) andalso ensure_application_load(Name), ok; {error, _} -> error(LibDir) end. @@ -379,3 +379,10 @@ is_emqx_application("emqx_" ++ _Rest) -> true; is_emqx_application(_) -> false. + +ensure_application_load(Name) -> + case application:load(Name) of + ok -> ok; + {error, {already_loaded, _}} -> ok; + {error, Reason} -> error({failed_to_load_application, Name, Reason}) + end. diff --git a/changes/ce/fix-10410.en.md b/changes/ce/fix-10410.en.md new file mode 100644 index 000000000..48b55ea31 --- /dev/null +++ b/changes/ce/fix-10410.en.md @@ -0,0 +1,2 @@ +Fix EMQX starting failed once any gateways configured in emqx.conf. +This issue was first introduced in v5.0.22 via [#10278](https://github.com/emqx/emqx/pull/10278). From 3bf642cdffbee12ed24d7f811bd5f941a680ed1c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 17 Apr 2023 17:35:50 +0800 Subject: [PATCH 180/279] chore: update changes/ce/fix-10410.en.md Co-authored-by: Zaiming (Stone) Shi --- changes/ce/fix-10410.en.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/ce/fix-10410.en.md b/changes/ce/fix-10410.en.md index 48b55ea31..aa219c96e 100644 --- a/changes/ce/fix-10410.en.md +++ b/changes/ce/fix-10410.en.md @@ -1,2 +1,2 @@ -Fix EMQX starting failed once any gateways configured in emqx.conf. -This issue was first introduced in v5.0.22 via [#10278](https://github.com/emqx/emqx/pull/10278). +Fix config check failed when gateways are configured in emqx.conf. +This issue was first introduced in v5.0.22 via [#10278](https://github.com/emqx/emqx/pull/10278), the boot-time config check was missing. From 18974a8e11132b4cc4ba2bb352f915dcfa519a5a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 7 Apr 2023 11:18:29 +0200 Subject: [PATCH 181/279] refactor: make schema dump and swagger spec work with split desc files --- Makefile | 1 - apps/emqx/rebar.config | 2 +- apps/emqx_conf/src/emqx_conf.erl | 89 ++++++++------- apps/emqx_dashboard/src/emqx_dashboard.erl | 63 +++-------- .../src/emqx_dashboard_desc_cache.erl | 105 ++++++++++++++++++ .../src/emqx_dashboard_listener.erl | 71 +++++++----- .../src/emqx_dashboard_schema.erl | 2 + .../emqx_dashboard/src/emqx_dashboard_sup.erl | 2 + .../src/emqx_dashboard_swagger.erl | 94 +++++++++------- .../test/emqx_dashboard_listener_SUITE.erl | 51 +++++++++ .../test/emqx_swagger_parameter_SUITE.erl | 1 - .../test/emqx_swagger_requestBody_SUITE.erl | 1 - .../test/emqx_swagger_response_SUITE.erl | 1 - .../test/emqx_rule_funcs_SUITE.erl | 1 - build | 3 +- mix.exs | 2 +- rebar.config | 2 +- rebar.config.erl | 3 +- scripts/merge-config.escript | 50 ++++++++- scripts/merge-i18n.escript | 35 ------ scripts/pre-compile.sh | 1 - 21 files changed, 375 insertions(+), 205 deletions(-) create mode 100644 apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl create mode 100644 apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl delete mode 100755 scripts/merge-i18n.escript diff --git a/Makefile b/Makefile index 45218bf46..fe75b01bc 100644 --- a/Makefile +++ b/Makefile @@ -239,7 +239,6 @@ $(foreach zt,$(ALL_DOCKERS),$(eval $(call gen-docker-target,$(zt)))) .PHONY: merge-config: @$(SCRIPTS)/merge-config.escript - @$(SCRIPTS)/merge-i18n.escript ## elixir target is to create release packages using Elixir's Mix .PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index d954a6b1e..5945ccc7c 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.1"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 1ecda913d..d77ffb680 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -25,9 +25,9 @@ -export([update/3, update/4]). -export([remove/2, remove/3]). -export([reset/2, reset/3]). --export([dump_schema/1, dump_schema/3]). +-export([dump_schema/2]). -export([schema_module/0]). --export([gen_example_conf/4]). +-export([gen_example_conf/2]). %% for rpc -export([get_node_and_config/1]). @@ -136,24 +136,21 @@ reset(Node, KeyPath, Opts) -> emqx_conf_proto_v2:reset(Node, KeyPath, Opts). %% @doc Called from build script. --spec dump_schema(file:name_all()) -> ok. -dump_schema(Dir) -> - I18nFile = emqx_dashboard:i18n_file(), - dump_schema(Dir, emqx_conf_schema, I18nFile). - -dump_schema(Dir, SchemaModule, I18nFile) -> +dump_schema(Dir, SchemaModule) -> + _ = application:load(emqx_dashboard), + ok = emqx_dashboard_desc_cache:init(), lists:foreach( fun(Lang) -> - gen_config_md(Dir, I18nFile, SchemaModule, Lang), - gen_api_schema_json(Dir, I18nFile, Lang), - gen_example_conf(Dir, I18nFile, SchemaModule, Lang), - gen_schema_json(Dir, I18nFile, SchemaModule, Lang) + ok = gen_config_md(Dir, SchemaModule, Lang), + ok = gen_api_schema_json(Dir, Lang), + ok = gen_schema_json(Dir, SchemaModule, Lang) end, ["en", "zh"] - ). + ), + ok = gen_example_conf(Dir, SchemaModule). %% for scripts/spellcheck. -gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> +gen_schema_json(Dir, SchemaModule, Lang) -> SchemaJsonFile = filename:join([Dir, "schema-" ++ Lang ++ ".json"]), io:format(user, "===< Generating: ~s~n", [SchemaJsonFile]), %% EMQX_SCHEMA_FULL_DUMP is quite a hidden API @@ -164,40 +161,44 @@ gen_schema_json(Dir, I18nFile, SchemaModule, Lang) -> false -> ?IMPORTANCE_LOW end, io:format(user, "===< Including fields from importance level: ~p~n", [IncludeImportance]), - Opts = #{desc_file => I18nFile, lang => Lang, include_importance_up_from => IncludeImportance}, + Opts = #{ + include_importance_up_from => IncludeImportance, + desc_resolver => make_desc_resolver(Lang) + }, JsonMap = hocon_schema_json:gen(SchemaModule, Opts), IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]), ok = file:write_file(SchemaJsonFile, IoData). -gen_api_schema_json(Dir, I18nFile, Lang) -> - emqx_dashboard:init_i18n(I18nFile, list_to_binary(Lang)), +gen_api_schema_json(Dir, Lang) -> gen_api_schema_json_hotconf(Dir, Lang), - gen_api_schema_json_bridge(Dir, Lang), - emqx_dashboard:clear_i18n(). + gen_api_schema_json_bridge(Dir, Lang). gen_api_schema_json_hotconf(Dir, Lang) -> SchemaInfo = #{title => <<"EMQX Hot Conf API Schema">>, version => <<"0.1.0">>}, File = schema_filename(Dir, "hot-config-schema-", Lang), - ok = do_gen_api_schema_json(File, emqx_mgmt_api_configs, SchemaInfo). + ok = do_gen_api_schema_json(File, emqx_mgmt_api_configs, SchemaInfo, Lang). gen_api_schema_json_bridge(Dir, Lang) -> SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>}, File = schema_filename(Dir, "bridge-api-", Lang), - ok = do_gen_api_schema_json(File, emqx_bridge_api, SchemaInfo). + ok = do_gen_api_schema_json(File, emqx_bridge_api, SchemaInfo, Lang). schema_filename(Dir, Prefix, Lang) -> Filename = Prefix ++ Lang ++ ".json", filename:join([Dir, Filename]). -gen_config_md(Dir, I18nFile, SchemaModule, Lang) -> +%% TODO: remove it and also remove hocon_md.erl and friends. +%% markdown generation from schema is a failure and we are moving to an interactive +%% viewer like swagger UI. +gen_config_md(Dir, SchemaModule, Lang) -> SchemaMdFile = filename:join([Dir, "config-" ++ Lang ++ ".md"]), io:format(user, "===< Generating: ~s~n", [SchemaMdFile]), - ok = gen_doc(SchemaMdFile, SchemaModule, I18nFile, Lang). + ok = gen_doc(SchemaMdFile, SchemaModule, Lang). -gen_example_conf(Dir, I18nFile, SchemaModule, Lang) -> - SchemaMdFile = filename:join([Dir, "emqx.conf." ++ Lang ++ ".example"]), +gen_example_conf(Dir, SchemaModule) -> + SchemaMdFile = filename:join([Dir, "emqx.conf.example"]), io:format(user, "===< Generating: ~s~n", [SchemaMdFile]), - ok = gen_example(SchemaMdFile, SchemaModule, I18nFile, Lang). + ok = gen_example(SchemaMdFile, SchemaModule). %% @doc return the root schema module. -spec schema_module() -> module(). @@ -211,35 +212,48 @@ schema_module() -> %% Internal functions %%-------------------------------------------------------------------- --spec gen_doc(file:name_all(), module(), file:name_all(), string()) -> ok. -gen_doc(File, SchemaModule, I18nFile, Lang) -> +%% @doc Make a resolver function that can be used to lookup the description by hocon_schema_json dump. +make_desc_resolver(Lang) -> + fun + ({desc, Namespace, Id}) -> + emqx_dashboard_desc_cache:lookup(Lang, Namespace, Id, desc); + (Desc) -> + unicode:characters_to_binary(Desc) + end. + +-spec gen_doc(file:name_all(), module(), string()) -> ok. +gen_doc(File, SchemaModule, Lang) -> Version = emqx_release:version(), Title = "# " ++ emqx_release:description() ++ " Configuration\n\n" ++ "", BodyFile = filename:join([rel, "emqx_conf.template." ++ Lang ++ ".md"]), {ok, Body} = file:read_file(BodyFile), - Opts = #{title => Title, body => Body, desc_file => I18nFile, lang => Lang}, + Resolver = make_desc_resolver(Lang), + Opts = #{title => Title, body => Body, desc_resolver => Resolver}, Doc = hocon_schema_md:gen(SchemaModule, Opts), file:write_file(File, Doc). -gen_example(File, SchemaModule, I18nFile, Lang) -> +gen_example(File, SchemaModule) -> + %% we do not generate description in example files + %% so there is no need for a desc_resolver Opts = #{ title => <<"EMQX Configuration Example">>, body => <<"">>, - desc_file => I18nFile, - lang => Lang, include_importance_up_from => ?IMPORTANCE_MEDIUM }, Example = hocon_schema_example:gen(SchemaModule, Opts), file:write_file(File, Example). %% Only gen hot_conf schema, not all configuration fields. -do_gen_api_schema_json(File, SchemaMod, SchemaInfo) -> +do_gen_api_schema_json(File, SchemaMod, SchemaInfo, Lang) -> io:format(user, "===< Generating: ~s~n", [File]), {ApiSpec0, Components0} = emqx_dashboard_swagger:spec( SchemaMod, - #{schema_converter => fun hocon_schema_to_spec/2} + #{ + schema_converter => fun hocon_schema_to_spec/2, + i18n_lang => Lang + } ), ApiSpec = lists:foldl( fun({Path, Spec, _, _}, Acc) -> @@ -278,13 +292,6 @@ do_gen_api_schema_json(File, SchemaMod, SchemaInfo) -> ), file:write_file(File, IoData). --define(INIT_SCHEMA, #{ - fields => #{}, - translations => #{}, - validations => [], - namespace => undefined -}). - -define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])). -define(TO_COMPONENTS_SCHEMA(_M_, _F_), iolist_to_binary([ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 6f0c8334a..08b7f0142 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -16,22 +16,13 @@ -module(emqx_dashboard). --define(APP, ?MODULE). - -export([ start_listeners/0, start_listeners/1, stop_listeners/1, stop_listeners/0, - list_listeners/0 -]). - --export([ - init_i18n/2, - init_i18n/0, - get_i18n/0, - i18n_file/0, - clear_i18n/0 + list_listeners/0, + wait_for_listeners/0 ]). %% Authorization @@ -90,30 +81,34 @@ start_listeners(Listeners) -> dispatch => Dispatch, middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler] }, - Res = + {OkListeners, ErrListeners} = lists:foldl( - fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, Acc) -> + fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) -> Minirest = BaseMinirest#{protocol => Protocol, protocol_options => ProtoOpts}, case minirest:start(Name, RanchOptions, Minirest) of {ok, _} -> ?ULOG("Listener ~ts on ~ts started.~n", [ Name, emqx_listeners:format_bind(Bind) ]), - Acc; + {[Name | OkAcc], ErrAcc}; {error, _Reason} -> %% Don't record the reason because minirest already does(too much logs noise). - [Name | Acc] + {OkAcc, [Name | ErrAcc]} end end, - [], + {[], []}, listeners(Listeners) ), - case Res of - [] -> ok; - _ -> {error, Res} + case ErrListeners of + [] -> + optvar:set(emqx_dashboard_listeners_ready, OkListeners), + ok; + _ -> + {error, ErrListeners} end. stop_listeners(Listeners) -> + optvar:unset(emqx_dashboard_listeners_ready), [ begin case minirest:stop(Name) of @@ -129,23 +124,8 @@ stop_listeners(Listeners) -> ], ok. -get_i18n() -> - application:get_env(emqx_dashboard, i18n). - -init_i18n(File, Lang) when is_atom(Lang) -> - init_i18n(File, atom_to_binary(Lang)); -init_i18n(File, Lang) when is_binary(Lang) -> - Cache = hocon_schema:new_desc_cache(File), - application:set_env(emqx_dashboard, i18n, #{lang => Lang, cache => Cache}). - -clear_i18n() -> - case application:get_env(emqx_dashboard, i18n) of - {ok, #{cache := Cache}} -> - hocon_schema:delete_desc_cache(Cache), - application:unset_env(emqx_dashboard, i18n); - undefined -> - ok - end. +wait_for_listeners() -> + optvar:read(emqx_dashboard_listeners_ready). %%-------------------------------------------------------------------- %% internal @@ -187,11 +167,6 @@ ip_port(error, Opts) -> {Opts#{port => 18083}, 18083}; ip_port({Port, Opts}, _) when is_integer(Port) -> {Opts#{port => Port}, Port}; ip_port({{IP, Port}, Opts}, _) -> {Opts#{port => Port, ip => IP}, {IP, Port}}. -init_i18n() -> - File = i18n_file(), - Lang = emqx_conf:get([dashboard, i18n_lang], en), - init_i18n(File, Lang). - ranch_opts(Options) -> Keys = [ handshake_timeout, @@ -255,12 +230,6 @@ return_unauthorized(Code, Message) -> }, #{code => Code, message => Message}}. -i18n_file() -> - case application:get_env(emqx_dashboard, i18n_file) of - undefined -> filename:join([code:priv_dir(emqx_dashboard), "i18n.conf"]); - {ok, File} -> File - end. - listeners() -> emqx_conf:get([dashboard, listeners], #{}). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl b/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl new file mode 100644 index 000000000..8fd6fe3d3 --- /dev/null +++ b/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl @@ -0,0 +1,105 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc This module is used to cache the description of the configuration items. +-module(emqx_dashboard_desc_cache). + +-export([init/0]). + +%% internal exports +-export([load_desc/2, lookup/4, lookup/5]). + +-include_lib("emqx/include/logger.hrl"). + +%% @doc Global ETS table to cache the description of the configuration items. +%% The table is owned by the emqx_dashboard_sup the root supervisor of emqx_dashboard. +%% The cache is initialized with the default language (English) and +%% all the desc..hocon files in the www/static directory (extracted from dashboard package). +init() -> + ok = ensure_app_loaded(emqx_dashboard), + PrivDir = code:priv_dir(emqx_dashboard), + EngDesc = filename:join([PrivDir, "desc.en.hocon"]), + WwwStaticDir = filename:join([PrivDir, "www", "static"]), + OtherLangDesc0 = filelib:wildcard("desc.*.hocon", WwwStaticDir), + OtherLangDesc = lists:map(fun(F) -> filename:join([WwwStaticDir, F]) end, OtherLangDesc0), + Files = [EngDesc | OtherLangDesc], + ?MODULE = ets:new(?MODULE, [named_table, public, set, {read_concurrency, true}]), + ok = lists:foreach(fun(F) -> load_desc(?MODULE, F) end, Files). + +%% @doc Load the description of the configuration items from the file. +%% Load is incremental, so it can be called multiple times. +%% NOTE: no garbage collection is done, because stale entries are harmless. +load_desc(EtsTab, File) -> + ?SLOG(info, #{msg => "loading desc", file => File}), + {ok, Descs} = hocon:load(File), + ["desc", Lang, "hocon"] = string:tokens(filename:basename(File), "."), + Insert = fun(Namespace, Id, Tag, Text) -> + Key = {bin(Lang), bin(Namespace), bin(Id), bin(Tag)}, + true = ets:insert(EtsTab, {Key, bin(Text)}), + ok + end, + walk_ns(Insert, maps:to_list(Descs)). + +%% @doc Lookup the description of the configuration item from the global cache. +lookup(Lang, Namespace, Id, Tag) -> + lookup(?MODULE, Lang, Namespace, Id, Tag). + +%% @doc Lookup the description of the configuration item from the given cache. +lookup(EtsTab, Lang0, Namespace, Id, Tag) -> + Lang = bin(Lang0), + case ets:lookup(EtsTab, {Lang, bin(Namespace), bin(Id), bin(Tag)}) of + [{_, Desc}] -> + Desc; + [] when Lang =/= <<"en">> -> + %% fallback to English + lookup(EtsTab, <<"en">>, Namespace, Id, Tag); + _ -> + %% undefined but not <<>> + undefined + end. + +%% The desc files are of names like: +%% desc.en.hocon or desc.zh.hocon +%% And with content like: +%% namespace.id.desc = "description" +%% namespace.id.label = "label" +walk_ns(_Insert, []) -> + ok; +walk_ns(Insert, [{Namespace, Ids} | Rest]) -> + walk_id(Insert, Namespace, maps:to_list(Ids)), + walk_ns(Insert, Rest). + +walk_id(_Insert, _Namespace, []) -> + ok; +walk_id(Insert, Namespace, [{Id, Tags} | Rest]) -> + walk_tag(Insert, Namespace, Id, maps:to_list(Tags)), + walk_id(Insert, Namespace, Rest). + +walk_tag(_Insert, _Namespace, _Id, []) -> + ok; +walk_tag(Insert, Namespace, Id, [{Tag, Text} | Rest]) -> + ok = Insert(Namespace, Id, Tag, Text), + walk_tag(Insert, Namespace, Id, Rest). + +bin(A) when is_atom(A) -> atom_to_binary(A, utf8); +bin(B) when is_binary(B) -> B; +bin(L) when is_list(L) -> list_to_binary(L). + +ensure_app_loaded(App) -> + case application:load(App) of + ok -> ok; + {error, {already_loaded, _}} -> ok + end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl index 01d96bdf0..6a306c288 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_listener.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_listener.erl @@ -15,9 +15,11 @@ %%-------------------------------------------------------------------- -module(emqx_dashboard_listener). --include_lib("emqx/include/logger.hrl"). -behaviour(emqx_config_handler). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + %% API -export([add_handler/0, remove_handler/0]). -export([pre_config_update/3, post_config_update/5]). @@ -54,12 +56,10 @@ init([]) -> {ok, undefined, {continue, regenerate_dispatch}}. handle_continue(regenerate_dispatch, _State) -> - NewState = regenerate_minirest_dispatch(), - {noreply, NewState, hibernate}. + %% initialize the swagger dispatches + ready = regenerate_minirest_dispatch(), + {noreply, ready, hibernate}. -handle_call(is_ready, _From, retry) -> - NewState = regenerate_minirest_dispatch(), - {reply, NewState, NewState, hibernate}; handle_call(is_ready, _From, State) -> {reply, State, State, hibernate}; handle_call(_Request, _From, State) -> @@ -68,6 +68,9 @@ handle_call(_Request, _From, State) -> handle_cast(_Request, State) -> {noreply, State, hibernate}. +handle_info(i18n_lang_changed, _State) -> + NewState = regenerate_minirest_dispatch(), + {noreply, NewState, hibernate}; handle_info({update_listeners, OldListeners, NewListeners}, _State) -> ok = emqx_dashboard:stop_listeners(OldListeners), ok = emqx_dashboard:start_listeners(NewListeners), @@ -83,29 +86,26 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -%% generate dispatch is very slow. +%% generate dispatch is very slow, takes about 1s. regenerate_minirest_dispatch() -> - try - emqx_dashboard:init_i18n(), - lists:foreach( - fun(Listener) -> - minirest:update_dispatch(element(1, Listener)) - end, - emqx_dashboard:list_listeners() - ), - ready - catch - T:E:S -> - ?SLOG(error, #{ - msg => "regenerate_minirest_dispatch_failed", - reason => E, - type => T, - stacktrace => S - }), - retry - after - emqx_dashboard:clear_i18n() - end. + %% optvar:read waits for the var to be set + Names = emqx_dashboard:wait_for_listeners(), + {Time, ok} = timer:tc(fun() -> do_regenerate_minirest_dispatch(Names) end), + Lang = emqx:get_config([dashboard, i18n_lang]), + ?tp(info, regenerate_minirest_dispatch, #{ + elapsed => erlang:convert_time_unit(Time, microsecond, millisecond), + listeners => Names, + i18n_lang => Lang + }), + ready. + +do_regenerate_minirest_dispatch(Names) -> + lists:foreach( + fun(Name) -> + ok = minirest:update_dispatch(Name) + end, + Names + ). add_handler() -> Roots = emqx_dashboard_schema:roots(), @@ -117,6 +117,12 @@ remove_handler() -> ok = emqx_config_handler:remove_handler(Roots), ok. +pre_config_update(_Path, {change_i18n_lang, NewLang}, RawConf) -> + %% e.g. emqx_conf:update([dashboard], {change_i18n_lang, zh}, #{}). + %% TODO: check if there is such a language (all languages are cached in emqx_dashboard_desc_cache) + Update = #{<<"i18n_lang">> => NewLang}, + NewConf = emqx_utils_maps:deep_merge(RawConf, Update), + {ok, NewConf}; pre_config_update(_Path, UpdateConf0, RawConf) -> UpdateConf = remove_sensitive_data(UpdateConf0), NewConf = emqx_utils_maps:deep_merge(RawConf, UpdateConf), @@ -139,6 +145,8 @@ remove_sensitive_data(Conf0) -> Conf1 end. +post_config_update(_, {change_i18n_lang, _}, _NewConf, _OldConf, _AppEnvs) -> + delay_job(i18n_lang_changed); post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) -> OldHttp = get_listener(http, OldConf), OldHttps = get_listener(https, OldConf), @@ -148,7 +156,12 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnvs) -> {StopHttps, StartHttps} = diff_listeners(https, OldHttps, NewHttps), Stop = maps:merge(StopHttp, StopHttps), Start = maps:merge(StartHttp, StartHttps), - _ = erlang:send_after(500, ?MODULE, {update_listeners, Stop, Start}), + delay_job({update_listeners, Stop, Start}). + +%% in post_config_update, the config is not yet persisted to persistent_term +%% so we need to delegate the listener update to the gen_server a bit later +delay_job(Msg) -> + _ = erlang:send_after(500, ?MODULE, Msg), ok. get_listener(Type, Conf) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index d3e4233d3..319c9cee1 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -233,6 +233,8 @@ cors(required) -> false; cors(desc) -> ?DESC(cors); cors(_) -> undefined. +%% TODO: change it to string type +%% It will be up to the dashboard package which languagues to support i18n_lang(type) -> ?ENUM([en, zh]); i18n_lang(default) -> en; i18n_lang('readOnly') -> true; diff --git a/apps/emqx_dashboard/src/emqx_dashboard_sup.erl b/apps/emqx_dashboard/src/emqx_dashboard_sup.erl index 896b44859..04d8ed1d5 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_sup.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_sup.erl @@ -28,6 +28,8 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> + %% supervisor owns the cache table + ok = emqx_dashboard_desc_cache:init(), {ok, {{one_for_one, 5, 100}, [ ?CHILD(emqx_dashboard_listener, brutal_kill), diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index b2ad69997..bdd5866f8 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -84,7 +84,8 @@ -type spec_opts() :: #{ check_schema => boolean() | filter(), translate_body => boolean(), - schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map()) + schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map()), + i18n_lang => atom() }. -type route_path() :: string() | binary(). @@ -333,11 +334,11 @@ check_request_body(#{body := Body}, Spec, _Module, _CheckFun, false) when is_map %% tags, description, summary, security, deprecated meta_to_spec(Meta, Module, Options) -> - {Params, Refs1} = parameters(maps:get(parameters, Meta, []), Module), + {Params, Refs1} = parameters(maps:get(parameters, Meta, []), Module, Options), {RequestBody, Refs2} = request_body(maps:get('requestBody', Meta, []), Module, Options), {Responses, Refs3} = responses(maps:get(responses, Meta, #{}), Module, Options), { - generate_method_desc(to_spec(Meta, Params, RequestBody, Responses)), + generate_method_desc(to_spec(Meta, Params, RequestBody, Responses), Options), lists:usort(Refs1 ++ Refs2 ++ Refs3) }. @@ -348,13 +349,13 @@ to_spec(Meta, Params, RequestBody, Responses) -> Spec = to_spec(Meta, Params, [], Responses), maps:put('requestBody', RequestBody, Spec). -generate_method_desc(Spec = #{desc := _Desc}) -> - Spec1 = trans_description(maps:remove(desc, Spec), Spec), +generate_method_desc(Spec = #{desc := _Desc}, Options) -> + Spec1 = trans_description(maps:remove(desc, Spec), Spec, Options), trans_tags(Spec1); -generate_method_desc(Spec = #{description := _Desc}) -> - Spec1 = trans_description(Spec, Spec), +generate_method_desc(Spec = #{description := _Desc}, Options) -> + Spec1 = trans_description(Spec, Spec, Options), trans_tags(Spec1); -generate_method_desc(Spec) -> +generate_method_desc(Spec, _Options) -> trans_tags(Spec). trans_tags(Spec = #{tags := Tags}) -> @@ -362,7 +363,7 @@ trans_tags(Spec = #{tags := Tags}) -> trans_tags(Spec) -> Spec. -parameters(Params, Module) -> +parameters(Params, Module, Options) -> {SpecList, AllRefs} = lists:foldl( fun(Param, {Acc, RefsAcc}) -> @@ -388,7 +389,7 @@ parameters(Params, Module) -> Type ), Spec1 = trans_required(Spec0, Required, In), - Spec2 = trans_description(Spec1, Type), + Spec2 = trans_description(Spec1, Type, Options), {[Spec2 | Acc], Refs ++ RefsAcc} end end, @@ -432,38 +433,38 @@ trans_required(Spec, true, _) -> Spec#{required => true}; trans_required(Spec, _, path) -> Spec#{required => true}; trans_required(Spec, _, _) -> Spec. -trans_desc(Init, Hocon, Func, Name) -> - Spec0 = trans_description(Init, Hocon), +trans_desc(Init, Hocon, Func, Name, Options) -> + Spec0 = trans_description(Init, Hocon, Options), case Func =:= fun hocon_schema_to_spec/2 of true -> Spec0; false -> - Spec1 = trans_label(Spec0, Hocon, Name), + Spec1 = trans_label(Spec0, Hocon, Name, Options), case Spec1 of #{description := _} -> Spec1; _ -> Spec1#{description => <>} end end. -trans_description(Spec, Hocon) -> +trans_description(Spec, Hocon, Options) -> Desc = case desc_struct(Hocon) of undefined -> undefined; - ?DESC(_, _) = Struct -> get_i18n(<<"desc">>, Struct, undefined); - Struct -> to_bin(Struct) + ?DESC(_, _) = Struct -> get_i18n(<<"desc">>, Struct, undefined, Options); + Text -> to_bin(Text) end, case Desc of undefined -> Spec; Desc -> Desc1 = binary:replace(Desc, [<<"\n">>], <<"
">>, [global]), - maybe_add_summary_from_label(Spec#{description => Desc1}, Hocon) + maybe_add_summary_from_label(Spec#{description => Desc1}, Hocon, Options) end. -maybe_add_summary_from_label(Spec, Hocon) -> +maybe_add_summary_from_label(Spec, Hocon, Options) -> Label = case desc_struct(Hocon) of - ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, undefined); + ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, undefined, Options); _ -> undefined end, case Label of @@ -471,29 +472,44 @@ maybe_add_summary_from_label(Spec, Hocon) -> _ -> Spec#{summary => Label} end. -get_i18n(Key, Struct, Default) -> - {ok, #{cache := Cache, lang := Lang}} = emqx_dashboard:get_i18n(), - Desc = hocon_schema:resolve_schema(Struct, Cache), - emqx_utils_maps:deep_get([Key, Lang], Desc, Default). +get_i18n(Tag, ?DESC(Namespace, Id), Default, Options) -> + Lang = get_lang(Options), + case emqx_dashboard_desc_cache:lookup(Lang, Namespace, Id, Tag) of + undefined -> + Default; + Text -> + Text + end. -trans_label(Spec, Hocon, Default) -> +%% So far i18n_lang in options is only used at build time. +%% At runtime, it's still the global config which controls the language. +get_lang(#{i18n_lang := Lang}) -> Lang; +get_lang(_) -> emqx:get_config([dashboard, i18n_lang]). + +trans_label(Spec, Hocon, Default, Options) -> Label = case desc_struct(Hocon) of - ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, Default); + ?DESC(_, _) = Struct -> get_i18n(<<"label">>, Struct, Default, Options); _ -> Default end, Spec#{label => Label}. desc_struct(Hocon) -> - case hocon_schema:field_schema(Hocon, desc) of - undefined -> - case hocon_schema:field_schema(Hocon, description) of - undefined -> get_ref_desc(Hocon); - Struct1 -> Struct1 - end; - Struct -> - Struct - end. + R = + case hocon_schema:field_schema(Hocon, desc) of + undefined -> + case hocon_schema:field_schema(Hocon, description) of + undefined -> get_ref_desc(Hocon); + Struct1 -> Struct1 + end; + Struct -> + Struct + end, + ensure_bin(R). + +ensure_bin(undefined) -> undefined; +ensure_bin(?DESC(_Namespace, _Id) = Desc) -> Desc; +ensure_bin(Text) -> to_bin(Text). get_ref_desc(?R_REF(Mod, Name)) -> case erlang:function_exported(Mod, desc, 1) of @@ -524,7 +540,7 @@ responses(Responses, Module, Options) -> {Spec, Refs}. response(Status, ?DESC(_Mod, _Id) = Schema, {Acc, RefsAcc, Module, Options}) -> - Desc = trans_description(#{}, #{desc => Schema}), + Desc = trans_description(#{}, #{desc => Schema}, Options), {Acc#{integer_to_binary(Status) => Desc}, RefsAcc, Module, Options}; response(Status, Bin, {Acc, RefsAcc, Module, Options}) when is_binary(Bin) -> {Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module, Options}; @@ -553,7 +569,7 @@ response(Status, Schema, {Acc, RefsAcc, Module, Options}) -> Hocon = hocon_schema:field_schema(Schema, type), Examples = hocon_schema:field_schema(Schema, examples), {Spec, Refs} = hocon_schema_to_spec(Hocon, Module), - Init = trans_description(#{}, Schema), + Init = trans_description(#{}, Schema, Options), Content = content(Spec, Examples), { Acc#{integer_to_binary(Status) => Init#{<<"content">> => Content}}, @@ -563,7 +579,7 @@ response(Status, Schema, {Acc, RefsAcc, Module, Options}) -> }; false -> {Props, Refs} = parse_object(Schema, Module, Options), - Init = trans_description(#{}, Schema), + Init = trans_description(#{}, Schema, Options), Content = Init#{<<"content">> => content(Props)}, {Acc#{integer_to_binary(Status) => Content}, Refs ++ RefsAcc, Module, Options} end. @@ -590,7 +606,7 @@ components(Options, [{Module, Field} | Refs], SpecAcc, SubRefsAcc) -> %% parameters in ref only have one value, not array components(Options, [{Module, Field, parameter} | Refs], SpecAcc, SubRefsAcc) -> Props = hocon_schema_fields(Module, Field), - {[Param], SubRefs} = parameters(Props, Module), + {[Param], SubRefs} = parameters(Props, Module, Options), Namespace = namespace(Module), NewSpecAcc = SpecAcc#{?TO_REF(Namespace, Field) => Param}, components(Options, Refs, NewSpecAcc, SubRefs ++ SubRefsAcc). @@ -869,7 +885,7 @@ parse_object_loop([{Name, Hocon} | Rest], Module, Options, Props, Required, Refs HoconType = hocon_schema:field_schema(Hocon, type), Init0 = init_prop([default | ?DEFAULT_FIELDS], #{}, Hocon), SchemaToSpec = schema_converter(Options), - Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin), + Init = trans_desc(Init0, Hocon, SchemaToSpec, NameBin, Options), {Prop, Refs1} = SchemaToSpec(HoconType, Module), NewRequiredAcc = case is_required(Hocon) of diff --git a/apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl new file mode 100644 index 000000000..7f28841fc --- /dev/null +++ b/apps/emqx_dashboard/test/emqx_dashboard_listener_SUITE.erl @@ -0,0 +1,51 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_dashboard_listener_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:init_suite([emqx_conf]), + ok = change_i18n_lang(en), + Config. + +end_per_suite(_Config) -> + ok = change_i18n_lang(en), + emqx_mgmt_api_test_util:end_suite([emqx_conf]). + +t_change_i18n_lang(_Config) -> + ?check_trace( + begin + ok = change_i18n_lang(zh), + {ok, _} = ?block_until(#{?snk_kind := regenerate_minirest_dispatch}, 10_000), + ok + end, + fun(ok, Trace) -> + ?assertMatch([#{i18n_lang := zh}], ?of_kind(regenerate_minirest_dispatch, Trace)) + end + ), + ok. + +change_i18n_lang(Lang) -> + {ok, _} = emqx_conf:update([dashboard], {change_i18n_lang, Lang}, #{}), + ok. diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index 472e90405..81b3f4402 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -64,7 +64,6 @@ groups() -> init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite([emqx_conf]), - emqx_dashboard:init_i18n(), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 3150ed097..e6fa62f77 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -33,7 +33,6 @@ init_per_suite(Config) -> mria:start(), application:load(emqx_dashboard), emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard], fun set_special_configs/1), - emqx_dashboard:init_i18n(), Config. set_special_configs(emqx_dashboard) -> diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 4d1501dae..753aaad7a 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -33,7 +33,6 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_mgmt_api_test_util:init_suite([emqx_conf]), - emqx_dashboard:init_i18n(), Config. end_per_suite(Config) -> diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index ee798868e..f6150e607 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -686,7 +686,6 @@ t_jq(_) -> %% Got timeout as expected got_timeout end, - _ConfigRootKey = emqx_rule_engine_schema:namespace(), ?assertThrow( {jq_exception, {timeout, _}}, apply_func(jq, [TOProgram, <<"-2">>]) diff --git a/build b/build index 3c558c19a..021cc80a5 100755 --- a/build +++ b/build @@ -117,8 +117,7 @@ make_docs() { mkdir -p "$docdir" "$dashboard_www_static" # shellcheck disable=SC2086 erl -noshell -pa $libs_dir1 $libs_dir2 $libs_dir3 -eval \ - "I18nFile = filename:join([apps, emqx_dashboard, priv, 'i18n.conf']), \ - ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE, I18nFile), \ + "ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE), \ halt(0)." cp "$docdir"/bridge-api-*.json "$dashboard_www_static" cp "$docdir"/hot-config-schema-*.json "$dashboard_www_static" diff --git a/mix.exs b/mix.exs index 2b8de4c54..6c6c7750f 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.38.1", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.1", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index 7e783b56d..040b1a7c0 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.1"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 7c00622c2..80f126096 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -479,8 +479,7 @@ etc_overlay(ReleaseType, Edition) -> [ {mkdir, "etc/"}, {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"}, - {copy, "_build/docgen/" ++ name(Edition) ++ "/emqx.conf.en.example", - "etc/emqx.conf.example"} + {copy, "_build/docgen/" ++ name(Edition) ++ "/emqx.conf.example", "etc/emqx.conf.example"} ] ++ lists:map( fun diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript index d30a0ca68..b3c214dd7 100755 --- a/scripts/merge-config.escript +++ b/scripts/merge-config.escript @@ -34,7 +34,10 @@ main(_) -> ok = file:write_file("apps/emqx_conf/etc/emqx-enterprise.conf.all", EnterpriseConf); false -> ok - end. + end, + merge_desc_files_per_lang("en"), + %% TODO: remove this when we have zh translation moved to dashboard package + merge_desc_files_per_lang("zh"). is_enterprise() -> Profile = os:getenv("PROFILE", "emqx"), @@ -96,3 +99,48 @@ try_enter_child(Dir, Files, Cfgs) -> true -> get_all_cfgs(filename:join([Dir, "src"]), Cfgs) end. + +%% Desc files merge is for now done locally in emqx.git repo for all languages. +%% When zh and other languages are moved to a separate repo, +%% we will only merge the en files. +%% The file for other languages will be merged in the other repo, +%% the built as a part of the dashboard package, +%% finally got pulled at build time as a part of the dashboard package. +merge_desc_files_per_lang(Lang) -> + BaseConf = <<"">>, + Cfgs0 = get_all_desc_files(Lang), + Conf = do_merge_desc_files_per_lang(BaseConf, Cfgs0), + OutputFile = case Lang of + "en" -> + %% en desc will always be in the priv dir of emqx_dashboard + "apps/emqx_dashboard/priv/desc.en.hocon"; + "zh" -> + %% so far we inject zh desc as if it's extracted from dashboard package + %% TODO: remove this when we have zh translation moved to dashboard package + "apps/emqx_dashboard/priv/www/static/desc.zh.hocon" + end, + ok = filelib:ensure_dir(OutputFile), + ok = file:write_file(OutputFile, Conf). + +do_merge_desc_files_per_lang(BaseConf, Cfgs) -> + lists:foldl( + fun(CfgFile, Acc) -> + case filelib:is_regular(CfgFile) of + true -> + {ok, Bin1} = file:read_file(CfgFile), + [Acc, io_lib:nl(), Bin1]; + false -> Acc + end + end, BaseConf, Cfgs). + +get_all_desc_files(Lang) -> + Dir = + case Lang of + "en" -> + filename:join(["rel", "i18n"]); + "zh" -> + %% TODO: remove this when we have zh translation moved to dashboard package + filename:join(["rel", "i18n", "zh"]) + end, + Files = filelib:wildcard("*.hocon", Dir), + lists:map(fun(Name) -> filename:join([Dir, Name]) end, Files). diff --git a/scripts/merge-i18n.escript b/scripts/merge-i18n.escript deleted file mode 100755 index 7ffd3aa8a..000000000 --- a/scripts/merge-i18n.escript +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env escript - --mode(compile). - -main(_) -> - main_per_lang("en"), - main_per_lang("zh"). - -main_per_lang(Lang) -> - BaseConf = <<"">>, - Cfgs0 = get_all_files(Lang), - Conf = merge(BaseConf, Cfgs0), - OutputFile = "apps/emqx_dashboard/priv/i18n." ++ Lang ++ ".conf", - ok = filelib:ensure_dir(OutputFile), - ok = file:write_file(OutputFile, Conf). - -merge(BaseConf, Cfgs) -> - lists:foldl( - fun(CfgFile, Acc) -> - case filelib:is_regular(CfgFile) of - true -> - {ok, Bin1} = file:read_file(CfgFile), - [Acc, io_lib:nl(), Bin1]; - false -> Acc - end - end, BaseConf, Cfgs). - -get_all_files(Lang) -> - Dir = - case Lang of - "en" -> filename:join(["rel", "i18n"]); - "zh" -> filename:join(["rel", "i18n", "zh"]) - end, - Files = filelib:wildcard("*.hocon", Dir), - lists:map(fun(Name) -> filename:join([Dir, Name]) end, Files). diff --git a/scripts/pre-compile.sh b/scripts/pre-compile.sh index 56b7d47b4..71251a03e 100755 --- a/scripts/pre-compile.sh +++ b/scripts/pre-compile.sh @@ -20,5 +20,4 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." ./scripts/get-dashboard.sh "$dashboard_version" ./scripts/merge-config.escript -./scripts/merge-i18n.escript ./scripts/update-bom.sh "$PROFILE_STR" ./rel From 8dc881f4b9f16c98a43d8800611e0a8ffbd0ad1a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 8 Apr 2023 09:41:55 +0200 Subject: [PATCH 182/279] build: fix mix build --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 6c6c7750f..e358edfcc 100644 --- a/mix.exs +++ b/mix.exs @@ -462,7 +462,7 @@ defmodule EMQXUmbrella.MixProject do profile = System.get_env("MIX_ENV") Mix.Generator.copy_file( - "_build/docgen/#{profile}/emqx.conf.en.example", + "_build/docgen/#{profile}/emqx.conf.example", Path.join(etc, "emqx.conf.example"), force: overwrite? ) From 56b966743638ffe3d52e5e42333265198797de46 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 8 Apr 2023 09:42:47 +0200 Subject: [PATCH 183/279] refactor(emqx_conf): prepare for dynamic api schema generation --- apps/emqx_conf/src/emqx_conf.erl | 44 ++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index d77ffb680..8632df139 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -29,6 +29,12 @@ -export([schema_module/0]). -export([gen_example_conf/2]). +%% TODO: move to emqx_dashboard when we stop building api schema at build time +-export([ + hotconf_schema_json/1, + bridge_schema_json/1 +]). + %% for rpc -export([get_node_and_config/1]). @@ -136,6 +142,7 @@ reset(Node, KeyPath, Opts) -> emqx_conf_proto_v2:reset(Node, KeyPath, Opts). %% @doc Called from build script. +%% TODO: move to a external escript after all refactoring is done dump_schema(Dir, SchemaModule) -> _ = application:load(emqx_dashboard), ok = emqx_dashboard_desc_cache:init(), @@ -169,19 +176,37 @@ gen_schema_json(Dir, SchemaModule, Lang) -> IoData = emqx_utils_json:encode(JsonMap, [pretty, force_utf8]), ok = file:write_file(SchemaJsonFile, IoData). +%% TODO: delete this function when we stop generating this JSON at build time. gen_api_schema_json(Dir, Lang) -> gen_api_schema_json_hotconf(Dir, Lang), gen_api_schema_json_bridge(Dir, Lang). +%% TODO: delete this function when we stop generating this JSON at build time. gen_api_schema_json_hotconf(Dir, Lang) -> - SchemaInfo = #{title => <<"EMQX Hot Conf API Schema">>, version => <<"0.1.0">>}, File = schema_filename(Dir, "hot-config-schema-", Lang), - ok = do_gen_api_schema_json(File, emqx_mgmt_api_configs, SchemaInfo, Lang). + IoData = hotconf_schema_json(Lang), + ok = write_api_schema_json_file(File, IoData). +%% TODO: delete this function when we stop generating this JSON at build time. gen_api_schema_json_bridge(Dir, Lang) -> - SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>}, File = schema_filename(Dir, "bridge-api-", Lang), - ok = do_gen_api_schema_json(File, emqx_bridge_api, SchemaInfo, Lang). + IoData = bridge_schema_json(Lang), + ok = write_api_schema_json_file(File, IoData). + +%% TODO: delete this function when we stop generating this JSON at build time. +write_api_schema_json_file(File, IoData) -> + io:format(user, "===< Generating: ~s~n", [File]), + file:write_file(File, IoData). + +%% TODO: move this function to emqx_dashboard when we stop generating this JSON at build time. +hotconf_schema_json(Lang) -> + SchemaInfo = #{title => <<"EMQX Hot Conf API Schema">>, version => <<"0.1.0">>}, + gen_api_schema_json_iodata(emqx_mgmt_api_configs, SchemaInfo, Lang). + +%% TODO: move this function to emqx_dashboard when we stop generating this JSON at build time. +bridge_schema_json(Lang) -> + SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>}, + gen_api_schema_json_iodata(emqx_bridge_api, SchemaInfo, Lang). schema_filename(Dir, Prefix, Lang) -> Filename = Prefix ++ Lang ++ ".json", @@ -245,9 +270,9 @@ gen_example(File, SchemaModule) -> Example = hocon_schema_example:gen(SchemaModule, Opts), file:write_file(File, Example). -%% Only gen hot_conf schema, not all configuration fields. -do_gen_api_schema_json(File, SchemaMod, SchemaInfo, Lang) -> - io:format(user, "===< Generating: ~s~n", [File]), +%% TODO: move this to emqx_dashboard when we stop generating +%% this JSON at build time. +gen_api_schema_json_iodata(SchemaMod, SchemaInfo, Lang) -> {ApiSpec0, Components0} = emqx_dashboard_swagger:spec( SchemaMod, #{ @@ -282,15 +307,14 @@ do_gen_api_schema_json(File, SchemaMod, SchemaInfo, Lang) -> ApiSpec0 ), Components = lists:foldl(fun(M, Acc) -> maps:merge(M, Acc) end, #{}, Components0), - IoData = emqx_utils_json:encode( + emqx_utils_json:encode( #{ info => SchemaInfo, paths => ApiSpec, components => #{schemas => Components} }, [pretty, force_utf8] - ), - file:write_file(File, IoData). + ). -define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])). -define(TO_COMPONENTS_SCHEMA(_M_, _F_), From 1aa5b528e9ba0a27f9977184c63cb277aa73dbc1 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 8 Apr 2023 11:05:16 +0200 Subject: [PATCH 184/279] feat: generate hotconf and bridge schema on the fly --- apps/emqx/include/http_api.hrl | 2 +- .../emqx_dashboard/src/emqx_dashboard_api.erl | 1 - .../src/emqx_dashboard_schema_api.erl | 84 +++++++++++++++++++ .../src/emqx_dashboard_swagger.erl | 10 ++- 4 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl diff --git a/apps/emqx/include/http_api.hrl b/apps/emqx/include/http_api.hrl index 08dd08362..6a5cefec1 100644 --- a/apps/emqx/include/http_api.hrl +++ b/apps/emqx/include/http_api.hrl @@ -57,7 +57,7 @@ -define(ERROR_CODES, [ {?BAD_USERNAME_OR_PWD, <<"Bad username or password">>}, {?BAD_API_KEY_OR_SECRET, <<"Bad API key or secret">>}, - {'BAD_REQUEST', <<"Request parameters are not legal">>}, + {'BAD_REQUEST', <<"Request parameters are not valid">>}, {'NOT_MATCH', <<"Conditions are not matched">>}, {'ALREADY_EXISTS', <<"Resource already existed">>}, {'BAD_CONFIG_SCHEMA', <<"Configuration data is not legal">>}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index d5655d99d..108cde379 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -18,7 +18,6 @@ -behaviour(minirest_api). --include("emqx_dashboard.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("typerefl/include/types.hrl"). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl new file mode 100644 index 000000000..898d95b3c --- /dev/null +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema_api.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% This module is for dashboard to retrieve the schema hot config and bridges. +-module(emqx_dashboard_schema_api). + +-behaviour(minirest_api). + +-include_lib("hocon/include/hoconsc.hrl"). + +%% minirest API +-export([api_spec/0, paths/0, schema/1]). + +-export([get_schema/2]). + +-define(TAGS, [<<"dashboard">>]). +-define(BAD_REQUEST, 'BAD_REQUEST'). + +%%-------------------------------------------------------------------- +%% minirest API and schema +%%-------------------------------------------------------------------- + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + ["/schemas/:name"]. + +%% This is a rather hidden API, so we don't need to add translations for the description. +schema("/schemas/:name") -> + #{ + 'operationId' => get_schema, + get => #{ + parameters => [ + {name, hoconsc:mk(hoconsc:enum([hotconf, bridges]), #{in => path})}, + {lang, + hoconsc:mk(typerefl:string(), #{ + in => query, + default => <<"en">>, + desc => <<"The language of the schema.">> + })} + ], + desc => << + "Get the schema JSON of the specified name. " + "NOTE: you should never need to make use of this API " + "unless you are building a multi-lang dashboaard." + >>, + tags => ?TAGS, + security => [], + responses => #{ + 200 => hoconsc:mk(binary(), #{desc => <<"The JSON schema of the specified name.">>}) + } + } + }. + +%%-------------------------------------------------------------------- +%% API Handler funcs +%%-------------------------------------------------------------------- + +get_schema(get, #{ + bindings := #{name := Name}, + query_string := #{<<"lang">> := Lang} +}) -> + {200, gen_schema(Name, iolist_to_binary(Lang))}; +get_schema(get, _) -> + {400, ?BAD_REQUEST, <<"unknown">>}. + +gen_schema(hotconf, Lang) -> + emqx_conf:hotconf_schema_json(Lang); +gen_schema(bridges, Lang) -> + emqx_conf:bridge_schema_json(Lang). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index bdd5866f8..e0b50346d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -85,7 +85,7 @@ check_schema => boolean() | filter(), translate_body => boolean(), schema_converter => fun((hocon_schema:schema(), Module :: atom()) -> map()), - i18n_lang => atom() + i18n_lang => atom() | string() | binary() }. -type route_path() :: string() | binary(). @@ -238,8 +238,12 @@ parse_spec_ref(Module, Path, Options) -> erlang:apply(Module, schema, [Path]) %% better error message catch - error:Reason -> - throw({error, #{mfa => {Module, schema, [Path]}, reason => Reason}}) + error:Reason:Stacktrace -> + erlang:raise( + error, + #{mfa => {Module, schema, [Path]}, reason => Reason}, + Stacktrace + ) end, {Specs, Refs} = maps:fold( fun(Method, Meta, {Acc, RefsAcc}) -> From 6969c2a670b0a6347c9a966668e2718a833546ce Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 8 Apr 2023 21:34:20 +0200 Subject: [PATCH 185/279] refactor: not leagal -> invalid --- apps/emqx/include/http_api.hrl | 6 +++--- .../emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/include/http_api.hrl b/apps/emqx/include/http_api.hrl index 6a5cefec1..ba1438374 100644 --- a/apps/emqx/include/http_api.hrl +++ b/apps/emqx/include/http_api.hrl @@ -57,16 +57,16 @@ -define(ERROR_CODES, [ {?BAD_USERNAME_OR_PWD, <<"Bad username or password">>}, {?BAD_API_KEY_OR_SECRET, <<"Bad API key or secret">>}, - {'BAD_REQUEST', <<"Request parameters are not valid">>}, + {'BAD_REQUEST', <<"Request parameters are invalid">>}, {'NOT_MATCH', <<"Conditions are not matched">>}, {'ALREADY_EXISTS', <<"Resource already existed">>}, - {'BAD_CONFIG_SCHEMA', <<"Configuration data is not legal">>}, + {'BAD_CONFIG_SCHEMA', <<"Configuration data is invalid">>}, {'BAD_LISTENER_ID', <<"Bad listener ID">>}, {'BAD_NODE_NAME', <<"Bad Node Name">>}, {'BAD_RPC', <<"RPC Failed. Check the cluster status and the requested node status">>}, {'BAD_TOPIC', <<"Topic syntax error, Topic needs to comply with the MQTT protocol standard">>}, {'EXCEED_LIMIT', <<"Create resources that exceed the maximum limit or minimum limit">>}, - {'INVALID_PARAMETER', <<"Request parameters is not legal and exceeds the boundary value">>}, + {'INVALID_PARAMETER', <<"Request parameters is invalid and exceeds the boundary value">>}, {'CONFLICT', <<"Conflicting request resources">>}, {'NO_DEFAULT_VALUE', <<"Request parameters do not use default values">>}, {'DEPENDENCY_EXISTS', <<"Resource is dependent by another resource">>}, diff --git a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl index c0a772d2d..588a69065 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_error_code_SUITE.erl @@ -57,7 +57,7 @@ t_look_up_code(_) -> t_description_code(_) -> {error, not_found} = emqx_dashboard_error_code:description('_____NOT_EXIST_NAME'), - {ok, <<"Request parameters are not legal">>} = + {ok, <<"Request parameters are invalid">>} = emqx_dashboard_error_code:description('BAD_REQUEST'), ok. @@ -79,7 +79,7 @@ t_api_code(_) -> Url = ?SERVER ++ "/error_codes/BAD_REQUEST", {ok, #{ <<"code">> := <<"BAD_REQUEST">>, - <<"description">> := <<"Request parameters are not legal">> + <<"description">> := <<"Request parameters are invalid">> }} = request(Url), ok. From 373e7b33f93b1ad0a4f4acb51f18e13838da54bc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 9 Apr 2023 23:40:49 +0200 Subject: [PATCH 186/279] test: allow tests to run without desc cache --- apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl b/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl index 8fd6fe3d3..9d8d1905d 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl @@ -60,7 +60,7 @@ lookup(Lang, Namespace, Id, Tag) -> %% @doc Lookup the description of the configuration item from the given cache. lookup(EtsTab, Lang0, Namespace, Id, Tag) -> Lang = bin(Lang0), - case ets:lookup(EtsTab, {Lang, bin(Namespace), bin(Id), bin(Tag)}) of + try ets:lookup(EtsTab, {Lang, bin(Namespace), bin(Id), bin(Tag)}) of [{_, Desc}] -> Desc; [] when Lang =/= <<"en">> -> @@ -69,6 +69,11 @@ lookup(EtsTab, Lang0, Namespace, Id, Tag) -> _ -> %% undefined but not <<>> undefined + catch + error:badarg -> + %% schema is not initialized + %% most likely in test cases + undefined end. %% The desc files are of names like: From 466a28daf25ca9788af9a4501ceaad0b9f1c6a2f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 8 Apr 2023 21:44:39 +0200 Subject: [PATCH 187/279] test: fix test cases to work with new exctption --- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 4 ++++ .../emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl | 7 +++++-- apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl | 6 +++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e0b50346d..f700ec146 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -239,6 +239,10 @@ parse_spec_ref(Module, Path, Options) -> %% better error message catch error:Reason:Stacktrace -> + %% raise a new error with the same stacktrace. + %% it's a bug if this happens. + %% i.e. if a path is listed in the spec but the module doesn't + %% implement it or crashes when trying to build the schema. erlang:raise( error, #{mfa => {Module, schema, [Path]}, reason => Reason}, diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index e6fa62f77..f2ba56e08 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -307,8 +307,11 @@ t_nest_ref(_Config) -> t_none_ref(_Config) -> Path = "/ref/none", - ?assertThrow( - {error, #{mfa := {?MODULE, schema, [Path]}}}, + ?assertError( + #{ + mfa := {?MODULE, schema, [Path]}, + reason := function_clause + }, emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}) ), ok. diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 753aaad7a..cda533cc2 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -277,11 +277,11 @@ t_bad_ref(_Config) -> t_none_ref(_Config) -> Path = "/ref/none", - ?assertThrow( - {error, #{ + ?assertError( + #{ mfa := {?MODULE, schema, ["/ref/none"]}, reason := function_clause - }}, + }, validate(Path, #{}, []) ), ok. From c03a94bae9bb60c578bc648f6fdd0970d1ceb675 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 17 Apr 2023 16:47:20 +0200 Subject: [PATCH 188/279] chore: e5.0.3-alpha.1 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 64fa12d3f..ea79dcd0e 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.0.22"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.0.2"). +-define(EMQX_RELEASE_EE, "5.0.3-alpha.1"). %% the HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 4b5382090..85bd20f6e 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.2 +version: 5.0.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.2 +appVersion: 5.0.3 From 9d15247dd54b6eb9565f0d019442971388c34d35 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 14 Apr 2023 17:33:25 -0300 Subject: [PATCH 189/279] feat(schema_registry): add support for protobuf schemas Fixes https://emqx.atlassian.net/browse/EMQX-9470 --- build | 5 +- changes/ee/feat-10409.en.md | 1 + .../include/emqx_ee_schema_registry.hrl | 21 +- lib-ee/emqx_ee_schema_registry/rebar.config | 3 +- .../src/emqx_ee_schema_registry.app.src | 3 +- .../src/emqx_ee_schema_registry.erl | 9 +- .../src/emqx_ee_schema_registry_schema.erl | 20 +- .../src/emqx_ee_schema_registry_serde.erl | 132 ++++++ .../test/emqx_ee_schema_registry_SUITE.erl | 390 +++++++++++++++--- ...emqx_ee_schema_registry_http_api_SUITE.erl | 88 +++- .../emqx_ee_schema_registry_serde_SUITE.erl | 53 ++- mix.exs | 2 +- rebar.config | 2 +- rel/i18n/emqx_ee_schema_registry_schema.hocon | 6 + .../zh/emqx_ee_schema_registry_schema.hocon | 6 + 15 files changed, 640 insertions(+), 101 deletions(-) create mode 100644 changes/ee/feat-10409.en.md diff --git a/build b/build index 021cc80a5..77d4dbfc8 100755 --- a/build +++ b/build @@ -124,10 +124,7 @@ make_docs() { } assert_no_compile_time_only_deps() { - if [ "$("$FIND" "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then - echo "gpb should not be included in the release" - exit 1 - fi + : } make_rel() { diff --git a/changes/ee/feat-10409.en.md b/changes/ee/feat-10409.en.md new file mode 100644 index 000000000..dfa9bfa76 --- /dev/null +++ b/changes/ee/feat-10409.en.md @@ -0,0 +1 @@ +Add support for [Protocol Buffers](https://protobuf.dev/) schemas in Schema Registry. diff --git a/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl b/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl index af49db6dd..058abf007 100644 --- a/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl +++ b/lib-ee/emqx_ee_schema_registry/include/emqx_ee_schema_registry.hrl @@ -10,14 +10,19 @@ -define(SCHEMA_REGISTRY_SHARD, emqx_ee_schema_registry_shard). -define(SERDE_TAB, emqx_ee_schema_registry_serde_tab). +-define(PROTOBUF_CACHE_TAB, emqx_ee_schema_registry_protobuf_cache_tab). -type schema_name() :: binary(). -type schema_source() :: binary(). -type encoded_data() :: iodata(). -type decoded_data() :: map(). --type serializer() :: fun((decoded_data()) -> encoded_data()). --type deserializer() :: fun((encoded_data()) -> decoded_data()). +-type serializer() :: + fun((decoded_data()) -> encoded_data()) + | fun((decoded_data(), term()) -> encoded_data()). +-type deserializer() :: + fun((encoded_data()) -> decoded_data()) + | fun((encoded_data(), term()) -> decoded_data()). -type destructor() :: fun(() -> ok). -type serde_type() :: avro. -type serde_opts() :: map(). @@ -29,6 +34,18 @@ destructor :: destructor() }). -type serde() :: #serde{}. + +-record(protobuf_cache, { + fingerprint, + module, + module_binary +}). +-type protobuf_cache() :: #protobuf_cache{ + fingerprint :: binary(), + module :: module(), + module_binary :: binary() +}. + -type serde_map() :: #{ name := schema_name(), serializer := serializer(), diff --git a/lib-ee/emqx_ee_schema_registry/rebar.config b/lib-ee/emqx_ee_schema_registry/rebar.config index 223ebf533..e42ff7278 100644 --- a/lib-ee/emqx_ee_schema_registry/rebar.config +++ b/lib-ee/emqx_ee_schema_registry/rebar.config @@ -4,7 +4,8 @@ {deps, [ {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}}, - {erlavro, {git, "https://github.com/klarna/erlavro.git", {tag, "2.9.8"}}} + {erlavro, {git, "https://github.com/klarna/erlavro.git", {tag, "2.9.8"}}}, + {gpb, "4.19.7"} ]}. {shell, [ diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src index c40fb808a..21f51b361 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.app.src @@ -6,7 +6,8 @@ {applications, [ kernel, stdlib, - erlavro + erlavro, + gpb ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl index 4c00903d5..59a224fc7 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry.erl @@ -176,7 +176,14 @@ create_tables() -> {record_name, serde}, {attributes, record_info(fields, serde)} ]), - ok = mria:wait_for_tables([?SERDE_TAB]), + ok = mria:create_table(?PROTOBUF_CACHE_TAB, [ + {type, set}, + {rlog_shard, ?SCHEMA_REGISTRY_SHARD}, + {storage, disc_only_copies}, + {record_name, protobuf_cache}, + {attributes, record_info(fields, protobuf_cache)} + ]), + ok = mria:wait_for_tables([?SERDE_TAB, ?PROTOBUF_CACHE_TAB]), ok. do_build_serdes(Schemas) -> diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl index bcdc63166..237ec706f 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_schema.erl @@ -53,10 +53,20 @@ fields(avro) -> mk(emqx_schema:json_binary(), #{required => true, desc => ?DESC("schema_source")})}, {description, mk(binary(), #{default => <<>>, desc => ?DESC("schema_description")})} ]; +fields(protobuf) -> + [ + {type, mk(protobuf, #{required => true, desc => ?DESC("schema_type")})}, + {source, mk(binary(), #{required => true, desc => ?DESC("schema_source")})}, + {description, mk(binary(), #{default => <<>>, desc => ?DESC("schema_description")})} + ]; fields("get_avro") -> [{name, mk(binary(), #{required => true, desc => ?DESC("schema_name")})} | fields(avro)]; +fields("get_protobuf") -> + [{name, mk(binary(), #{required => true, desc => ?DESC("schema_name")})} | fields(protobuf)]; fields("put_avro") -> fields(avro); +fields("put_protobuf") -> + fields(protobuf); fields("post_" ++ Type) -> fields("get_" ++ Type). @@ -64,6 +74,8 @@ desc(?CONF_KEY_ROOT) -> ?DESC("schema_registry_root"); desc(avro) -> ?DESC("avro_type"); +desc(protobuf) -> + ?DESC("protobuf_type"); desc(_) -> undefined. @@ -96,7 +108,7 @@ mk(Type, Meta) -> hoconsc:mk(Type, Meta). ref(Name) -> hoconsc:ref(?MODULE, Name). supported_serde_types() -> - [avro]. + [avro, protobuf]. refs() -> [ref(Type) || Type <- supported_serde_types()]. @@ -105,6 +117,8 @@ refs(#{<<"type">> := TypeAtom} = Value) when is_atom(TypeAtom) -> refs(Value#{<<"type">> := atom_to_binary(TypeAtom)}); refs(#{<<"type">> := <<"avro">>}) -> [ref(avro)]; +refs(#{<<"type">> := <<"protobuf">>}) -> + [ref(protobuf)]; refs(_) -> Expected = lists:join(" | ", [atom_to_list(T) || T <- supported_serde_types()]), throw(#{ @@ -113,12 +127,14 @@ refs(_) -> }). refs_get_api() -> - [ref("get_avro")]. + [ref("get_avro"), ref("get_protobuf")]. refs_get_api(#{<<"type">> := TypeAtom} = Value) when is_atom(TypeAtom) -> refs(Value#{<<"type">> := atom_to_binary(TypeAtom)}); refs_get_api(#{<<"type">> := <<"avro">>}) -> [ref("get_avro")]; +refs_get_api(#{<<"type">> := <<"protobuf">>}) -> + [ref("get_protobuf")]; refs_get_api(_) -> Expected = lists:join(" | ", [atom_to_list(T) || T <- supported_serde_types()]), throw(#{ diff --git a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl index 9835ec7c2..c65574032 100644 --- a/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl +++ b/lib-ee/emqx_ee_schema_registry/src/emqx_ee_schema_registry_serde.erl @@ -4,8 +4,11 @@ -module(emqx_ee_schema_registry_serde). -include("emqx_ee_schema_registry.hrl"). +-include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_ee_schema_registry_serde]}}]). + %% API -export([ decode/2, @@ -55,6 +58,27 @@ make_serde(avro, Name, Source0) -> ?tp(serde_destroyed, #{type => avro, name => Name}), ok end, + {Serializer, Deserializer, Destructor}; +make_serde(protobuf, Name, Source) -> + SerdeMod = make_protobuf_serde_mod(Name, Source), + Serializer = + fun(DecodedData0, MessageName0) -> + DecodedData = emqx_utils_maps:safe_atom_key_map(DecodedData0), + MessageName = binary_to_existing_atom(MessageName0, utf8), + SerdeMod:encode_msg(DecodedData, MessageName) + end, + Deserializer = + fun(EncodedData, MessageName0) -> + MessageName = binary_to_existing_atom(MessageName0, utf8), + Decoded = SerdeMod:decode_msg(EncodedData, MessageName), + emqx_utils_maps:binary_key_map(Decoded) + end, + Destructor = + fun() -> + unload_code(SerdeMod), + ?tp(serde_destroyed, #{type => protobuf, name => Name}), + ok + end, {Serializer, Deserializer, Destructor}. %%------------------------------------------------------------------------------ @@ -68,3 +92,111 @@ inject_avro_name(Name, Source0) -> Schema0 = emqx_utils_json:decode(Source0, [return_maps]), Schema = Schema0#{<<"name">> => Name}, emqx_utils_json:encode(Schema). + +-spec make_protobuf_serde_mod(schema_name(), schema_source()) -> module(). +make_protobuf_serde_mod(Name, Source) -> + {SerdeMod0, SerdeModFileName} = protobuf_serde_mod_name(Name), + case lazy_generate_protobuf_code(SerdeMod0, Source) of + {ok, SerdeMod, ModBinary} -> + load_code(SerdeMod, SerdeModFileName, ModBinary), + SerdeMod; + {error, #{error := Error, warnings := Warnings}} -> + ?SLOG( + warning, + #{ + msg => "error_generating_protobuf_code", + error => Error, + warnings => Warnings + } + ), + error({invalid_protobuf_schema, Error}) + end. + +-spec protobuf_serde_mod_name(schema_name()) -> {module(), string()}. +protobuf_serde_mod_name(Name) -> + %% must be a string (list) + SerdeModName = "$schema_parser_" ++ binary_to_list(Name), + SerdeMod = list_to_atom(SerdeModName), + %% the "path" to the module, for `code:load_binary'. + SerdeModFileName = SerdeModName ++ ".memory", + {SerdeMod, SerdeModFileName}. + +-spec lazy_generate_protobuf_code(module(), schema_source()) -> + {ok, module(), binary()} | {error, #{error := term(), warnings := [term()]}}. +lazy_generate_protobuf_code(SerdeMod0, Source) -> + %% We run this inside a transaction with locks to avoid running + %% the compile on all nodes; only one will get the lock, compile + %% the schema, and other nodes will simply read the final result. + {atomic, Res} = mria:transaction( + ?SCHEMA_REGISTRY_SHARD, + fun lazy_generate_protobuf_code_trans/2, + [SerdeMod0, Source] + ), + Res. + +-spec lazy_generate_protobuf_code_trans(module(), schema_source()) -> + {ok, module(), binary()} | {error, #{error := term(), warnings := [term()]}}. +lazy_generate_protobuf_code_trans(SerdeMod0, Source) -> + Fingerprint = erlang:md5(Source), + _ = mnesia:lock({record, ?PROTOBUF_CACHE_TAB, Fingerprint}, write), + case mnesia:read(?PROTOBUF_CACHE_TAB, Fingerprint) of + [#protobuf_cache{module = SerdeMod, module_binary = ModBinary}] -> + ?tp(schema_registry_protobuf_cache_hit, #{}), + {ok, SerdeMod, ModBinary}; + [] -> + ?tp(schema_registry_protobuf_cache_miss, #{}), + case generate_protobuf_code(SerdeMod0, Source) of + {ok, SerdeMod, ModBinary} -> + CacheEntry = #protobuf_cache{ + fingerprint = Fingerprint, + module = SerdeMod, + module_binary = ModBinary + }, + ok = mnesia:write(?PROTOBUF_CACHE_TAB, CacheEntry, write), + {ok, SerdeMod, ModBinary}; + {ok, SerdeMod, ModBinary, _Warnings} -> + CacheEntry = #protobuf_cache{ + fingerprint = Fingerprint, + module = SerdeMod, + module_binary = ModBinary + }, + ok = mnesia:write(?PROTOBUF_CACHE_TAB, CacheEntry, write), + {ok, SerdeMod, ModBinary}; + error -> + {error, #{error => undefined, warnings => []}}; + {error, Error} -> + {error, #{error => Error, warnings => []}}; + {error, Error, Warnings} -> + {error, #{error => Error, warnings => Warnings}} + end + end. + +generate_protobuf_code(SerdeMod, Source) -> + gpb_compile:string( + SerdeMod, + Source, + [ + binary, + strings_as_binaries, + {maps, true}, + %% Fixme: currently, some bug in `gpb' prevents this + %% option from working with `oneof' types... We're then + %% forced to use atom key maps. + %% {maps_key_type, binary}, + {maps_oneof, flat}, + {verify, always}, + {maps_unset_optional, omitted} + ] + ). + +-spec load_code(module(), string(), binary()) -> ok. +load_code(SerdeMod, SerdeModFileName, ModBinary) -> + _ = code:purge(SerdeMod), + {module, SerdeMod} = code:load_binary(SerdeMod, SerdeModFileName, ModBinary), + ok. + +-spec unload_code(module()) -> ok. +unload_code(SerdeMod) -> + _ = code:purge(SerdeMod), + _ = code:delete(SerdeMod), + ok. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl index 5bfba34b3..7ad01fa06 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_SUITE.erl @@ -21,11 +21,19 @@ %%------------------------------------------------------------------------------ all() -> - [{group, avro}]. + [{group, avro}, {group, protobuf}]. groups() -> - TCs = emqx_common_test_helpers:all(?MODULE), - [{avro, TCs}]. + AllTCs = emqx_common_test_helpers:all(?MODULE), + ProtobufOnlyTCs = protobuf_only_tcs(), + TCs = AllTCs -- ProtobufOnlyTCs, + [{avro, TCs}, {protobuf, AllTCs}]. + +protobuf_only_tcs() -> + [ + t_protobuf_union_encode, + t_protobuf_union_decode + ]. init_per_suite(Config) -> emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), @@ -38,6 +46,8 @@ end_per_suite(_Config) -> init_per_group(avro, Config) -> [{serde_type, avro} | Config]; +init_per_group(protobuf, Config) -> + [{serde_type, protobuf} | Config]; init_per_group(_Group, Config) -> Config. @@ -95,8 +105,12 @@ create_rule_http(RuleParams) -> Path = emqx_mgmt_api_test_util:api_path(["rules"]), AuthHeader = emqx_mgmt_api_test_util:auth_header_(), case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of - {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; - Error -> Error + {ok, Res0} -> + Res = #{<<"id">> := RuleId} = emqx_utils_json:decode(Res0, [return_maps]), + on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), + {ok, Res}; + Error -> + Error end. schema_params(avro) -> @@ -108,35 +122,174 @@ schema_params(avro) -> ] }, SourceBin = emqx_utils_json:encode(Source), - #{type => avro, source => SourceBin}. + #{type => avro, source => SourceBin}; +schema_params(protobuf) -> + SourceBin = + << + "message Person {\n" + " required string name = 1;\n" + " required int32 id = 2;\n" + " optional string email = 3;\n" + " }\n" + "message UnionValue {\n" + " oneof u {\n" + " int32 a = 1;\n" + " string b = 2;\n" + " }\n" + "}\n" + >>, + #{type => protobuf, source => SourceBin}. create_serde(SerdeType, SerdeName) -> Schema = schema_params(SerdeType), ok = emqx_ee_schema_registry:add_schema(SerdeName, Schema), ok. -sql_for(avro, encode_decode1) -> - << - "select\n" - " schema_decode('my_serde',\n" - " schema_encode('my_serde', json_decode(payload))) as decoded,\n" - " decoded.i as decoded_int,\n" - " decoded.s as decoded_string\n" - " from t" - >>; -sql_for(avro, encode1) -> - << - "select\n" - " schema_encode('my_serde', json_decode(payload)) as encoded\n" - " from t" - >>; -sql_for(avro, decode1) -> - << - "select\n" - " schema_decode('my_serde', payload) as decoded\n" - " from t" - >>; -sql_for(Type, Name) -> +test_params_for(avro, encode_decode1) -> + SQL = + << + "select\n" + " schema_decode('my_serde',\n" + " schema_encode('my_serde', json_decode(payload))) as decoded,\n\n" + " decoded.i as decoded_int,\n" + " decoded.s as decoded_string\n" + "from t\n" + >>, + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + ExpectedRuleOutput = + #{ + <<"decoded">> => #{<<"i">> => 10, <<"s">> => <<"text">>}, + <<"decoded_int">> => 10, + <<"decoded_string">> => <<"text">> + }, + ExtraArgs = [], + #{ + sql => SQL, + payload => Payload, + expected_rule_output => ExpectedRuleOutput, + extra_args => ExtraArgs + }; +test_params_for(avro, encode1) -> + SQL = + << + "select\n" + " schema_encode('my_serde', json_decode(payload)) as encoded\n" + "from t\n" + >>, + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + ExtraArgs = [], + #{ + sql => SQL, + payload => Payload, + extra_args => ExtraArgs + }; +test_params_for(avro, decode1) -> + SQL = + << + "select\n" + " schema_decode('my_serde', payload) as decoded\n" + "from t\n" + >>, + Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + ExtraArgs = [], + #{ + sql => SQL, + payload => Payload, + extra_args => ExtraArgs + }; +test_params_for(protobuf, encode_decode1) -> + SQL = + << + "select\n" + " schema_decode('my_serde',\n" + " schema_encode('my_serde', json_decode(payload), 'Person'),\n" + " 'Person') as decoded,\n" + " decoded.name as decoded_name,\n" + " decoded.email as decoded_email,\n" + " decoded.id as decoded_id\n" + "from t\n" + >>, + Payload = #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>}, + ExpectedRuleOutput = + #{ + <<"decoded">> => + #{ + <<"email">> => <<"emqx@emqx.io">>, + <<"id">> => 10, + <<"name">> => <<"some name">> + }, + <<"decoded_email">> => <<"emqx@emqx.io">>, + <<"decoded_id">> => 10, + <<"decoded_name">> => <<"some name">> + }, + ExtraArgs = [<<"Person">>], + #{ + sql => SQL, + payload => Payload, + extra_args => ExtraArgs, + expected_rule_output => ExpectedRuleOutput + }; +test_params_for(protobuf, decode1) -> + SQL = + << + "select\n" + " schema_decode('my_serde', payload, 'Person') as decoded\n" + "from t\n" + >>, + Payload = #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>}, + ExtraArgs = [<<"Person">>], + #{ + sql => SQL, + payload => Payload, + extra_args => ExtraArgs + }; +test_params_for(protobuf, encode1) -> + SQL = + << + "select\n" + " schema_encode('my_serde', json_decode(payload), 'Person') as encoded\n" + "from t\n" + >>, + Payload = #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>}, + ExtraArgs = [<<"Person">>], + #{ + sql => SQL, + payload => Payload, + extra_args => ExtraArgs + }; +test_params_for(protobuf, union1) -> + SQL = + << + "select\n" + " schema_decode('my_serde', payload, 'UnionValue') as decoded,\n" + " decoded.a as decoded_a,\n" + " decoded.b as decoded_b\n" + "from t\n" + >>, + PayloadA = #{<<"a">> => 10}, + PayloadB = #{<<"b">> => <<"string">>}, + ExtraArgs = [<<"UnionValue">>], + #{ + sql => SQL, + payload => #{a => PayloadA, b => PayloadB}, + extra_args => ExtraArgs + }; +test_params_for(protobuf, union2) -> + SQL = + << + "select\n" + " schema_encode('my_serde', json_decode(payload), 'UnionValue') as encoded\n" + "from t\n" + >>, + PayloadA = #{<<"a">> => 10}, + PayloadB = #{<<"b">> => <<"string">>}, + ExtraArgs = [<<"UnionValue">>], + #{ + sql => SQL, + payload => #{a => PayloadA, b => PayloadB}, + extra_args => ExtraArgs + }; +test_params_for(Type, Name) -> ct:fail("unimplemented: ~p", [{Type, Name}]). clear_schemas() -> @@ -238,6 +391,40 @@ wait_for_cluster_rpc(Node) -> true = is_pid(erpc:call(Node, erlang, whereis, [emqx_config_handler])) ). +serde_deletion_calls_destructor_spec(#{serde_type := SerdeType}, Trace) -> + ?assert( + ?strict_causality( + #{?snk_kind := will_delete_schema}, + #{?snk_kind := serde_destroyed, type := SerdeType}, + Trace + ) + ), + ok. + +protobuf_unique_cache_hit_spec(#{serde_type := protobuf} = Res, Trace) -> + #{nodes := Nodes} = Res, + CacheEvents = ?of_kind( + [ + schema_registry_protobuf_cache_hit, + schema_registry_protobuf_cache_miss + ], + Trace + ), + ?assertMatch( + [ + schema_registry_protobuf_cache_hit, + schema_registry_protobuf_cache_miss + ], + lists:sort(?projection(?snk_kind, CacheEvents)) + ), + ?assertEqual( + lists:usort(Nodes), + lists:usort([N || #{?snk_meta := #{node := N}} <- CacheEvents]) + ), + ok; +protobuf_unique_cache_hit_spec(_Res, _Trace) -> + ok. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -259,27 +446,16 @@ t_encode_decode(Config) -> SerdeType = ?config(serde_type, Config), SerdeName = my_serde, ok = create_serde(SerdeType, SerdeName), - {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode_decode1)}), - on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), - Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + #{ + sql := SQL, + payload := Payload, + expected_rule_output := ExpectedRuleOutput + } = test_params_for(SerdeType, encode_decode1), + {ok, _} = create_rule_http(#{sql => SQL}), PayloadBin = emqx_utils_json:encode(Payload), emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), Res = receive_action_results(), - ?assertMatch( - #{ - data := - #{ - <<"decoded">> := - #{ - <<"i">> := 10, - <<"s">> := <<"text">> - }, - <<"decoded_int">> := 10, - <<"decoded_string">> := <<"text">> - } - }, - Res - ), + ?assertMatch(#{data := ExpectedRuleOutput}, Res), ok. t_delete_serde(Config) -> @@ -308,9 +484,12 @@ t_encode(Config) -> SerdeType = ?config(serde_type, Config), SerdeName = my_serde, ok = create_serde(SerdeType, SerdeName), - {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, encode1)}), - on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), - Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + #{ + sql := SQL, + payload := Payload, + extra_args := ExtraArgs + } = test_params_for(SerdeType, encode1), + {ok, _} = create_rule_http(#{sql => SQL}), PayloadBin = emqx_utils_json:encode(Payload), emqx:publish(emqx_message:make(<<"t">>, PayloadBin)), Published = receive_published(?LINE), @@ -320,18 +499,21 @@ t_encode(Config) -> ), #{payload := #{<<"encoded">> := Encoded}} = Published, {ok, #{deserializer := Deserializer}} = emqx_ee_schema_registry:get_serde(SerdeName), - ?assertEqual(Payload, Deserializer(Encoded)), + ?assertEqual(Payload, apply(Deserializer, [Encoded | ExtraArgs])), ok. t_decode(Config) -> SerdeType = ?config(serde_type, Config), SerdeName = my_serde, ok = create_serde(SerdeType, SerdeName), - {ok, #{<<"id">> := RuleId}} = create_rule_http(#{sql => sql_for(SerdeType, decode1)}), - on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end), - Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, + #{ + sql := SQL, + payload := Payload, + extra_args := ExtraArgs + } = test_params_for(SerdeType, decode1), + {ok, _} = create_rule_http(#{sql => SQL}), {ok, #{serializer := Serializer}} = emqx_ee_schema_registry:get_serde(SerdeName), - EncodedBin = Serializer(Payload), + EncodedBin = apply(Serializer, [Payload | ExtraArgs]), emqx:publish(emqx_message:make(<<"t">>, EncodedBin)), Published = receive_published(?LINE), ?assertMatch( @@ -342,6 +524,76 @@ t_decode(Config) -> ?assertEqual(Payload, Decoded), ok. +t_protobuf_union_encode(Config) -> + SerdeType = ?config(serde_type, Config), + ?assertEqual(protobuf, SerdeType), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + #{ + sql := SQL, + payload := #{a := PayloadA, b := PayloadB}, + extra_args := ExtraArgs + } = test_params_for(SerdeType, union1), + {ok, _} = create_rule_http(#{sql => SQL}), + {ok, #{serializer := Serializer}} = emqx_ee_schema_registry:get_serde(SerdeName), + + EncodedBinA = apply(Serializer, [PayloadA | ExtraArgs]), + emqx:publish(emqx_message:make(<<"t">>, EncodedBinA)), + PublishedA = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"decoded">> := _}}, + PublishedA + ), + #{payload := #{<<"decoded">> := DecodedA}} = PublishedA, + ?assertEqual(PayloadA, DecodedA), + + EncodedBinB = apply(Serializer, [PayloadB | ExtraArgs]), + emqx:publish(emqx_message:make(<<"t">>, EncodedBinB)), + PublishedB = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"decoded">> := _}}, + PublishedB + ), + #{payload := #{<<"decoded">> := DecodedB}} = PublishedB, + ?assertEqual(PayloadB, DecodedB), + + ok. + +t_protobuf_union_decode(Config) -> + SerdeType = ?config(serde_type, Config), + ?assertEqual(protobuf, SerdeType), + SerdeName = my_serde, + ok = create_serde(SerdeType, SerdeName), + #{ + sql := SQL, + payload := #{a := PayloadA, b := PayloadB}, + extra_args := ExtraArgs + } = test_params_for(SerdeType, union2), + {ok, _} = create_rule_http(#{sql => SQL}), + {ok, #{deserializer := Deserializer}} = emqx_ee_schema_registry:get_serde(SerdeName), + + EncodedBinA = emqx_utils_json:encode(PayloadA), + emqx:publish(emqx_message:make(<<"t">>, EncodedBinA)), + PublishedA = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"encoded">> := _}}, + PublishedA + ), + #{payload := #{<<"encoded">> := EncodedA}} = PublishedA, + ?assertEqual(PayloadA, apply(Deserializer, [EncodedA | ExtraArgs])), + + EncodedBinB = emqx_utils_json:encode(PayloadB), + emqx:publish(emqx_message:make(<<"t">>, EncodedBinB)), + PublishedB = receive_published(?LINE), + ?assertMatch( + #{payload := #{<<"encoded">> := _}}, + PublishedB + ), + #{payload := #{<<"encoded">> := EncodedB}} = PublishedB, + ?assertEqual(PayloadB, apply(Deserializer, [EncodedB | ExtraArgs])), + + ok. + t_fail_rollback(Config) -> SerdeType = ?config(serde_type, Config), OkSchema = emqx_utils_maps:binary_key_map(schema_params(SerdeType)), @@ -369,6 +621,10 @@ t_cluster_serde_build(Config) -> Cluster = cluster(Config), SerdeName = my_serde, Schema = schema_params(SerdeType), + #{ + payload := Payload, + extra_args := ExtraArgs + } = test_params_for(SerdeType, encode_decode1), ?check_trace( begin Nodes = [N1, N2 | _] = start_cluster(Cluster), @@ -385,8 +641,14 @@ t_cluster_serde_build(Config) -> Res0 = emqx_ee_schema_registry:get_serde(SerdeName), ?assertMatch({ok, #{}}, Res0, #{node => N}), {ok, #{serializer := Serializer, deserializer := Deserializer}} = Res0, - Payload = #{<<"i">> => 10, <<"s">> => <<"text">>}, - ?assertEqual(Payload, Deserializer(Serializer(Payload)), #{node => N}), + ?assertEqual( + Payload, + apply( + Deserializer, + [apply(Serializer, [Payload | ExtraArgs]) | ExtraArgs] + ), + #{node => N} + ), ok end) end, @@ -417,17 +679,11 @@ t_cluster_serde_build(Config) -> end, Nodes ), - ok + #{serde_type => SerdeType, nodes => Nodes} end, - fun(Trace) -> - ?assert( - ?strict_causality( - #{?snk_kind := will_delete_schema}, - #{?snk_kind := serde_destroyed, type := SerdeType}, - Trace - ) - ), - ok - end + [ + {"destructor is always called", fun ?MODULE:serde_deletion_calls_destructor_spec/2}, + {"protobuf is only built on one node", fun ?MODULE:protobuf_unique_cache_hit_spec/2} + ] ), ok. diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl index e7034d562..ee6a693db 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_http_api_SUITE.erl @@ -19,7 +19,17 @@ %%------------------------------------------------------------------------------ all() -> - emqx_common_test_helpers:all(?MODULE). + [ + {group, avro}, + {group, protobuf} + ]. + +groups() -> + AllTCs = emqx_common_test_helpers:all(?MODULE), + [ + {avro, AllTCs}, + {protobuf, AllTCs} + ]. init_per_suite(Config) -> emqx_config:save_schema_mod_and_names(emqx_ee_schema_registry_schema), @@ -30,6 +40,48 @@ end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(lists:reverse(?APPS)), ok. +init_per_group(avro, Config) -> + Source = #{ + type => record, + fields => [ + #{name => <<"i">>, type => <<"int">>}, + #{name => <<"s">>, type => <<"string">>} + ] + }, + SourceBin = emqx_utils_json:encode(Source), + InvalidSourceBin = <<"{}">>, + [ + {serde_type, avro}, + {schema_source, SourceBin}, + {invalid_schema_source, InvalidSourceBin} + | Config + ]; +init_per_group(protobuf, Config) -> + SourceBin = + << + "message Person {\n" + " required string name = 1;\n" + " required int32 id = 2;\n" + " optional string email = 3;\n" + " }\n" + "message UnionValue {\n" + " oneof u {\n" + " int32 a = 1;\n" + " string b = 2;\n" + " }\n" + "}\n" + >>, + InvalidSourceBin = <<"xxxx">>, + [ + {serde_type, protobuf}, + {schema_source, SourceBin}, + {invalid_schema_source, InvalidSourceBin} + | Config + ]. + +end_per_group(_Group, _Config) -> + ok. + init_per_testcase(_TestCase, Config) -> clear_schemas(), ok = snabbkaffe:start_trace(), @@ -93,18 +145,14 @@ clear_schemas() -> %% Testcases %%------------------------------------------------------------------------------ -t_crud(_Config) -> - SchemaName = <<"my_avro_schema">>, - Source = #{ - type => record, - fields => [ - #{name => <<"i">>, type => <<"int">>}, - #{name => <<"s">>, type => <<"string">>} - ] - }, - SourceBin = emqx_utils_json:encode(Source), +t_crud(Config) -> + SerdeType = ?config(serde_type, Config), + SourceBin = ?config(schema_source, Config), + InvalidSourceBin = ?config(invalid_schema_source, Config), + SerdeTypeBin = atom_to_binary(SerdeType), + SchemaName = <<"my_schema">>, Params = #{ - <<"type">> => <<"avro">>, + <<"type">> => SerdeTypeBin, <<"source">> => SourceBin, <<"name">> => SchemaName, <<"description">> => <<"My schema">> @@ -138,7 +186,7 @@ t_crud(_Config) -> %% create a schema ?assertMatch( {ok, 201, #{ - <<"type">> := <<"avro">>, + <<"type">> := SerdeTypeBin, <<"source">> := SourceBin, <<"name">> := SchemaName, <<"description">> := <<"My schema">> @@ -147,7 +195,7 @@ t_crud(_Config) -> ), ?assertMatch( {ok, 200, #{ - <<"type">> := <<"avro">>, + <<"type">> := SerdeTypeBin, <<"source">> := SourceBin, <<"name">> := SchemaName, <<"description">> := <<"My schema">> @@ -157,7 +205,7 @@ t_crud(_Config) -> ?assertMatch( {ok, 200, [ #{ - <<"type">> := <<"avro">>, + <<"type">> := SerdeTypeBin, <<"source">> := SourceBin, <<"name">> := SchemaName, <<"description">> := <<"My schema">> @@ -168,7 +216,7 @@ t_crud(_Config) -> UpdateParams1 = UpdateParams#{<<"description">> := <<"My new schema">>}, ?assertMatch( {ok, 200, #{ - <<"type">> := <<"avro">>, + <<"type">> := SerdeTypeBin, <<"source">> := SourceBin, <<"name">> := SchemaName, <<"description">> := <<"My new schema">> @@ -188,9 +236,9 @@ t_crud(_Config) -> {ok, 400, #{ <<"code">> := <<"BAD_REQUEST">>, <<"message">> := - <<"{post_config_update,emqx_ee_schema_registry,{not_found,<<\"type\">>}}">> + <<"{post_config_update,emqx_ee_schema_registry,", _/binary>> }}, - request({put, SchemaName, UpdateParams#{<<"source">> := <<"{}">>}}) + request({put, SchemaName, UpdateParams#{<<"source">> := InvalidSourceBin}}) ), ?assertMatch( @@ -229,9 +277,9 @@ t_crud(_Config) -> {ok, 400, #{ <<"code">> := <<"BAD_REQUEST">>, <<"message">> := - <<"{post_config_update,emqx_ee_schema_registry,{not_found,<<\"type\">>}}">> + <<"{post_config_update,emqx_ee_schema_registry,", _/binary>> }}, - request({post, Params#{<<"source">> := <<"{}">>}}) + request({post, Params#{<<"source">> := InvalidSourceBin}}) ), %% unknown serde type diff --git a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl index 12798c6a2..1ab5e3c01 100644 --- a/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl +++ b/lib-ee/emqx_ee_schema_registry/test/emqx_ee_schema_registry_serde_SUITE.erl @@ -60,7 +60,25 @@ schema_params(avro) -> ] }, SourceBin = emqx_utils_json:encode(Source), - #{type => avro, source => SourceBin}. + #{type => avro, source => SourceBin}; +schema_params(protobuf) -> + SourceBin = + << + "\n" + " message Person {\n" + " required string name = 1;\n" + " required int32 id = 2;\n" + " optional string email = 3;\n" + " }\n" + " message UnionValue {\n" + " oneof u {\n" + " int32 a = 1;\n" + " string b = 2;\n" + " }\n" + " }\n" + " " + >>, + #{type => protobuf, source => SourceBin}. assert_roundtrip(SerdeName, Original) -> Encoded = emqx_ee_schema_registry_serde:encode(SerdeName, Original), @@ -119,3 +137,36 @@ t_serde_not_found(_Config) -> emqx_ee_schema_registry_serde:decode(NonexistentSerde, Original) ), ok. + +t_roundtrip_protobuf(_Config) -> + SerdeName = my_serde, + Params = schema_params(protobuf), + ok = emqx_ee_schema_registry:add_schema(SerdeName, Params), + ExtraArgsPerson = [<<"Person">>], + + Original0 = #{<<"name">> => <<"some name">>, <<"id">> => 10, <<"email">> => <<"emqx@emqx.io">>}, + assert_roundtrip(SerdeName, Original0, ExtraArgsPerson, ExtraArgsPerson), + + %% removing optional field + Original1 = #{<<"name">> => <<"some name">>, <<"id">> => 10}, + assert_roundtrip(SerdeName, Original1, ExtraArgsPerson, ExtraArgsPerson), + + %% `oneof' fields + ExtraArgsUnion = [<<"UnionValue">>], + Original2 = #{<<"a">> => 1}, + assert_roundtrip(SerdeName, Original2, ExtraArgsUnion, ExtraArgsUnion), + + Original3 = #{<<"b">> => <<"string">>}, + assert_roundtrip(SerdeName, Original3, ExtraArgsUnion, ExtraArgsUnion), + + ok. + +t_protobuf_invalid_schema(_Config) -> + SerdeName = my_serde, + Params = schema_params(protobuf), + WrongParams = Params#{source := <<"xxxx">>}, + ?assertMatch( + {error, {post_config_update, _, {invalid_protobuf_schema, _}}}, + emqx_ee_schema_registry:add_schema(SerdeName, WrongParams) + ), + ok. diff --git a/mix.exs b/mix.exs index e358edfcc..36ea9e157 100644 --- a/mix.exs +++ b/mix.exs @@ -94,7 +94,7 @@ defmodule EMQXUmbrella.MixProject do {:ranch, github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true}, # in conflict by grpc and eetcd - {:gpb, "4.19.5", override: true, runtime: false}, + {:gpb, "4.19.7", override: true, runtime: false}, {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true} ] ++ emqx_apps(profile_info, version) ++ diff --git a/rebar.config b/rebar.config index 040b1a7c0..edb544298 100644 --- a/rebar.config +++ b/rebar.config @@ -53,7 +53,7 @@ [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} , {redbug, "2.0.8"} , {covertool, {git, "https://github.com/zmstone/covertool", {tag, "2.0.4.1"}}} - , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps + , {gpb, "4.19.7"} , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.4.7"}}} diff --git a/rel/i18n/emqx_ee_schema_registry_schema.hocon b/rel/i18n/emqx_ee_schema_registry_schema.hocon index 667c4c0a4..3d1ce4072 100644 --- a/rel/i18n/emqx_ee_schema_registry_schema.hocon +++ b/rel/i18n/emqx_ee_schema_registry_schema.hocon @@ -6,6 +6,12 @@ avro_type.desc: avro_type.label: """Apache Avro""" +protobuf_type.desc: +"""[Protocol Buffers](https://protobuf.dev/) serialization format.""" + +protobuf_type.label: +"""Protocol Buffers""" + schema_description.desc: """A description for this schema.""" diff --git a/rel/i18n/zh/emqx_ee_schema_registry_schema.hocon b/rel/i18n/zh/emqx_ee_schema_registry_schema.hocon index 3bf0a7dc8..2f4c972ec 100644 --- a/rel/i18n/zh/emqx_ee_schema_registry_schema.hocon +++ b/rel/i18n/zh/emqx_ee_schema_registry_schema.hocon @@ -6,6 +6,12 @@ avro_type.desc: avro_type.label: """阿帕奇-阿夫罗""" +protobuf_type.desc: +"""[协议缓冲器](https://protobuf.dev/) 序列化格式。""" + +protobuf_type.label: +"""协议缓冲器""" + schema_description.desc: """对该模式的描述。""" From 3b5ac8321df01c65113162e593e88b8133c25708 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 17 Apr 2023 21:02:07 +0200 Subject: [PATCH 190/279] ci: bump emqx-builder to 5.0-34 to fix el9 build --- .github/workflows/build_and_push_docker_images.yaml | 4 ++-- .github/workflows/build_packages.yaml | 8 ++++---- .github/workflows/build_slim_packages.yaml | 2 +- .github/workflows/check_deps_integrity.yaml | 2 +- .github/workflows/code_style_check.yaml | 2 +- .github/workflows/elixir_apps_check.yaml | 2 +- .github/workflows/elixir_deps_check.yaml | 2 +- .github/workflows/elixir_release.yml | 2 +- .github/workflows/run_emqx_app_tests.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 6 +++--- .github/workflows/run_relup_tests.yaml | 2 +- .github/workflows/run_test_cases.yaml | 6 +++--- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 7391adb5c..7c0a5dc87 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -25,7 +25,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" outputs: PROFILE: ${{ steps.get_profile.outputs.PROFILE }} @@ -121,7 +121,7 @@ jobs: # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix builder: - - 5.0-33 # update to latest + - 5.0-34 # update to latest otp: - 24.3.4.2-3 # switch to 25 once ready to release 5.1 elixir: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 2afe23f67..60b4c1b84 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -24,7 +24,7 @@ jobs: prepare: runs-on: ubuntu-22.04 if: (github.repository_owner == 'emqx' && github.event_name == 'schedule') || github.event_name != 'schedule' - container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -221,7 +221,7 @@ jobs: - aws-arm64 - ubuntu-22.04 builder: - - 5.0-33 + - 5.0-34 elixir: - 1.13.4 exclude: @@ -235,7 +235,7 @@ jobs: arch: amd64 os: ubuntu22.04 build_machine: ubuntu-22.04 - builder: 5.0-33 + builder: 5.0-34 elixir: 1.13.4 release_with: elixir - profile: emqx @@ -243,7 +243,7 @@ jobs: arch: amd64 os: amzn2 build_machine: ubuntu-22.04 - builder: 5.0-33 + builder: 5.0-34 elixir: 1.13.4 release_with: elixir diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index d6e4fc961..9ae5ba944 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -35,7 +35,7 @@ jobs: - ["emqx-enterprise", "24.3.4.2-3", "amzn2", "erlang"] - ["emqx-enterprise", "25.1.2-3", "ubuntu20.04", "erlang"] builder: - - 5.0-33 + - 5.0-34 elixir: - '1.13.4' diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 62dfa24ef..52ebf9efc 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -6,7 +6,7 @@ on: jobs: check_deps_integrity: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 97c6b0c88..1508cdd6e 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 247f67a8f..7e942f3f3 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -9,7 +9,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-22.04 # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index d6449f563..e967c186b 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -8,7 +8,7 @@ on: jobs: elixir_deps_check: runs-on: ubuntu-22.04 - container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 steps: - name: Checkout diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 7bd6102ff..9a916d332 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -17,7 +17,7 @@ jobs: profile: - emqx - emqx-enterprise - container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04 + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-25.1.2-3-ubuntu22.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 0a15f6c0b..551d0d9e6 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: builder: - - 5.0-33 + - 5.0-34 otp: - 24.3.4.2-3 - 25.1.2-3 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index bb5aa4a1a..1bb0486f1 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -17,7 +17,7 @@ jobs: prepare: runs-on: ubuntu-22.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-debian11 + container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-debian11 steps: - uses: actions/checkout@v3 @@ -50,7 +50,7 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-33 + - 5.0-34 otp: - 24.3.4.2-3 elixir: @@ -123,7 +123,7 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-33 + - 5.0-34 otp: - 24.3.4.2-3 elixir: diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 8727f4d9d..2ad7c3345 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -15,7 +15,7 @@ concurrency: jobs: relup_test_plan: runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 8702cd849..fb4f264e7 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -31,12 +31,12 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.0-33", + builder: "5.0-34", otp: "25.1.2-3", elixir: "1.13.4" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.0-33", + builder: "5.0-34", otp: ["24.3.4.2-3", "25.1.2-3"][], elixir: "1.13.4" }) @@ -230,7 +230,7 @@ jobs: - ct - ct_docker runs-on: ubuntu-22.04 - container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04" + container: "ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 From 88ca94b4171c4880013f15312fed2d33541d4f8e Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 17 Apr 2023 20:00:57 +0300 Subject: [PATCH 191/279] fix(auth): fix uri path handling Fix uri path handling `emqx_connector_http`, HTTP authentication and authorization backends. --- apps/emqx_authn/src/emqx_authn_utils.erl | 13 ++++ .../src/simple_authn/emqx_authn_http.erl | 14 ++-- .../emqx_authn/test/emqx_authn_http_SUITE.erl | 49 +++++++++++++- apps/emqx_authz/src/emqx_authz_http.erl | 14 ++-- apps/emqx_authz/src/emqx_authz_utils.erl | 14 +++- .../emqx_authz/test/emqx_authz_http_SUITE.erl | 10 +-- .../src/emqx_connector_http.erl | 65 +++++++++++++++---- changes/ce/fix-10420.en.md | 3 + changes/ce/fix-10420.zh.md | 0 9 files changed, 146 insertions(+), 36 deletions(-) create mode 100644 changes/ce/fix-10420.en.md create mode 100644 changes/ce/fix-10420.zh.md diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl index 1352e3daf..12520251e 100644 --- a/apps/emqx_authn/src/emqx_authn_utils.erl +++ b/apps/emqx_authn/src/emqx_authn_utils.erl @@ -28,6 +28,7 @@ parse_sql/2, render_deep/2, render_str/2, + render_urlencoded_str/2, render_sql_params/2, is_superuser/1, bin/1, @@ -129,6 +130,13 @@ render_str(Template, Credential) -> #{return => full_binary, var_trans => fun handle_var/2} ). +render_urlencoded_str(Template, Credential) -> + emqx_placeholder:proc_tmpl( + Template, + mapping_credential(Credential), + #{return => full_binary, var_trans => fun urlencode_var/2} + ). + render_sql_params(ParamList, Credential) -> emqx_placeholder:proc_tmpl( ParamList, @@ -217,6 +225,11 @@ without_password(Credential, [Name | Rest]) -> without_password(Credential, Rest) end. +urlencode_var({var, _} = Var, Value) -> + emqx_http_lib:uri_encode(handle_var(Var, Value)); +urlencode_var(Var, Value) -> + handle_var(Var, Value). + handle_var({var, _Name}, undefined) -> <<>>; handle_var({var, <<"peerhost">>}, PeerHost) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 3c34d878e..502562e2c 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -285,9 +285,9 @@ parse_url(Url) -> BaseUrl = iolist_to_binary([Scheme, "//", HostPort]), case string:split(Remaining, "?", leading) of [Path, QueryString] -> - {BaseUrl, Path, QueryString}; + {BaseUrl, <<"/", Path/binary>>, QueryString}; [Path] -> - {BaseUrl, Path, <<>>} + {BaseUrl, <<"/", Path/binary>>, <<>>} end; [HostPort] -> {iolist_to_binary([Scheme, "//", HostPort]), <<>>, <<>>} @@ -328,7 +328,7 @@ generate_request(Credential, #{ body_template := BodyTemplate }) -> Headers = maps:to_list(Headers0), - Path = emqx_authn_utils:render_str(BasePathTemplate, Credential), + Path = emqx_authn_utils:render_urlencoded_str(BasePathTemplate, Credential), Query = emqx_authn_utils:render_deep(BaseQueryTemplate, Credential), Body = emqx_authn_utils:render_deep(BodyTemplate, Credential), case Method of @@ -343,9 +343,9 @@ generate_request(Credential, #{ end. append_query(Path, []) -> - encode_path(Path); + Path; append_query(Path, Query) -> - encode_path(Path) ++ "?" ++ binary_to_list(qs(Query)). + Path ++ "?" ++ binary_to_list(qs(Query)). qs(KVs) -> qs(KVs, []). @@ -407,10 +407,6 @@ parse_body(ContentType, _) -> uri_encode(T) -> emqx_http_lib:uri_encode(to_list(T)). -encode_path(Path) -> - Parts = string:split(Path, "/", all), - lists:flatten(["/" ++ Part || Part <- lists:map(fun uri_encode/1, Parts)]). - request_for_log(Credential, #{url := Url} = State) -> SafeCredential = emqx_authn_utils:without_password(Credential), case generate_request(SafeCredential, State) of diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl index 851e80f6d..b08167a5b 100644 --- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl @@ -47,7 +47,6 @@ }) ). --define(SERVER_RESPONSE_URLENCODE(Result), ?SERVER_RESPONSE_URLENCODE(Result, false)). -define(SERVER_RESPONSE_URLENCODE(Result, IsSuperuser), list_to_binary( "result=" ++ @@ -166,6 +165,54 @@ test_user_auth(#{ ?GLOBAL ). +t_authenticate_path_placeholders(_Config) -> + ok = emqx_authn_http_test_server:stop(), + {ok, _} = emqx_authn_http_test_server:start_link(?HTTP_PORT, <<"/[...]">>), + ok = emqx_authn_http_test_server:set_handler( + fun(Req0, State) -> + Req = + case cowboy_req:path(Req0) of + <<"/my/p%20ath//us%20er/auth//">> -> + cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + emqx_utils_json:encode(#{result => allow, is_superuser => false}), + Req0 + ); + Path -> + ct:pal("Unexpected path: ~p", [Path]), + cowboy_req:reply(403, Req0) + end, + {ok, Req, State} + end + ), + + Credentials = ?CREDENTIALS#{ + username => <<"us er">> + }, + + AuthConfig = maps:merge( + raw_http_auth_config(), + #{ + <<"url">> => <<"http://127.0.0.1:32333/my/p%20ath//${username}/auth//">>, + <<"body">> => #{} + } + ), + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig} + ), + + ?assertMatch( + {ok, #{is_superuser := false}}, + emqx_access_control:authenticate(Credentials) + ), + + _ = emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL + ). + t_no_value_for_placeholder(_Config) -> Handler = fun(Req0, State) -> {ok, RawBody, Req1} = cowboy_req:read_body(Req0), diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 53378d9c2..5747e6eeb 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -161,9 +161,9 @@ parse_url(Url) -> BaseUrl = iolist_to_binary([Scheme, "//", HostPort]), case string:split(Remaining, "?", leading) of [Path, QueryString] -> - {BaseUrl, Path, QueryString}; + {BaseUrl, <<"/", Path/binary>>, QueryString}; [Path] -> - {BaseUrl, Path, <<>>} + {BaseUrl, <<"/", Path/binary>>, <<>>} end; [HostPort] -> {iolist_to_binary([Scheme, "//", HostPort]), <<>>, <<>>} @@ -185,7 +185,7 @@ generate_request( } ) -> Values = client_vars(Client, PubSub, Topic), - Path = emqx_authz_utils:render_str(BasePathTemplate, Values), + Path = emqx_authz_utils:render_urlencoded_str(BasePathTemplate, Values), Query = emqx_authz_utils:render_deep(BaseQueryTemplate, Values), Body = emqx_authz_utils:render_deep(BodyTemplate, Values), case Method of @@ -202,9 +202,9 @@ generate_request( end. append_query(Path, []) -> - encode_path(Path); + to_list(Path); append_query(Path, Query) -> - encode_path(Path) ++ "?" ++ to_list(query_string(Query)). + to_list(Path) ++ "?" ++ to_list(query_string(Query)). query_string(Body) -> query_string(Body, []). @@ -222,10 +222,6 @@ query_string([{K, V} | More], Acc) -> uri_encode(T) -> emqx_http_lib:uri_encode(to_list(T)). -encode_path(Path) -> - Parts = string:split(Path, "/", all), - lists:flatten(["/" ++ Part || Part <- lists:map(fun uri_encode/1, Parts)]). - serialize_body(<<"application/json">>, Body) -> emqx_utils_json:encode(Body); serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index 560141d0a..c01505680 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -16,7 +16,6 @@ -module(emqx_authz_utils). --include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx_authz.hrl"). -export([ @@ -28,6 +27,7 @@ update_config/2, parse_deep/2, parse_str/2, + render_urlencoded_str/2, parse_sql/3, render_deep/2, render_str/2, @@ -128,6 +128,13 @@ render_str(Template, Values) -> #{return => full_binary, var_trans => fun handle_var/2} ). +render_urlencoded_str(Template, Values) -> + emqx_placeholder:proc_tmpl( + Template, + client_vars(Values), + #{return => full_binary, var_trans => fun urlencode_var/2} + ). + render_sql_params(ParamList, Values) -> emqx_placeholder:proc_tmpl( ParamList, @@ -181,6 +188,11 @@ convert_client_var({dn, DN}) -> {cert_subject, DN}; convert_client_var({protocol, Proto}) -> {proto_name, Proto}; convert_client_var(Other) -> Other. +urlencode_var({var, _} = Var, Value) -> + emqx_http_lib:uri_encode(handle_var(Var, Value)); +urlencode_var(Var, Value) -> + handle_var(Var, Value). + handle_var({var, _Name}, undefined) -> <<>>; handle_var({var, <<"peerhost">>}, IpAddr) -> diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index 9ff84b805..702bf2756 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -199,7 +199,7 @@ t_query_params(_Config) -> peerhost := <<"127.0.0.1">>, proto_name := <<"MQTT">>, mountpoint := <<"MOUNTPOINT">>, - topic := <<"t">>, + topic := <<"t/1">>, action := <<"publish">> } = cowboy_req:match_qs( [ @@ -241,7 +241,7 @@ t_query_params(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, publish, <<"t/1">>) ). t_path(_Config) -> @@ -249,13 +249,13 @@ t_path(_Config) -> fun(Req0, State) -> ?assertEqual( << - "/authz/users/" + "/authz/use%20rs/" "user%20name/" "client%20id/" "127.0.0.1/" "MQTT/" "MOUNTPOINT/" - "t/1/" + "t%2F1/" "publish" >>, cowboy_req:path(Req0) @@ -264,7 +264,7 @@ t_path(_Config) -> end, #{ <<"url">> => << - "http://127.0.0.1:33333/authz/users/" + "http://127.0.0.1:33333/authz/use%20rs/" "${username}/" "${clientid}/" "${peerhost}/" diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index ef2e11eb7..610632e36 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -47,7 +47,7 @@ namespace/0 ]). --export([check_ssl_opts/2, validate_method/1]). +-export([check_ssl_opts/2, validate_method/1, join_paths/2]). -type connect_timeout() :: emqx_schema:duration() | infinity. -type pool_type() :: random | hash. @@ -458,7 +458,7 @@ preprocess_request( } = Req ) -> #{ - method => emqx_plugin_libs_rule:preproc_tmpl(bin(Method)), + method => emqx_plugin_libs_rule:preproc_tmpl(to_bin(Method)), path => emqx_plugin_libs_rule:preproc_tmpl(Path), body => maybe_preproc_tmpl(body, Req), headers => preproc_headers(Headers), @@ -471,8 +471,8 @@ preproc_headers(Headers) when is_map(Headers) -> fun(K, V, Acc) -> [ { - emqx_plugin_libs_rule:preproc_tmpl(bin(K)), - emqx_plugin_libs_rule:preproc_tmpl(bin(V)) + emqx_plugin_libs_rule:preproc_tmpl(to_bin(K)), + emqx_plugin_libs_rule:preproc_tmpl(to_bin(V)) } | Acc ] @@ -484,8 +484,8 @@ preproc_headers(Headers) when is_list(Headers) -> lists:map( fun({K, V}) -> { - emqx_plugin_libs_rule:preproc_tmpl(bin(K)), - emqx_plugin_libs_rule:preproc_tmpl(bin(V)) + emqx_plugin_libs_rule:preproc_tmpl(to_bin(K)), + emqx_plugin_libs_rule:preproc_tmpl(to_bin(V)) } end, Headers @@ -553,15 +553,41 @@ formalize_request(Method, BasePath, {Path, Headers, _Body}) when -> formalize_request(Method, BasePath, {Path, Headers}); formalize_request(_Method, BasePath, {Path, Headers, Body}) -> - {filename:join(BasePath, Path), Headers, Body}; + {join_paths(BasePath, Path), Headers, Body}; formalize_request(_Method, BasePath, {Path, Headers}) -> - {filename:join(BasePath, Path), Headers}. + {join_paths(BasePath, Path), Headers}. -bin(Bin) when is_binary(Bin) -> +%% By default, we cannot treat HTTP paths as "file" or "resource" paths, +%% because an HTTP server may handle paths like +%% "/a/b/c/", "/a/b/c" and "/a//b/c" differently. +%% +%% So we try to avoid unneccessary path normalization. +%% +%% See also: `join_paths_test_/0` +join_paths(Path1, Path2) -> + do_join_paths(lists:reverse(to_list(Path1)), to_list(Path2)). + +%% "abc/" + "/cde" +do_join_paths([$/ | Path1], [$/ | Path2]) -> + lists:reverse(Path1) ++ [$/ | Path2]; +%% "abc/" + "cde" +do_join_paths([$/ | Path1], Path2) -> + lists:reverse(Path1) ++ [$/ | Path2]; +%% "abc" + "/cde" +do_join_paths(Path1, [$/ | Path2]) -> + lists:reverse(Path1) ++ [$/ | Path2]; +%% "abc" + "cde" +do_join_paths(Path1, Path2) -> + lists:reverse(Path1) ++ [$/ | Path2]. + +to_list(List) when is_list(List) -> List; +to_list(Bin) when is_binary(Bin) -> binary_to_list(Bin). + +to_bin(Bin) when is_binary(Bin) -> Bin; -bin(Str) when is_list(Str) -> +to_bin(Str) when is_list(Str) -> list_to_binary(Str); -bin(Atom) when is_atom(Atom) -> +to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). reply_delegator(ReplyFunAndArgs, Result) -> @@ -642,4 +668,21 @@ redact_test_() -> ?_assertNotEqual(TestData2, redact(TestData2)) ]. +join_paths_test_() -> + [ + ?_assertEqual("abc/cde", join_paths("abc", "cde")), + ?_assertEqual("abc/cde", join_paths("abc", "/cde")), + ?_assertEqual("abc/cde", join_paths("abc/", "cde")), + ?_assertEqual("abc/cde", join_paths("abc/", "/cde")), + + ?_assertEqual("/", join_paths("", "")), + ?_assertEqual("/cde", join_paths("", "cde")), + ?_assertEqual("/cde", join_paths("", "/cde")), + ?_assertEqual("/cde", join_paths("/", "cde")), + ?_assertEqual("/cde", join_paths("/", "/cde")), + + ?_assertEqual("//cde/", join_paths("/", "//cde/")), + ?_assertEqual("abc///cde/", join_paths("abc//", "//cde/")) + ]. + -endif. diff --git a/changes/ce/fix-10420.en.md b/changes/ce/fix-10420.en.md new file mode 100644 index 000000000..70afd8b45 --- /dev/null +++ b/changes/ce/fix-10420.en.md @@ -0,0 +1,3 @@ +Fix HTTP path handling when composing the URL for the HTTP requests in authentication and authorization modules. +* Avoid unnecessary URL normalization since we cannot assume that external servers treat original and normalized URLs equally. This led to bugs like [#10411](https://github.com/emqx/emqx/issues/10411). +* Fix the issue that path segments could be HTTP encoded twice. diff --git a/changes/ce/fix-10420.zh.md b/changes/ce/fix-10420.zh.md new file mode 100644 index 000000000..e69de29bb From 933e6727ba492067d7dd9d08ce2a30b820b11cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Sun, 9 Apr 2023 21:35:46 +0800 Subject: [PATCH 192/279] feat: node array support array(atom()) and comma_separated_atoms --- apps/emqx_conf/src/emqx_conf_schema.erl | 7 +++- .../emqx_conf/test/emqx_conf_schema_tests.erl | 40 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 362fc1587..ae3691682 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -135,7 +135,7 @@ fields("cluster") -> )}, {"core_nodes", sc( - emqx_schema:comma_separated_atoms(), + node_array(), #{ mapping => "mria.core_nodes", default => [], @@ -203,7 +203,7 @@ fields(cluster_static) -> [ {"seeds", sc( - hoconsc:array(atom()), + node_array(), #{ default => [], desc => ?DESC(cluster_static_seeds), @@ -1312,3 +1312,6 @@ validator_string_re(Val, RE, Error) -> catch _:_ -> {error, Error} end. + +node_array() -> + hoconsc:union([emqx_schema:comma_separated_atoms(), hoconsc:array(atom())]). diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 3653b9d19..453aca7a8 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -5,6 +5,46 @@ -module(emqx_conf_schema_tests). -include_lib("eunit/include/eunit.hrl"). +array_nodes_test() -> + ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], + BaseConf = + "" + "\n" + " node {\n" + " name = \"emqx1@127.0.0.1\"\n" + " cookie = \"emqxsecretcookie\"\n" + " data_dir = \"data\"\n" + " }\n" + " cluster {\n" + " name = emqxcl\n" + " discovery_strategy = static\n" + " static.seeds = ~p\n" + " core_nodes = ~p\n" + " }\n" + " " + "", + lists:foreach( + fun(Nodes) -> + ConfFile = iolist_to_binary(io_lib:format(BaseConf, [Nodes, Nodes])), + {ok, Conf} = hocon:binary(ConfFile, #{format => richmap}), + ConfList = hocon_tconf:generate(emqx_conf_schema, Conf), + ClusterDiscovery = proplists:get_value( + cluster_discovery, proplists:get_value(ekka, ConfList) + ), + ?assertEqual( + {static, [{seeds, ExpectNodes}]}, + ClusterDiscovery, + Nodes + ), + ?assertEqual( + ExpectNodes, + proplists:get_value(core_nodes, proplists:get_value(mria, ConfList)), + Nodes + ) + end, + [["emqx1@127.0.0.1", "emqx2@127.0.0.1"], "emqx1@127.0.0.1, emqx2@127.0.0.1"] + ), + ok. doc_gen_test() -> %% the json file too large to encode. From 1ef74f1adac67566e62bd29434626dcbc9bdfc6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Thu, 13 Apr 2023 20:19:39 +0800 Subject: [PATCH 193/279] chore: upgrade hocon to 0.39.1 --- changes/ce/feat-10389.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/feat-10389.en.md diff --git a/changes/ce/feat-10389.en.md b/changes/ce/feat-10389.en.md new file mode 100644 index 000000000..51d48bdfb --- /dev/null +++ b/changes/ce/feat-10389.en.md @@ -0,0 +1,2 @@ +Now `cluster.core_nodes` and `cluster.statics.seeds` are specified in the same way. +configure them as `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` and `"emqx1@127.0.0.1,emqx2@127.0.0.1"` has the same effect. From ad3e529994d22cc6519c7cc73d080460980897da Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 13 Apr 2023 21:02:37 +0800 Subject: [PATCH 194/279] chore: update changes/ce/feat-10389.en.md Co-authored-by: Thales Macedo Garitezi --- apps/emqx/rebar.config | 2 +- changes/ce/feat-10389.en.md | 2 +- mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 5945ccc7c..6788b4f40 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/changes/ce/feat-10389.en.md b/changes/ce/feat-10389.en.md index 51d48bdfb..d6b8d236f 100644 --- a/changes/ce/feat-10389.en.md +++ b/changes/ce/feat-10389.en.md @@ -1,2 +1,2 @@ Now `cluster.core_nodes` and `cluster.statics.seeds` are specified in the same way. -configure them as `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` and `"emqx1@127.0.0.1,emqx2@127.0.0.1"` has the same effect. +Configuring them as `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` and `"emqx1@127.0.0.1,emqx2@127.0.0.1"` has the same effect. diff --git a/mix.exs b/mix.exs index 36ea9e157..1dbde8980 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.7", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.1", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.2", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index edb544298..de520f124 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.7"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.1"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.2"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From 7eacbffae9a1c3c2536f57403ffbf71068c090b9 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 14 Apr 2023 17:16:50 +0800 Subject: [PATCH 195/279] chore: create dirs and README template for all ee bridges --- apps/emqx_bridge_cassandra/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_cassandra/README.md | 39 +++++++++++ apps/emqx_bridge_clickhouse/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_clickhouse/README.md | 37 +++++++++++ apps/emqx_bridge_dynamo/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_dynamo/README.md | 40 ++++++++++++ apps/emqx_bridge_gcp_pubsub/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_gcp_pubsub/README.md | 35 ++++++++++ apps/emqx_bridge_hstreamdb/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_hstreamdb/README.md | 38 +++++++++++ apps/emqx_bridge_influxdb/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_influxdb/README.md | 49 ++++++++++++++ apps/emqx_bridge_matrix/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_matrix/README.md | 36 ++++++++++ apps/emqx_bridge_mongodb/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_mongodb/README.md | 39 +++++++++++ apps/emqx_bridge_mysql/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_mysql/README.md | 36 ++++++++++ apps/emqx_bridge_pgsql/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_pgsql/README.md | 38 +++++++++++ apps/emqx_bridge_redis/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_redis/README.md | 37 +++++++++++ apps/emqx_bridge_rocketmq/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_rocketmq/README.md | 37 +++++++++++ apps/emqx_bridge_tdengine/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_tdengine/README.md | 39 +++++++++++ apps/emqx_bridge_timescale/BSL.txt | 94 +++++++++++++++++++++++++++ apps/emqx_bridge_timescale/README.md | 38 +++++++++++ 28 files changed, 1854 insertions(+) create mode 100644 apps/emqx_bridge_cassandra/BSL.txt create mode 100644 apps/emqx_bridge_cassandra/README.md create mode 100644 apps/emqx_bridge_clickhouse/BSL.txt create mode 100644 apps/emqx_bridge_clickhouse/README.md create mode 100644 apps/emqx_bridge_dynamo/BSL.txt create mode 100644 apps/emqx_bridge_dynamo/README.md create mode 100644 apps/emqx_bridge_gcp_pubsub/BSL.txt create mode 100644 apps/emqx_bridge_gcp_pubsub/README.md create mode 100644 apps/emqx_bridge_hstreamdb/BSL.txt create mode 100644 apps/emqx_bridge_hstreamdb/README.md create mode 100644 apps/emqx_bridge_influxdb/BSL.txt create mode 100644 apps/emqx_bridge_influxdb/README.md create mode 100644 apps/emqx_bridge_matrix/BSL.txt create mode 100644 apps/emqx_bridge_matrix/README.md create mode 100644 apps/emqx_bridge_mongodb/BSL.txt create mode 100644 apps/emqx_bridge_mongodb/README.md create mode 100644 apps/emqx_bridge_mysql/BSL.txt create mode 100644 apps/emqx_bridge_mysql/README.md create mode 100644 apps/emqx_bridge_pgsql/BSL.txt create mode 100644 apps/emqx_bridge_pgsql/README.md create mode 100644 apps/emqx_bridge_redis/BSL.txt create mode 100644 apps/emqx_bridge_redis/README.md create mode 100644 apps/emqx_bridge_rocketmq/BSL.txt create mode 100644 apps/emqx_bridge_rocketmq/README.md create mode 100644 apps/emqx_bridge_tdengine/BSL.txt create mode 100644 apps/emqx_bridge_tdengine/README.md create mode 100644 apps/emqx_bridge_timescale/BSL.txt create mode 100644 apps/emqx_bridge_timescale/README.md diff --git a/apps/emqx_bridge_cassandra/BSL.txt b/apps/emqx_bridge_cassandra/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_cassandra/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_cassandra/README.md b/apps/emqx_bridge_cassandra/README.md new file mode 100644 index 000000000..fbf57660d --- /dev/null +++ b/apps/emqx_bridge_cassandra/README.md @@ -0,0 +1,39 @@ +# EMQX Cassandra Bridge + +[Apache Cassandra](https://github.com/apache/cassandra) is an open-source, distributed +NoSQL database management system that is designed to manage large amounts of structured +and semi-structured data across many commodity servers, providing high availability +with no single point of failure. +It is commonly used in web and mobile applications, IoT, and other systems that +require storing, querying, and analyzing large amounts of data. + +The application is used to connect EMQX and Cassandra. User can create a rule +and easily ingest IoT data into Cassandra by leveraging the +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into Cassandra](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-cassa.html) + for how to use EMQX dashboard to ingest IoT data into Cassandra. +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). + diff --git a/apps/emqx_bridge_clickhouse/BSL.txt b/apps/emqx_bridge_clickhouse/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_clickhouse/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_clickhouse/README.md b/apps/emqx_bridge_clickhouse/README.md new file mode 100644 index 000000000..fb61fea9c --- /dev/null +++ b/apps/emqx_bridge_clickhouse/README.md @@ -0,0 +1,37 @@ +# EMQX ClickHouse Bridge + +[ClickHouse](https://github.com/ClickHouse/ClickHouse)is an open-source, column-based +database management system. It is designed for real-time processing of large volumes of +data and is known for its high performance and scalability. + +The application is used to connect EMQX and ClickHouse. +User can create a rule and easily ingest IoT data into ClickHouse by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into ClickHouse](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-clickhouse.html) + for how to use EMQX dashboard to ingest IoT data into ClickHouse. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_dynamo/BSL.txt b/apps/emqx_bridge_dynamo/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_dynamo/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_dynamo/README.md b/apps/emqx_bridge_dynamo/README.md new file mode 100644 index 000000000..deb9d1879 --- /dev/null +++ b/apps/emqx_bridge_dynamo/README.md @@ -0,0 +1,40 @@ +# EMQX DynamoDB Bridge + +[Dynamodb](https://aws.amazon.com/dynamodb/) is a high-performance NoSQL database +service provided by Amazon that's designed for scalability and low-latency access +to structured data. + +It's often used in applications that require fast and reliable access to data, +such as mobile, ad tech, and IoT. + +The application is used to connect EMQX and DynamoDB. +User can create a rule and easily ingest IoT data into DynamoDB by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into DynamoDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-dynamo.html) + for how to use EMQX dashboard to ingest IoT data into DynamoDB. + +- Refer to [Rules engine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_gcp_pubsub/BSL.txt b/apps/emqx_bridge_gcp_pubsub/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_gcp_pubsub/README.md b/apps/emqx_bridge_gcp_pubsub/README.md new file mode 100644 index 000000000..2e876ef0f --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/README.md @@ -0,0 +1,35 @@ +# EMQX GCP Pub/Sub Bridge + +[Google Cloud Pub/Sub](https://cloud.google.com/pubsub) is a messaging service provided by Google Cloud Platform (GCP). + +The application is used to connect EMQX and GCP Pub/Sub. +User can create a rule and easily ingest IoT data into GCP Pub/Sub by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into GCP Pub/Sub](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-gcp-pubsub.html) + for how to use EMQX dashboard to ingest IoT data into GCP Pub/Sub. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_hstreamdb/BSL.txt b/apps/emqx_bridge_hstreamdb/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_hstreamdb/README.md b/apps/emqx_bridge_hstreamdb/README.md new file mode 100644 index 000000000..0201fa087 --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/README.md @@ -0,0 +1,38 @@ +# EMQX HStreamDB Bridge + +[HStreamDB](https://hstream.io/)is streaming database purpose-built to ingest, +store, process, and analyze massive data streams. It is a modern data infrastructure +that unifies messaging, stream processing, and storage to help get value out of +your data in real-time. + +The application is used to connect EMQX and HStreamDB. +User can create a rule and easily ingest IoT data into HStreamDB by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into HStreamDB](todo) + for how to use EMQX dashboard to ingest IoT data into HStreamDB. + +- Refer to [Rules engine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_influxdb/BSL.txt b/apps/emqx_bridge_influxdb/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_influxdb/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_influxdb/README.md b/apps/emqx_bridge_influxdb/README.md new file mode 100644 index 000000000..e9ad733f9 --- /dev/null +++ b/apps/emqx_bridge_influxdb/README.md @@ -0,0 +1,49 @@ +# EMQX InfluxDB Bridge + +[InfluxDB](https://github.com/influxdata/influxdb) is an open-source time-series +database that is optimized for storing, retrieving, and querying large volumes of +time-stamped data. +It is commonly used for monitoring and analysis of metrics, events, and real-time +analytics. +InfluxDB is designed to be fast, efficient, and scalable, and it has a SQL-like +query language that makes it easy to extract insights from time-series data. + +The application is used to connect EMQX and InfluxDB. User can create a rule and +easily ingest IoT data into InfluxDB by leveraging the +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into InfluxDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-influxdb.html) + for how to use EMQX dashboard to ingest IoT data into InfluxDB. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information. + +- [Create bridge API doc](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges/paths/~1bridges/post) + list required parameters for creating a InfluxDB bridge. + There are two types of InfluxDB API (`v1` and `v2`), please select the right + version of InfluxDB. Below are several important parameters for `v1`, + - `server`: The IPv4 or IPv6 address or the hostname to connect to. + - `database`: InfluxDB database name + - `write_syntax`: Conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). + diff --git a/apps/emqx_bridge_matrix/BSL.txt b/apps/emqx_bridge_matrix/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_matrix/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_matrix/README.md b/apps/emqx_bridge_matrix/README.md new file mode 100644 index 000000000..c0098e3f6 --- /dev/null +++ b/apps/emqx_bridge_matrix/README.md @@ -0,0 +1,36 @@ +# EMQX MatrixDB Bridge + +[MatrixDB](http://matrixdb.univ-lyon1.fr/) is a biological database focused on +molecular interactions between extracellular proteins and polysaccharides. + +The application is used to connect EMQX and MatrixDB. +User can create a rule and easily ingest IoT data into MatrixDB by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into MatrixDB](todo) + for how to use EMQX dashboard to ingest IoT data into MatrixDB. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_mongodb/BSL.txt b/apps/emqx_bridge_mongodb/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_mongodb/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_mongodb/README.md b/apps/emqx_bridge_mongodb/README.md new file mode 100644 index 000000000..4608b2e84 --- /dev/null +++ b/apps/emqx_bridge_mongodb/README.md @@ -0,0 +1,39 @@ +# EMQX MongoDB Bridge + +[MongoDB](https://github.com/mongodb/mongo) is a source-available cross-platform +document-oriented database. It is a NoSQL database that stores flexible JSON-like +documents for faster iteration and better data organization. +It provides high availability and scaling with its built-in replication and sharding +features, and is used in a variety of industries + +The application is used to connect EMQX and MongoDB. +User can create a rule and easily ingest IoT data into MongoDB by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into MongoDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mongodb.html) + for how to use EMQX dashboard to ingest IoT data into MongoDB. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_mysql/BSL.txt b/apps/emqx_bridge_mysql/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_mysql/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_mysql/README.md b/apps/emqx_bridge_mysql/README.md new file mode 100644 index 000000000..11d854d2d --- /dev/null +++ b/apps/emqx_bridge_mysql/README.md @@ -0,0 +1,36 @@ +# EMQX MySQL Bridge + +[MySQL](https://github.com/MySQL/MySQL) is a popular open-source relational database +management system. + +The application is used to connect EMQX and MySQL. +User can create a rule and easily ingest IoT data into MySQL by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into MySQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mysql.html) + for how to use EMQX dashboard to ingest IoT data into MySQL. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_pgsql/BSL.txt b/apps/emqx_bridge_pgsql/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_pgsql/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_pgsql/README.md b/apps/emqx_bridge_pgsql/README.md new file mode 100644 index 000000000..3cc33e8ed --- /dev/null +++ b/apps/emqx_bridge_pgsql/README.md @@ -0,0 +1,38 @@ +# EMQX PostgreSQL Bridge + +[PostgreSQL](https://github.com/PostgreSQL/PostgreSQL) is an open-source relational +database management system (RDBMS) that uses and extends the SQL language. +It is known for its reliability, data integrity, and advanced features such as +support for JSON, XML, and other data formats. + +The application is used to connect EMQX and PostgreSQL. +User can create a rule and easily ingest IoT data into PostgreSQL by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into PostgreSQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-pgsql.html) + for how to use EMQX dashboard to ingest IoT data into PostgreSQL. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_redis/BSL.txt b/apps/emqx_bridge_redis/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_redis/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_redis/README.md b/apps/emqx_bridge_redis/README.md new file mode 100644 index 000000000..6a2f4a46e --- /dev/null +++ b/apps/emqx_bridge_redis/README.md @@ -0,0 +1,37 @@ +# EMQX Redis Bridge + +[Redis](https://github.com/redis/redis) is an in-memory data structure store, +used as a distributed, in-memory key–value database, cache and message broker, +with optional durability. + +The application is used to connect EMQX and Redis. +User can create a rule and easily ingest IoT data into Redis by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into Redis](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-redis.html) + for how to use EMQX dashboard to ingest IoT data into Redis. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_rocketmq/BSL.txt b/apps/emqx_bridge_rocketmq/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_rocketmq/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_rocketmq/README.md b/apps/emqx_bridge_rocketmq/README.md new file mode 100644 index 000000000..b2337fcef --- /dev/null +++ b/apps/emqx_bridge_rocketmq/README.md @@ -0,0 +1,37 @@ +# EMQX RocketMQ Bridge + +[RocketMQ](https://github.com/apache/rocketmq) is a distributed messaging and +streaming platform developed by the Apache Software Foundation. +It provides reliable, scalable, and high-throughput messaging services for modern cloud-native applications + +The application is used to connect EMQX and RocketMQ. +User can create a rule and easily ingest IoT data into RocketMQ by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into RocketMQ](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-rocketmq.html) + for how to use EMQX dashboard to ingest IoT data into RocketMQ. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_tdengine/BSL.txt b/apps/emqx_bridge_tdengine/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_tdengine/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_tdengine/README.md b/apps/emqx_bridge_tdengine/README.md new file mode 100644 index 000000000..65c3d415c --- /dev/null +++ b/apps/emqx_bridge_tdengine/README.md @@ -0,0 +1,39 @@ +# EMQX TDEngine Bridge + +[TDEngine](https://github.com/taosdata/TDengine) is an open-source, cloud-native +time series database (TSDB) optimized for Internet of Things (IoT), Connected Cars, +and Industrial IoT. +It enables efficient, real-time ingestion, processing, and monitoring of petabytes +of data per day, generated by billions of sensors and data collectors. + +The application is used to connect EMQX and TDEngine. +User can create a rule and easily ingest IoT data into TDEngine by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into TDEngine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-tdengine.html) + for how to use EMQX dashboard to ingest IoT data into TDEngine. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_timescale/BSL.txt b/apps/emqx_bridge_timescale/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_timescale/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_timescale/README.md b/apps/emqx_bridge_timescale/README.md new file mode 100644 index 000000000..c8bcde173 --- /dev/null +++ b/apps/emqx_bridge_timescale/README.md @@ -0,0 +1,38 @@ +# EMQX TimescaleDB Bridge + +[TimescaleDB](https://github.com/timescaleDB/timescaleDB) is an open-source database +designed to make SQL scalable for time-series data. +It is engineered up from PostgreSQL and packaged as a PostgreSQL extension, +providing automatic partitioning across time and space (partitioning key), as well as full SQL support. + +The application is used to connect EMQX and TimescaleDB. +User can create a rule and easily ingest IoT data into TimescaleDB by leveraging +the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + + +# Documention + +- Refer to [Ingest data into TimescaleDB](todo) + for how to use EMQX dashboard to ingest IoT data into TimescaleDB. + +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). From ec8d8b805fa3d0efd2bbf030a236ea231577976c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Sun, 16 Apr 2023 14:32:24 +0800 Subject: [PATCH 196/279] chore: add ee bridge apps --- .../src/emqx_bridge_cassandra.app.src | 9 +++++++++ .../src/emqx_bridge_clickhouse.app.src | 9 +++++++++ apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src | 9 +++++++++ .../src/emqx_bridge_gcp_pubsub.app.src | 9 +++++++++ .../src/emqx_bridge_hstreamdb.app.src | 9 +++++++++ .../src/emqx_bridge_influxdb.app.src | 9 +++++++++ apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src | 9 +++++++++ apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src | 9 +++++++++ apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src | 9 +++++++++ apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src | 9 +++++++++ apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src | 9 +++++++++ .../src/emqx_bridge_rocketmq.app.src | 9 +++++++++ .../src/emqx_bridge_tdengine.app.src | 9 +++++++++ .../src/emqx_bridge_timescale.app.src | 9 +++++++++ 14 files changed, 126 insertions(+) create mode 100644 apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src create mode 100644 apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src create mode 100644 apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src create mode 100644 apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src create mode 100644 apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src create mode 100644 apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src create mode 100644 apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src create mode 100644 apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src create mode 100644 apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src create mode 100644 apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src create mode 100644 apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src create mode 100644 apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src create mode 100644 apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src create mode 100644 apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src new file mode 100644 index 000000000..1ed65ea9f --- /dev/null +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_cassandra, [ + {description, "EMQX Enterprise Cassandra Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src new file mode 100644 index 000000000..a0b409d5b --- /dev/null +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_clickhouse, [ + {description, "EMQX Enterprise ClickHouse Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src new file mode 100644 index 000000000..51c717220 --- /dev/null +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_dynamo, [ + {description, "EMQX Enterprise Dynamo Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src new file mode 100644 index 000000000..0e1427888 --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_gcp_pubsub, [ + {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src new file mode 100644 index 000000000..1cb3742b3 --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_hstreamdb, [ + {description, "EMQX Enterprise HStreamDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src new file mode 100644 index 000000000..5443417c3 --- /dev/null +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_influxdb, [ + {description, "EMQX Enterprise InfluxDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src new file mode 100644 index 000000000..e2a17e070 --- /dev/null +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_matrix, [ + {description, "EMQX Enterprise MatrixDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src new file mode 100644 index 000000000..008a9e164 --- /dev/null +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_mongodb, [ + {description, "EMQX Enterprise MongoDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src new file mode 100644 index 000000000..2e36587a7 --- /dev/null +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_mysql, [ + {description, "EMQX Enterprise MySQL Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src new file mode 100644 index 000000000..c695283f3 --- /dev/null +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_pgsql, [ + {description, "EMQX Enterprise PostgreSQL Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src new file mode 100644 index 000000000..6b57c6cd7 --- /dev/null +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_redis, [ + {description, "EMQX Enterprise Redis Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src new file mode 100644 index 000000000..202bb38f2 --- /dev/null +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_rocketmq, [ + {description, "EMQX Enterprise RcoketMQ Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src new file mode 100644 index 000000000..05e8a6f9f --- /dev/null +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_tdengine, [ + {description, "EMQX Enterprise TDEngine Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src new file mode 100644 index 000000000..5b4431f73 --- /dev/null +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src @@ -0,0 +1,9 @@ +{application, emqx_bridge_timescale, [ + {description, "EMQX Enterprise TimescaleDB Bridge"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {links, []} +]}. From 862817408af0f7741aa3504c3b63db0a257d4be0 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 17 Apr 2023 14:37:25 +0800 Subject: [PATCH 197/279] chore: ignore no suites tests --- Makefile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index fe75b01bc..0403a07e9 100644 --- a/Makefile +++ b/Makefile @@ -89,12 +89,17 @@ APPS=$(shell $(SCRIPTS)/find-apps.sh) .PHONY: $(APPS:%=%-ct) define gen-app-ct-target $1-ct: $(REBAR) - @$(SCRIPTS)/pre-compile.sh $(PROFILE) - @ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \ - --readable=$(CT_READABLE) \ - --name $(CT_NODE_NAME) \ - --cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \ - --suite $(shell $(SCRIPTS)/find-suites.sh $1) + $(eval SUITES := $(shell $(SCRIPTS)/find-suites.sh $1)) +ifneq ($(SUITES),) + @$(SCRIPTS)/pre-compile.sh $(PROFILE) + @ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \ + --readable=$(CT_READABLE) \ + --name $(CT_NODE_NAME) \ + --cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \ + --suite $(SUITES) +else + @echo 'No suites found for $1' +endif endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) From 8cb9389d215de0f69a2b5a1fa4c165f25668aa13 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 18 Apr 2023 09:32:35 +0800 Subject: [PATCH 198/279] chore: apply suggestions from code review Co-authored-by: Thales Macedo Garitezi --- apps/emqx_bridge_cassandra/README.md | 2 +- apps/emqx_bridge_gcp_pubsub/README.md | 2 +- apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src | 2 +- apps/emqx_bridge_timescale/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge_cassandra/README.md b/apps/emqx_bridge_cassandra/README.md index fbf57660d..30dd9b6d5 100644 --- a/apps/emqx_bridge_cassandra/README.md +++ b/apps/emqx_bridge_cassandra/README.md @@ -12,7 +12,7 @@ and easily ingest IoT data into Cassandra by leveraging the [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into Cassandra](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-cassa.html) for how to use EMQX dashboard to ingest IoT data into Cassandra. diff --git a/apps/emqx_bridge_gcp_pubsub/README.md b/apps/emqx_bridge_gcp_pubsub/README.md index 2e876ef0f..aa62ca0db 100644 --- a/apps/emqx_bridge_gcp_pubsub/README.md +++ b/apps/emqx_bridge_gcp_pubsub/README.md @@ -4,7 +4,7 @@ The application is used to connect EMQX and GCP Pub/Sub. User can create a rule and easily ingest IoT data into GCP Pub/Sub by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +the [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). # Documention diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index 202bb38f2..e1916034c 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -1,5 +1,5 @@ {application, emqx_bridge_rocketmq, [ - {description, "EMQX Enterprise RcoketMQ Bridge"}, + {description, "EMQX Enterprise RocketMQ Bridge"}, {vsn, "0.1.0"}, {registered, []}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_bridge_timescale/README.md b/apps/emqx_bridge_timescale/README.md index c8bcde173..ac631c003 100644 --- a/apps/emqx_bridge_timescale/README.md +++ b/apps/emqx_bridge_timescale/README.md @@ -12,7 +12,7 @@ the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/ # Documention -- Refer to [Ingest data into TimescaleDB](todo) +- Refer to [Ingest data into TimescaleDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-timescaledb.html) for how to use EMQX dashboard to ingest IoT data into TimescaleDB. - Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) From acb30c79573c9907acf1f161d015668d9baa7cbb Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 18 Apr 2023 10:47:07 +0800 Subject: [PATCH 199/279] chore: update changes/ce/feat-10389.en.md Co-authored-by: JianBo He --- changes/ce/feat-10389.en.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changes/ce/feat-10389.en.md b/changes/ce/feat-10389.en.md index d6b8d236f..b656f5362 100644 --- a/changes/ce/feat-10389.en.md +++ b/changes/ce/feat-10389.en.md @@ -1,2 +1,2 @@ -Now `cluster.core_nodes` and `cluster.statics.seeds` are specified in the same way. -Configuring them as `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` and `"emqx1@127.0.0.1,emqx2@127.0.0.1"` has the same effect. +Unify the config formats for `cluster.core_nodes` and `cluster.statics.seeds`. +Now they both support formats in arrays `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` or semicolon-separated strings `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. From 77d300482cdf5144e3fa2db67a0ea45e412a1a43 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 18 Apr 2023 10:49:51 +0800 Subject: [PATCH 200/279] chore: update changes/ce/feat-10389.en.md --- changes/ce/feat-10389.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/feat-10389.en.md b/changes/ce/feat-10389.en.md index b656f5362..3789e80ae 100644 --- a/changes/ce/feat-10389.en.md +++ b/changes/ce/feat-10389.en.md @@ -1,2 +1,2 @@ Unify the config formats for `cluster.core_nodes` and `cluster.statics.seeds`. -Now they both support formats in arrays `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` or semicolon-separated strings `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. +Now they both support formats in array `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` or semicolon-separated string `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. From c53ccfea61e6a6e17b33f9b23aef9489d2932dcf Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 18 Apr 2023 09:36:14 +0800 Subject: [PATCH 201/279] chore: fix typos --- apps/emqx_bridge_cassandra/README.md | 2 +- apps/emqx_bridge_clickhouse/README.md | 6 +++--- apps/emqx_bridge_dynamo/README.md | 4 ++-- apps/emqx_bridge_gcp_pubsub/README.md | 7 ++++--- apps/emqx_bridge_hstreamdb/README.md | 8 ++++---- apps/emqx_bridge_influxdb/README.md | 4 ++-- apps/emqx_bridge_matrix/README.md | 4 ++-- apps/emqx_bridge_mongodb/README.md | 4 ++-- apps/emqx_bridge_mysql/README.md | 4 ++-- apps/emqx_bridge_pgsql/README.md | 4 ++-- apps/emqx_bridge_redis/README.md | 4 ++-- apps/emqx_bridge_rocketmq/README.md | 4 ++-- apps/emqx_bridge_tdengine/README.md | 4 ++-- apps/emqx_bridge_timescale/README.md | 4 ++-- apps/emqx_machine/README.md | 2 +- 15 files changed, 33 insertions(+), 32 deletions(-) diff --git a/apps/emqx_bridge_cassandra/README.md b/apps/emqx_bridge_cassandra/README.md index 30dd9b6d5..d26bd2fbb 100644 --- a/apps/emqx_bridge_cassandra/README.md +++ b/apps/emqx_bridge_cassandra/README.md @@ -8,7 +8,7 @@ It is commonly used in web and mobile applications, IoT, and other systems that require storing, querying, and analyzing large amounts of data. The application is used to connect EMQX and Cassandra. User can create a rule -and easily ingest IoT data into Cassandra by leveraging the +and easily ingest IoT data into Cassandra by leveraging [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). diff --git a/apps/emqx_bridge_clickhouse/README.md b/apps/emqx_bridge_clickhouse/README.md index fb61fea9c..ff870e87d 100644 --- a/apps/emqx_bridge_clickhouse/README.md +++ b/apps/emqx_bridge_clickhouse/README.md @@ -1,15 +1,15 @@ # EMQX ClickHouse Bridge -[ClickHouse](https://github.com/ClickHouse/ClickHouse)is an open-source, column-based +[ClickHouse](https://github.com/ClickHouse/ClickHouse) is an open-source, column-based database management system. It is designed for real-time processing of large volumes of data and is known for its high performance and scalability. The application is used to connect EMQX and ClickHouse. User can create a rule and easily ingest IoT data into ClickHouse by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into ClickHouse](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-clickhouse.html) for how to use EMQX dashboard to ingest IoT data into ClickHouse. diff --git a/apps/emqx_bridge_dynamo/README.md b/apps/emqx_bridge_dynamo/README.md index deb9d1879..48dcb781d 100644 --- a/apps/emqx_bridge_dynamo/README.md +++ b/apps/emqx_bridge_dynamo/README.md @@ -9,10 +9,10 @@ such as mobile, ad tech, and IoT. The application is used to connect EMQX and DynamoDB. User can create a rule and easily ingest IoT data into DynamoDB by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into DynamoDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-dynamo.html) for how to use EMQX dashboard to ingest IoT data into DynamoDB. diff --git a/apps/emqx_bridge_gcp_pubsub/README.md b/apps/emqx_bridge_gcp_pubsub/README.md index aa62ca0db..e33c5ab15 100644 --- a/apps/emqx_bridge_gcp_pubsub/README.md +++ b/apps/emqx_bridge_gcp_pubsub/README.md @@ -1,13 +1,14 @@ # EMQX GCP Pub/Sub Bridge -[Google Cloud Pub/Sub](https://cloud.google.com/pubsub) is a messaging service provided by Google Cloud Platform (GCP). +[Google Cloud Pub/Sub](https://cloud.google.com/pubsub) is a messaging service +provided by Google Cloud Platform (GCP). The application is used to connect EMQX and GCP Pub/Sub. User can create a rule and easily ingest IoT data into GCP Pub/Sub by leveraging -the [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into GCP Pub/Sub](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-gcp-pubsub.html) for how to use EMQX dashboard to ingest IoT data into GCP Pub/Sub. diff --git a/apps/emqx_bridge_hstreamdb/README.md b/apps/emqx_bridge_hstreamdb/README.md index 0201fa087..3a7c6b49d 100644 --- a/apps/emqx_bridge_hstreamdb/README.md +++ b/apps/emqx_bridge_hstreamdb/README.md @@ -1,21 +1,21 @@ # EMQX HStreamDB Bridge -[HStreamDB](https://hstream.io/)is streaming database purpose-built to ingest, +[HStreamDB](https://hstream.io/) is streaming database purpose-built to ingest, store, process, and analyze massive data streams. It is a modern data infrastructure that unifies messaging, stream processing, and storage to help get value out of your data in real-time. The application is used to connect EMQX and HStreamDB. User can create a rule and easily ingest IoT data into HStreamDB by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into HStreamDB](todo) for how to use EMQX dashboard to ingest IoT data into HStreamDB. -- Refer to [Rules engine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) +- Refer to [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) for the EMQX rules engine introduction. diff --git a/apps/emqx_bridge_influxdb/README.md b/apps/emqx_bridge_influxdb/README.md index e9ad733f9..fe0f14600 100644 --- a/apps/emqx_bridge_influxdb/README.md +++ b/apps/emqx_bridge_influxdb/README.md @@ -9,11 +9,11 @@ InfluxDB is designed to be fast, efficient, and scalable, and it has a SQL-like query language that makes it easy to extract insights from time-series data. The application is used to connect EMQX and InfluxDB. User can create a rule and -easily ingest IoT data into InfluxDB by leveraging the +easily ingest IoT data into InfluxDB by leveraging [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into InfluxDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-influxdb.html) for how to use EMQX dashboard to ingest IoT data into InfluxDB. diff --git a/apps/emqx_bridge_matrix/README.md b/apps/emqx_bridge_matrix/README.md index c0098e3f6..976120ffe 100644 --- a/apps/emqx_bridge_matrix/README.md +++ b/apps/emqx_bridge_matrix/README.md @@ -5,10 +5,10 @@ molecular interactions between extracellular proteins and polysaccharides. The application is used to connect EMQX and MatrixDB. User can create a rule and easily ingest IoT data into MatrixDB by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into MatrixDB](todo) for how to use EMQX dashboard to ingest IoT data into MatrixDB. diff --git a/apps/emqx_bridge_mongodb/README.md b/apps/emqx_bridge_mongodb/README.md index 4608b2e84..088c8467f 100644 --- a/apps/emqx_bridge_mongodb/README.md +++ b/apps/emqx_bridge_mongodb/README.md @@ -8,10 +8,10 @@ features, and is used in a variety of industries The application is used to connect EMQX and MongoDB. User can create a rule and easily ingest IoT data into MongoDB by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into MongoDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mongodb.html) for how to use EMQX dashboard to ingest IoT data into MongoDB. diff --git a/apps/emqx_bridge_mysql/README.md b/apps/emqx_bridge_mysql/README.md index 11d854d2d..73f6987b6 100644 --- a/apps/emqx_bridge_mysql/README.md +++ b/apps/emqx_bridge_mysql/README.md @@ -5,10 +5,10 @@ management system. The application is used to connect EMQX and MySQL. User can create a rule and easily ingest IoT data into MySQL by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into MySQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-mysql.html) for how to use EMQX dashboard to ingest IoT data into MySQL. diff --git a/apps/emqx_bridge_pgsql/README.md b/apps/emqx_bridge_pgsql/README.md index 3cc33e8ed..fc0bd3c6f 100644 --- a/apps/emqx_bridge_pgsql/README.md +++ b/apps/emqx_bridge_pgsql/README.md @@ -7,10 +7,10 @@ support for JSON, XML, and other data formats. The application is used to connect EMQX and PostgreSQL. User can create a rule and easily ingest IoT data into PostgreSQL by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into PostgreSQL](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-pgsql.html) for how to use EMQX dashboard to ingest IoT data into PostgreSQL. diff --git a/apps/emqx_bridge_redis/README.md b/apps/emqx_bridge_redis/README.md index 6a2f4a46e..73ec41f07 100644 --- a/apps/emqx_bridge_redis/README.md +++ b/apps/emqx_bridge_redis/README.md @@ -6,10 +6,10 @@ with optional durability. The application is used to connect EMQX and Redis. User can create a rule and easily ingest IoT data into Redis by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into Redis](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-redis.html) for how to use EMQX dashboard to ingest IoT data into Redis. diff --git a/apps/emqx_bridge_rocketmq/README.md b/apps/emqx_bridge_rocketmq/README.md index b2337fcef..252e6beac 100644 --- a/apps/emqx_bridge_rocketmq/README.md +++ b/apps/emqx_bridge_rocketmq/README.md @@ -6,10 +6,10 @@ It provides reliable, scalable, and high-throughput messaging services for moder The application is used to connect EMQX and RocketMQ. User can create a rule and easily ingest IoT data into RocketMQ by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into RocketMQ](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-rocketmq.html) for how to use EMQX dashboard to ingest IoT data into RocketMQ. diff --git a/apps/emqx_bridge_tdengine/README.md b/apps/emqx_bridge_tdengine/README.md index 65c3d415c..25faf4c14 100644 --- a/apps/emqx_bridge_tdengine/README.md +++ b/apps/emqx_bridge_tdengine/README.md @@ -8,10 +8,10 @@ of data per day, generated by billions of sensors and data collectors. The application is used to connect EMQX and TDEngine. User can create a rule and easily ingest IoT data into TDEngine by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into TDEngine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-tdengine.html) for how to use EMQX dashboard to ingest IoT data into TDEngine. diff --git a/apps/emqx_bridge_timescale/README.md b/apps/emqx_bridge_timescale/README.md index ac631c003..96f70f847 100644 --- a/apps/emqx_bridge_timescale/README.md +++ b/apps/emqx_bridge_timescale/README.md @@ -7,10 +7,10 @@ providing automatic partitioning across time and space (partitioning key), as we The application is used to connect EMQX and TimescaleDB. User can create a rule and easily ingest IoT data into TimescaleDB by leveraging -the EMQX [EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). -# Documention +# Documentation - Refer to [Ingest data into TimescaleDB](https://docs.emqx.com/en/enterprise/v5.0/data-integration/data-bridge-timescaledb.html) for how to use EMQX dashboard to ingest IoT data into TimescaleDB. diff --git a/apps/emqx_machine/README.md b/apps/emqx_machine/README.md index 8c2bb6391..a8221ba73 100644 --- a/apps/emqx_machine/README.md +++ b/apps/emqx_machine/README.md @@ -31,7 +31,7 @@ It helps to shut down EMQX broker gracefully when it receives `SIGTERM` signal. Currently `emqx_machine` boots the business apps before starting autocluster, so a fresh node joining the cluster actually starts business application twice: first in the singleton mode, and then in clustered mode. -# Documention links +# Documentation links Configuration: [node.global_gc_interval](https://www.emqx.io/docs/en/v5.0/configuration/configuration-manual.html#node-and-cookie) From a947df1ea3312b188ec11d2bce4ccc507e835227 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 11 Apr 2023 13:59:08 +0200 Subject: [PATCH 202/279] fix(emqx_management): Ignore results from the nodes that are down --- .../emqx_management/src/emqx_mgmt_api_stats.erl | 17 +++++++++++++---- changes/ce/fix-10369.en.md | 6 ++++++ 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 changes/ce/fix-10369.en.md diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 1e752aaac..080a37b4d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -127,10 +127,19 @@ list(get, #{query_string := Qs}) -> true -> {200, emqx_mgmt:get_stats()}; _ -> - Data = [ - maps:from_list(emqx_mgmt:get_stats(Node) ++ [{node, Node}]) - || Node <- running_nodes() - ], + Data = lists:foldl( + fun(Node, Acc) -> + case emqx_mgmt:get_stats(Node) of + {error, _Err} -> + Acc; + Stats when is_list(Stats) -> + Data = maps:from_list([{node, Node} | Stats]), + [Data | Acc] + end + end, + [], + mria:running_nodes() + ), {200, Data} end. diff --git a/changes/ce/fix-10369.en.md b/changes/ce/fix-10369.en.md new file mode 100644 index 000000000..594e91fc9 --- /dev/null +++ b/changes/ce/fix-10369.en.md @@ -0,0 +1,6 @@ +Fix error `/api/v5/monitor_current` API endpoint when some EMQX nodes are down. + +Prior to this fix, sometimes the request returned HTTP code 500 and the following message: +``` +{"code":"INTERNAL_ERROR","message":"error, badarg, [{erlang,'++',[{error,nodedown},[{node,'emqx@10.42.0.150'}]], ... +``` From 282bfee8ff120bee4ad7da501142b2c63438ad3c Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 12 Apr 2023 10:56:03 +0200 Subject: [PATCH 203/279] feat(emqx): Add an API that returns the list of running nodes --- apps/emqx/src/emqx.erl | 18 ++++++ apps/emqx/test/emqx_SUITE.erl | 8 +++ apps/emqx_management/src/emqx_mgmt.erl | 57 +++++++++++-------- apps/emqx_management/src/emqx_mgmt_api.erl | 2 +- .../src/emqx_mgmt_api_cluster.erl | 2 +- .../src/emqx_mgmt_api_configs.erl | 2 +- .../src/emqx_mgmt_api_listeners.erl | 2 +- .../src/emqx_mgmt_api_metrics.erl | 2 +- .../src/emqx_mgmt_api_stats.erl | 2 +- .../src/emqx_mgmt_api_trace.erl | 10 ++-- apps/emqx_management/test/emqx_mgmt_SUITE.erl | 10 ++-- .../test/emqx_mgmt_api_SUITE.erl | 6 +- .../test/emqx_mgmt_api_configs_SUITE.erl | 4 +- .../test/emqx_mgmt_api_listeners_SUITE.erl | 4 +- .../test/emqx_mgmt_api_stats_SUITE.erl | 2 + apps/emqx_prometheus/src/emqx_prometheus.erl | 4 +- changes/ce/fix-10369.en.md | 2 +- 17 files changed, 86 insertions(+), 51 deletions(-) diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index ef870685a..ffee5fba7 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -30,6 +30,12 @@ stop/0 ]). +%% Cluster API +-export([ + cluster_nodes/1, + running_nodes/0 +]). + %% PubSub API -export([ subscribe/1, @@ -102,6 +108,18 @@ is_running() -> _ -> true end. +%%-------------------------------------------------------------------- +%% Cluster API +%%-------------------------------------------------------------------- + +-spec running_nodes() -> [node()]. +running_nodes() -> + mria:running_nodes(). + +-spec cluster_nodes(all | running | cores | stopped) -> [node()]. +cluster_nodes(Type) -> + mria:cluster_nodes(Type). + %%-------------------------------------------------------------------- %% PubSub API %%-------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_SUITE.erl b/apps/emqx/test/emqx_SUITE.erl index 09d5d8017..64ed2ea19 100644 --- a/apps/emqx/test/emqx_SUITE.erl +++ b/apps/emqx/test/emqx_SUITE.erl @@ -148,6 +148,14 @@ t_run_hook(_) -> ?assertEqual(3, emqx:run_fold_hook(foldl_filter2_hook, [arg], 1)), ?assertEqual(2, emqx:run_fold_hook(foldl_filter2_hook, [arg1], 1)). +t_cluster_nodes(_) -> + Expected = [node()], + ?assertEqual(Expected, emqx:running_nodes()), + ?assertEqual(Expected, emqx:cluster_nodes(running)), + ?assertEqual(Expected, emqx:cluster_nodes(all)), + ?assertEqual(Expected, emqx:cluster_nodes(cores)), + ?assertEqual([], emqx:cluster_nodes(stopped)). + %%-------------------------------------------------------------------- %% Hook fun %%-------------------------------------------------------------------- diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 5ba12646f..0b91817f0 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -112,8 +112,8 @@ %%-------------------------------------------------------------------- list_nodes() -> - Running = mria:cluster_nodes(running), - Stopped = mria:cluster_nodes(stopped), + Running = emqx:cluster_nodes(running), + Stopped = emqx:cluster_nodes(stopped), DownNodes = lists:map(fun stopped_node_info/1, Stopped), [{Node, Info} || #{node := Node} = Info <- node_info(Running)] ++ DownNodes. @@ -199,7 +199,7 @@ vm_stats() -> %%-------------------------------------------------------------------- list_brokers() -> - Running = mria:running_nodes(), + Running = emqx:running_nodes(), [{Node, Broker} || #{node := Node} = Broker <- broker_info(Running)]. lookup_broker(Node) -> @@ -223,7 +223,7 @@ broker_info(Nodes) -> %%-------------------------------------------------------------------- get_metrics() -> - nodes_info_count([get_metrics(Node) || Node <- mria:running_nodes()]). + nodes_info_count([get_metrics(Node) || Node <- emqx:running_nodes()]). get_metrics(Node) -> unwrap_rpc(emqx_proto_v1:get_metrics(Node)). @@ -238,13 +238,20 @@ get_stats() -> 'subscriptions.shared.count', 'subscriptions.shared.max' ], - CountStats = nodes_info_count([ - begin - Stats = get_stats(Node), - delete_keys(Stats, GlobalStatsKeys) - end - || Node <- mria:running_nodes() - ]), + CountStats = nodes_info_count( + lists:foldl( + fun(Node, Acc) -> + case get_stats(Node) of + {error, _} -> + Acc; + Stats -> + [delete_keys(Stats, GlobalStatsKeys) | Acc] + end + end, + [], + emqx:running_nodes() + ) + ), GlobalStats = maps:with(GlobalStatsKeys, maps:from_list(get_stats(node()))), maps:merge(CountStats, GlobalStats). @@ -275,12 +282,12 @@ nodes_info_count(PropList) -> lookup_client({clientid, ClientId}, FormatFun) -> lists:append([ lookup_client(Node, {clientid, ClientId}, FormatFun) - || Node <- mria:running_nodes() + || Node <- emqx:running_nodes() ]); lookup_client({username, Username}, FormatFun) -> lists:append([ lookup_client(Node, {username, Username}, FormatFun) - || Node <- mria:running_nodes() + || Node <- emqx:running_nodes() ]). lookup_client(Node, Key, FormatFun) -> @@ -307,7 +314,7 @@ kickout_client(ClientId) -> [] -> {error, not_found}; _ -> - Results = [kickout_client(Node, ClientId) || Node <- mria:running_nodes()], + Results = [kickout_client(Node, ClientId) || Node <- emqx:running_nodes()], check_results(Results) end. @@ -322,7 +329,7 @@ list_client_subscriptions(ClientId) -> [] -> {error, not_found}; _ -> - Results = [client_subscriptions(Node, ClientId) || Node <- mria:running_nodes()], + Results = [client_subscriptions(Node, ClientId) || Node <- emqx:running_nodes()], Filter = fun ({error, _}) -> @@ -340,18 +347,18 @@ client_subscriptions(Node, ClientId) -> {Node, unwrap_rpc(emqx_broker_proto_v1:list_client_subscriptions(Node, ClientId))}. clean_authz_cache(ClientId) -> - Results = [clean_authz_cache(Node, ClientId) || Node <- mria:running_nodes()], + Results = [clean_authz_cache(Node, ClientId) || Node <- emqx:running_nodes()], check_results(Results). clean_authz_cache(Node, ClientId) -> unwrap_rpc(emqx_proto_v1:clean_authz_cache(Node, ClientId)). clean_authz_cache_all() -> - Results = [{Node, clean_authz_cache_all(Node)} || Node <- mria:running_nodes()], + Results = [{Node, clean_authz_cache_all(Node)} || Node <- emqx:running_nodes()], wrap_results(Results). clean_pem_cache_all() -> - Results = [{Node, clean_pem_cache_all(Node)} || Node <- mria:running_nodes()], + Results = [{Node, clean_pem_cache_all(Node)} || Node <- emqx:running_nodes()], wrap_results(Results). wrap_results(Results) -> @@ -379,7 +386,7 @@ set_keepalive(_ClientId, _Interval) -> %% @private call_client(ClientId, Req) -> - Results = [call_client(Node, ClientId, Req) || Node <- mria:running_nodes()], + Results = [call_client(Node, ClientId, Req) || Node <- emqx:running_nodes()], Expected = lists:filter( fun ({error, _}) -> false; @@ -428,7 +435,7 @@ list_subscriptions(Node) -> list_subscriptions_via_topic(Topic, FormatFun) -> lists:append([ list_subscriptions_via_topic(Node, Topic, FormatFun) - || Node <- mria:running_nodes() + || Node <- emqx:running_nodes() ]). list_subscriptions_via_topic(Node, Topic, _FormatFun = {M, F}) -> @@ -442,7 +449,7 @@ list_subscriptions_via_topic(Node, Topic, _FormatFun = {M, F}) -> %%-------------------------------------------------------------------- subscribe(ClientId, TopicTables) -> - subscribe(mria:running_nodes(), ClientId, TopicTables). + subscribe(emqx:running_nodes(), ClientId, TopicTables). subscribe([Node | Nodes], ClientId, TopicTables) -> case unwrap_rpc(emqx_management_proto_v3:subscribe(Node, ClientId, TopicTables)) of @@ -467,7 +474,7 @@ publish(Msg) -> -spec unsubscribe(emqx_types:clientid(), emqx_types:topic()) -> {unsubscribe, _} | {error, channel_not_found}. unsubscribe(ClientId, Topic) -> - unsubscribe(mria:running_nodes(), ClientId, Topic). + unsubscribe(emqx:running_nodes(), ClientId, Topic). -spec unsubscribe([node()], emqx_types:clientid(), emqx_types:topic()) -> {unsubscribe, _} | {error, channel_not_found}. @@ -490,7 +497,7 @@ do_unsubscribe(ClientId, Topic) -> -spec unsubscribe_batch(emqx_types:clientid(), [emqx_types:topic()]) -> {unsubscribe, _} | {error, channel_not_found}. unsubscribe_batch(ClientId, Topics) -> - unsubscribe_batch(mria:running_nodes(), ClientId, Topics). + unsubscribe_batch(emqx:running_nodes(), ClientId, Topics). -spec unsubscribe_batch([node()], emqx_types:clientid(), [emqx_types:topic()]) -> {unsubscribe_batch, _} | {error, channel_not_found}. @@ -515,7 +522,7 @@ do_unsubscribe_batch(ClientId, Topics) -> %%-------------------------------------------------------------------- get_alarms(Type) -> - [{Node, get_alarms(Node, Type)} || Node <- mria:running_nodes()]. + [{Node, get_alarms(Node, Type)} || Node <- emqx:running_nodes()]. get_alarms(Node, Type) -> add_duration_field(unwrap_rpc(emqx_proto_v1:get_alarms(Node, Type))). @@ -524,7 +531,7 @@ deactivate(Node, Name) -> unwrap_rpc(emqx_proto_v1:deactivate_alarm(Node, Name)). delete_all_deactivated_alarms() -> - [delete_all_deactivated_alarms(Node) || Node <- mria:running_nodes()]. + [delete_all_deactivated_alarms(Node) || Node <- emqx:running_nodes()]. delete_all_deactivated_alarms(Node) -> unwrap_rpc(emqx_proto_v1:delete_all_deactivated_alarms(Node)). diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index c77752f7d..8365b983c 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -163,7 +163,7 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) -> {error, page_limit_invalid}; Meta -> {_CodCnt, NQString} = parse_qstring(QString, QSchema), - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), ResultAcc = init_query_result(), QueryState = init_query_state(Tab, NQString, MsFun, Meta), NResultAcc = do_cluster_query( diff --git a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl index 68bb6c81d..e74b6c362 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_cluster.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_cluster.erl @@ -101,7 +101,7 @@ cluster_info(get, _) -> ClusterName = application:get_env(ekka, cluster_name, emqxcl), Info = #{ name => ClusterName, - nodes => mria:running_nodes(), + nodes => emqx:running_nodes(), self => node() }, {200, Info}. diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index c94dc17b6..0efaace20 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -260,7 +260,7 @@ configs(get, Params, _Req) -> QS = maps:get(query_string, Params, #{}), Node = maps:get(<<"node">>, QS, node()), case - lists:member(Node, mria:running_nodes()) andalso + lists:member(Node, emqx:running_nodes()) andalso emqx_management_proto_v2:get_full_config(Node) of false -> diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index d7f3ff321..de86700ef 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -483,7 +483,7 @@ err_msg_str(Reason) -> io_lib:format("~p", [Reason]). list_listeners() -> - [list_listeners(Node) || Node <- mria:running_nodes()]. + [list_listeners(Node) || Node <- emqx:running_nodes()]. list_listeners(Node) -> wrap_rpc(emqx_management_proto_v2:list_listeners(Node)). diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index 1c5c8f62a..0fcc45d8e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -59,7 +59,7 @@ metrics(get, #{query_string := Qs}) -> maps:from_list( emqx_mgmt:get_metrics(Node) ++ [{node, Node}] ) - || Node <- mria:running_nodes() + || Node <- emqx:running_nodes() ], {200, Data} end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 080a37b4d..19857f267 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -138,7 +138,7 @@ list(get, #{query_string := Qs}) -> end end, [], - mria:running_nodes() + emqx:running_nodes() ), {200, Data} end. diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index 5df641add..25cc2734f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -390,7 +390,7 @@ trace(get, _Params) -> fun(#{start_at := A}, #{start_at := B}) -> A > B end, emqx_trace:format(List0) ), - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), TraceSize = wrap_rpc(emqx_mgmt_trace_proto_v2:get_trace_size(Nodes)), AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize), Now = erlang:system_time(second), @@ -464,7 +464,7 @@ format_trace(Trace0) -> LogSize = lists:foldl( fun(Node, Acc) -> Acc#{Node => 0} end, #{}, - mria:running_nodes() + emqx:running_nodes() ), Trace2 = maps:without([enable, filter], Trace1), Trace2#{ @@ -560,13 +560,13 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) -> ). collect_trace_file(undefined, TraceLog) -> - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file(Nodes, TraceLog)); collect_trace_file(Node, TraceLog) -> wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file([Node], TraceLog)). collect_trace_file_detail(TraceLog) -> - Nodes = mria:running_nodes(), + Nodes = emqx:running_nodes(), wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file_detail(Nodes, TraceLog)). wrap_rpc({GoodRes, BadNodes}) -> @@ -696,7 +696,7 @@ parse_node(Query, Default) -> {ok, Default}; {ok, NodeBin} -> Node = binary_to_existing_atom(NodeBin), - true = lists:member(Node, mria:running_nodes()), + true = lists:member(Node, emqx:running_nodes()), {ok, Node} end catch diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl index 71b51b67c..3eb37060e 100644 --- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -36,16 +36,16 @@ end_per_suite(_) -> emqx_mgmt_api_test_util:end_suite([emqx_management, emqx_conf]). init_per_testcase(TestCase, Config) -> - meck:expect(mria, running_nodes, 0, [node()]), + meck:expect(emqx, running_nodes, 0, [node()]), emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config). end_per_testcase(TestCase, Config) -> - meck:unload(mria), + meck:unload(emqx), emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config). t_list_nodes(init, Config) -> meck:expect( - mria, + emqx, cluster_nodes, fun (running) -> [node()]; @@ -125,7 +125,7 @@ t_lookup_client(_Config) -> emqx_mgmt:lookup_client({username, <<"user1">>}, ?FORMATFUN) ), ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, ?FORMATFUN)), - meck:expect(mria, running_nodes, 0, [node(), 'fake@nonode']), + meck:expect(emqx, running_nodes, 0, [node(), 'fake@nonode']), ?assertMatch( [_ | {error, nodedown}], emqx_mgmt:lookup_client({clientid, <<"client1">>}, ?FORMATFUN) ). @@ -188,7 +188,7 @@ t_clean_cache(_Config) -> {error, _}, emqx_mgmt:clean_pem_cache_all() ), - meck:expect(mria, running_nodes, 0, [node(), 'fake@nonode']), + meck:expect(emqx, running_nodes, 0, [node(), 'fake@nonode']), ?assertMatch( {error, [{'fake@nonode', {error, _}}]}, emqx_mgmt:clean_authz_cache_all() diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index bacec718d..a53ffc9c4 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -179,14 +179,14 @@ t_bad_rpc(_) -> ClientLs1 = [start_emqtt_client(node(), I, 1883) || I <- lists:seq(1, 10)], Path = emqx_mgmt_api_test_util:api_path(["clients?limit=2&page=2"]), try - meck:expect(mria, running_nodes, 0, ['fake@nohost']), + meck:expect(emqx, running_nodes, 0, ['fake@nohost']), {error, {_, 500, _}} = emqx_mgmt_api_test_util:request_api(get, Path), %% good cop, bad cop - meck:expect(mria, running_nodes, 0, [node(), 'fake@nohost']), + meck:expect(emqx, running_nodes, 0, [node(), 'fake@nohost']), {error, {_, 500, _}} = emqx_mgmt_api_test_util:request_api(get, Path) after _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1), - meck:unload(mria), + meck:unload(emqx), emqx_mgmt_api_test_util:end_suite() end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 1638b815a..5a0116a4d 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -246,7 +246,7 @@ t_dashboard(_Config) -> t_configs_node({'init', Config}) -> Node = node(), - meck:expect(mria, running_nodes, fun() -> [Node, bad_node, other_node] end), + meck:expect(emqx, running_nodes, fun() -> [Node, bad_node, other_node] end), meck:expect( emqx_management_proto_v2, get_full_config, @@ -258,7 +258,7 @@ t_configs_node({'init', Config}) -> ), Config; t_configs_node({'end', _}) -> - meck:unload([mria, emqx_management_proto_v2]); + meck:unload([emqx, emqx_management_proto_v2]); t_configs_node(_) -> Node = atom_to_list(node()), diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index 62f689a84..33cb66eb2 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -168,8 +168,8 @@ t_api_listeners_list_not_ready(Config) when is_list(Config) -> L3 = get_tcp_listeners(Node2), Comment = #{ - node1 => rpc:call(Node1, mria, running_nodes, []), - node2 => rpc:call(Node2, mria, running_nodes, []) + node1 => rpc:call(Node1, emqx, running_nodes, []), + node2 => rpc:call(Node2, emqx, running_nodes, []) }, ?assert(length(L1) > length(L2), Comment), diff --git a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl index 4099426b8..2550bdbe2 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_stats_SUITE.erl @@ -24,10 +24,12 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> + meck:expect(emqx, running_nodes, 0, [node(), 'fake@node']), emqx_mgmt_api_test_util:init_suite(), Config. end_per_suite(_) -> + meck:unload(emqx), emqx_mgmt_api_test_util:end_suite(). t_stats_api(_) -> diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 05d9508b6..d999f294e 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -599,8 +599,8 @@ emqx_cluster() -> ]. emqx_cluster_data() -> - Running = mria:cluster_nodes(running), - Stopped = mria:cluster_nodes(stopped), + Running = emqx:cluster_nodes(running), + Stopped = emqx:cluster_nodes(stopped), [ {nodes_running, length(Running)}, {nodes_stopped, length(Stopped)} diff --git a/changes/ce/fix-10369.en.md b/changes/ce/fix-10369.en.md index 594e91fc9..3c32d33f3 100644 --- a/changes/ce/fix-10369.en.md +++ b/changes/ce/fix-10369.en.md @@ -1,4 +1,4 @@ -Fix error `/api/v5/monitor_current` API endpoint when some EMQX nodes are down. +Fix error in `/api/v5/monitor_current` API endpoint that happens when some EMQX nodes are down. Prior to this fix, sometimes the request returned HTTP code 500 and the following message: ``` From 721125a9f66d6a971a77823b889cc0ff696b598f Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 18 Apr 2023 15:34:42 +0800 Subject: [PATCH 204/279] fix(retainer): mark `flow-control` as non-importance field --- apps/emqx_retainer/src/emqx_retainer_schema.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index dbe1ad9d5..74f1e2371 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -53,7 +53,8 @@ fields("retainer") -> sc( ?R_REF(flow_control), flow_control, - #{} + #{}, + ?IMPORTANCE_HIDDEN )}, {max_payload_size, sc( @@ -125,7 +126,9 @@ desc(_) -> %% hoconsc:mk(Type, #{desc => ?DESC(DescId)}). sc(Type, DescId, Default) -> - hoconsc:mk(Type, #{default => Default, desc => ?DESC(DescId)}). + sc(Type, DescId, Default, ?DEFAULT_IMPORTANCE). +sc(Type, DescId, Default, Importance) -> + hoconsc:mk(Type, #{default => Default, desc => ?DESC(DescId), importance => Importance}). backend_config() -> hoconsc:mk(hoconsc:ref(?MODULE, mnesia_config), #{desc => ?DESC(backend)}). From 6dd7befaab936706cd217ed928c7f5f2885991bd Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 17 Apr 2023 20:58:21 +0200 Subject: [PATCH 205/279] refactor: unify authn authz type names --- apps/emqx/src/emqx_schema.erl | 16 ++++-- .../emqx_enhanced_authn_scram_mnesia.erl | 12 ++-- .../src/simple_authn/emqx_authn_http.erl | 21 +++---- .../src/simple_authn/emqx_authn_jwt.erl | 57 +++++++++++++------ .../src/simple_authn/emqx_authn_mnesia.erl | 14 +++-- .../src/simple_authn/emqx_authn_mongodb.erl | 31 +++++----- .../src/simple_authn/emqx_authn_mysql.erl | 14 +++-- .../src/simple_authn/emqx_authn_pgsql.erl | 14 +++-- .../src/simple_authn/emqx_authn_redis.erl | 31 +++++----- .../test/emqx_authn_pgsql_SUITE.erl | 2 +- .../test/emqx_authn_redis_SUITE.erl | 2 +- .../test/emqx_authn_schema_SUITE.erl | 4 +- .../test/emqx_authn_schema_tests.erl | 2 +- apps/emqx_authz/src/emqx_authz_schema.erl | 15 +++-- apps/emqx_conf/src/emqx_conf_schema.erl | 4 +- 15 files changed, 140 insertions(+), 99 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8335d69b8..8073e19b5 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -164,7 +164,7 @@ roots(high) -> } )}, {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication(global)}, - %% NOTE: authorization schema here is only to keep emqx app prue + %% NOTE: authorization schema here is only to keep emqx app pure %% the full schema for EMQX node is injected in emqx_conf_schema. {?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, sc( @@ -2762,10 +2762,16 @@ str(S) when is_list(S) -> S. authentication(Which) -> - Desc = + {Importance, Desc} = case Which of - global -> ?DESC(global_authentication); - listener -> ?DESC(listener_authentication) + global -> + %% For root level authentication, it is recommended to configure + %% from the dashboard or API. + %% Hence it's considered a low-importance when it comes to + %% configuration importance. + {?IMPORTANCE_LOW, ?DESC(global_authentication)}; + listener -> + {?IMPORTANCE_HIDDEN, ?DESC(listener_authentication)} end, %% poor man's dependency injection %% this is due to the fact that authn is implemented outside of 'emqx' app. @@ -2781,7 +2787,7 @@ authentication(Which) -> hoconsc:mk(Type, #{ desc => Desc, converter => fun ensure_array/2, - importance => ?IMPORTANCE_HIDDEN + importance => Importance }). %% the older version schema allows individual element (instead of a chain) in config diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index 84f2c9525..b11b89081 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -105,14 +105,16 @@ mnesia(boot) -> %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-scram-builtin_db". +namespace() -> "authn". tags() -> [<<"Authentication">>]. -roots() -> [?CONF_NS]. +%% used for config check when the schema module is resolved +roots() -> + [{?CONF_NS, hoconsc:mk(hoconsc:ref(?MODULE, scram))}]. -fields(?CONF_NS) -> +fields(scram) -> [ {mechanism, emqx_authn_schema:mechanism(scram)}, {backend, emqx_authn_schema:backend(built_in_database)}, @@ -120,7 +122,7 @@ fields(?CONF_NS) -> {iteration_count, fun iteration_count/1} ] ++ emqx_authn_schema:common_fields(). -desc(?CONF_NS) -> +desc(scram) -> "Settings for Salted Challenge Response Authentication Mechanism\n" "(SCRAM) authentication."; desc(_) -> @@ -141,7 +143,7 @@ iteration_count(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, ?CONF_NS)]. + [hoconsc:ref(?MODULE, scram)]. create( AuthenticatorID, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 3c34d878e..27eb8cc6e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -51,34 +51,35 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-http". +namespace() -> "authn". tags() -> [<<"Authentication">>]. +%% used for config check when the schema module is resolved roots() -> [ {?CONF_NS, hoconsc:mk( - hoconsc:union(fun union_member_selector/1), + hoconsc:union(fun ?MODULE:union_member_selector/1), #{} )} ]. -fields(get) -> +fields(http_get) -> [ {method, #{type => get, required => true, desc => ?DESC(method)}}, {headers, fun headers_no_content_type/1} ] ++ common_fields(); -fields(post) -> +fields(http_post) -> [ {method, #{type => post, required => true, desc => ?DESC(method)}}, {headers, fun headers/1} ] ++ common_fields(). -desc(get) -> +desc(http_get) -> ?DESC(get); -desc(post) -> +desc(http_post) -> ?DESC(post); desc(_) -> undefined. @@ -156,8 +157,8 @@ request_timeout(_) -> undefined. refs() -> [ - hoconsc:ref(?MODULE, get), - hoconsc:ref(?MODULE, post) + hoconsc:ref(?MODULE, http_get), + hoconsc:ref(?MODULE, http_post) ]. union_member_selector(all_union_members) -> @@ -166,9 +167,9 @@ union_member_selector({value, Value}) -> refs(Value). refs(#{<<"method">> := <<"get">>}) -> - [hoconsc:ref(?MODULE, get)]; + [hoconsc:ref(?MODULE, http_get)]; refs(#{<<"method">> := <<"post">>}) -> - [hoconsc:ref(?MODULE, post)]; + [hoconsc:ref(?MODULE, http_post)]; refs(_) -> throw(#{ field_name => method, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index a891a55e2..92c6970cc 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -43,36 +43,57 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-jwt". +namespace() -> "authn". tags() -> [<<"Authentication">>]. +%% used for config check when the schema module is resolved roots() -> [ {?CONF_NS, hoconsc:mk( - hoconsc:union(fun union_member_selector/1), + hoconsc:union(fun ?MODULE:union_member_selector/1), #{} )} ]. -fields('hmac-based') -> +fields(jwt_hmac) -> [ - {use_jwks, sc(hoconsc:enum([false]), #{required => true, desc => ?DESC(use_jwks)})}, + %% for hmac, it's the 'algorithm' field which selects this type + %% use_jwks field can be ignored (kept for backward compatibility) + {use_jwks, + sc( + hoconsc:enum([false]), + #{ + required => false, + desc => ?DESC(use_jwks), + importance => ?IMPORTANCE_HIDDEN + } + )}, {algorithm, sc(hoconsc:enum(['hmac-based']), #{required => true, desc => ?DESC(algorithm)})}, {secret, fun secret/1}, {secret_base64_encoded, fun secret_base64_encoded/1} ] ++ common_fields(); -fields('public-key') -> +fields(jwt_public_key) -> [ - {use_jwks, sc(hoconsc:enum([false]), #{required => true, desc => ?DESC(use_jwks)})}, + %% for public-key, it's the 'algorithm' field which selects this type + %% use_jwks field can be ignored (kept for backward compatibility) + {use_jwks, + sc( + hoconsc:enum([false]), + #{ + required => false, + desc => ?DESC(use_jwks), + importance => ?IMPORTANCE_HIDDEN + } + )}, {algorithm, sc(hoconsc:enum(['public-key']), #{required => true, desc => ?DESC(algorithm)})}, {public_key, fun public_key/1} ] ++ common_fields(); -fields('jwks') -> +fields(jwt_jwks) -> [ {use_jwks, sc(hoconsc:enum([true]), #{required => true, desc => ?DESC(use_jwks)})}, {endpoint, fun endpoint/1}, @@ -85,12 +106,12 @@ fields('jwks') -> }} ] ++ common_fields(). -desc('hmac-based') -> - ?DESC('hmac-based'); -desc('public-key') -> - ?DESC('public-key'); -desc('jwks') -> - ?DESC('jwks'); +desc(jwt_hmac) -> + ?DESC(jwt_hmac); +desc(jwt_public_key) -> + ?DESC(jwt_public_key); +desc(jwt_jwks) -> + ?DESC(jwt_jwks); desc(undefined) -> undefined. @@ -160,9 +181,9 @@ from(_) -> undefined. refs() -> [ - hoconsc:ref(?MODULE, 'hmac-based'), - hoconsc:ref(?MODULE, 'public-key'), - hoconsc:ref(?MODULE, 'jwks') + hoconsc:ref(?MODULE, jwt_hmac), + hoconsc:ref(?MODULE, jwt_public_key), + hoconsc:ref(?MODULE, jwt_jwks) ]. union_member_selector(all_union_members) -> @@ -181,9 +202,9 @@ boolean(Other) -> Other. select_ref(true, _) -> [hoconsc:ref(?MODULE, 'jwks')]; select_ref(false, #{<<"public_key">> := _}) -> - [hoconsc:ref(?MODULE, 'public-key')]; + [hoconsc:ref(?MODULE, jwt_public_key)]; select_ref(false, _) -> - [hoconsc:ref(?MODULE, 'hmac-based')]; + [hoconsc:ref(?MODULE, jwt_hmac)]; select_ref(_, _) -> throw(#{ field_name => use_jwks, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index 20b604e70..d57e9e00e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -107,14 +107,16 @@ mnesia(boot) -> %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-builtin_db". +namespace() -> "authn". tags() -> [<<"Authentication">>]. -roots() -> [?CONF_NS]. +%% used for config check when the schema module is resolved +roots() -> + [{?CONF_NS, hoconsc:mk(hoconsc:ref(?MODULE, builtin_db))}]. -fields(?CONF_NS) -> +fields(builtin_db) -> [ {mechanism, emqx_authn_schema:mechanism(password_based)}, {backend, emqx_authn_schema:backend(built_in_database)}, @@ -122,8 +124,8 @@ fields(?CONF_NS) -> {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1} ] ++ emqx_authn_schema:common_fields(). -desc(?CONF_NS) -> - ?DESC(?CONF_NS); +desc(builtin_db) -> + ?DESC(builtin_db); desc(_) -> undefined. @@ -138,7 +140,7 @@ user_id_type(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, ?CONF_NS)]. + [hoconsc:ref(?MODULE, builtin_db)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 22b930485..1a766b975 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -44,32 +44,33 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-mongodb". +namespace() -> "authn". tags() -> [<<"Authentication">>]. +%% used for config check when the schema module is resolved roots() -> [ {?CONF_NS, hoconsc:mk( - hoconsc:union(fun union_member_selector/1), + hoconsc:union(fun ?MODULE:union_member_selector/1), #{} )} ]. -fields(standalone) -> +fields(mongo_single) -> common_fields() ++ emqx_connector_mongo:fields(single); -fields('replica-set') -> +fields(mongo_rs) -> common_fields() ++ emqx_connector_mongo:fields(rs); -fields('sharded-cluster') -> +fields(mongo_sharded) -> common_fields() ++ emqx_connector_mongo:fields(sharded). -desc(standalone) -> - ?DESC(standalone); -desc('replica-set') -> +desc(mongo_single) -> + ?DESC(single); +desc(mongo_rs) -> ?DESC('replica-set'); -desc('sharded-cluster') -> +desc(mongo_sharded) -> ?DESC('sharded-cluster'); desc(_) -> undefined. @@ -126,9 +127,9 @@ is_superuser_field(_) -> undefined. refs() -> [ - hoconsc:ref(?MODULE, standalone), - hoconsc:ref(?MODULE, 'replica-set'), - hoconsc:ref(?MODULE, 'sharded-cluster') + hoconsc:ref(?MODULE, mongo_single), + hoconsc:ref(?MODULE, mongo_rs), + hoconsc:ref(?MODULE, mongo_sharded) ]. create(_AuthenticatorID, Config) -> @@ -254,11 +255,11 @@ union_member_selector({value, Value}) -> refs(Value). refs(#{<<"mongo_type">> := <<"single">>}) -> - [hoconsc:ref(?MODULE, standalone)]; + [hoconsc:ref(?MODULE, mongo_single)]; refs(#{<<"mongo_type">> := <<"rs">>}) -> - [hoconsc:ref(?MODULE, 'replica-set')]; + [hoconsc:ref(?MODULE, mongo_rs)]; refs(#{<<"mongo_type">> := <<"sharded">>}) -> - [hoconsc:ref(?MODULE, 'sharded-cluster')]; + [hoconsc:ref(?MODULE, mongo_sharded)]; refs(_) -> throw(#{ field_name => mongo_type, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index bedd169e2..d8e631885 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -45,14 +45,16 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-mysql". +namespace() -> "authn". tags() -> [<<"Authentication">>]. -roots() -> [?CONF_NS]. +%% used for config check when the schema module is resolved +roots() -> + [{?CONF_NS, hoconsc:mk(hoconsc:ref(?MODULE, mysql))}]. -fields(?CONF_NS) -> +fields(mysql) -> [ {mechanism, emqx_authn_schema:mechanism(password_based)}, {backend, emqx_authn_schema:backend(mysql)}, @@ -62,8 +64,8 @@ fields(?CONF_NS) -> ] ++ emqx_authn_schema:common_fields() ++ proplists:delete(prepare_statement, emqx_connector_mysql:fields(config)). -desc(?CONF_NS) -> - ?DESC(?CONF_NS); +desc(mysql) -> + ?DESC(mysql); desc(_) -> undefined. @@ -82,7 +84,7 @@ query_timeout(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, ?CONF_NS)]. + [hoconsc:ref(?MODULE, mysql)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 3d9e1e08f..d9526cc7b 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -49,14 +49,16 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-postgresql". +namespace() -> "authn". tags() -> [<<"Authentication">>]. -roots() -> [?CONF_NS]. +%% used for config check when the schema module is resolved +roots() -> + [{?CONF_NS, hoconsc:mk(hoconsc:ref(?MODULE, postgresql))}]. -fields(?CONF_NS) -> +fields(postgresql) -> [ {mechanism, emqx_authn_schema:mechanism(password_based)}, {backend, emqx_authn_schema:backend(postgresql)}, @@ -66,8 +68,8 @@ fields(?CONF_NS) -> emqx_authn_schema:common_fields() ++ proplists:delete(prepare_statement, emqx_connector_pgsql:fields(config)). -desc(?CONF_NS) -> - ?DESC(?CONF_NS); +desc(postgresql) -> + ?DESC(postgresql); desc(_) -> undefined. @@ -81,7 +83,7 @@ query(_) -> undefined. %%------------------------------------------------------------------------------ refs() -> - [hoconsc:ref(?MODULE, ?CONF_NS)]. + [hoconsc:ref(?MODULE, postgresql)]. create(_AuthenticatorID, Config) -> create(Config). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index ff81fd4ca..27d8c540a 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -44,32 +44,33 @@ %% Hocon Schema %%------------------------------------------------------------------------------ -namespace() -> "authn-redis". +namespace() -> "authn". tags() -> [<<"Authentication">>]. +%% used for config check when the schema module is resolved roots() -> [ {?CONF_NS, hoconsc:mk( - hoconsc:union(fun union_member_selector/1), + hoconsc:union(fun ?MODULE:union_member_selector/1), #{} )} ]. -fields(standalone) -> +fields(redis_single) -> common_fields() ++ emqx_connector_redis:fields(single); -fields(cluster) -> +fields(redis_cluster) -> common_fields() ++ emqx_connector_redis:fields(cluster); -fields(sentinel) -> +fields(redis_sentinel) -> common_fields() ++ emqx_connector_redis:fields(sentinel). -desc(standalone) -> - ?DESC(standalone); -desc(cluster) -> +desc(redis_single) -> + ?DESC(single); +desc(redis_cluster) -> ?DESC(cluster); -desc(sentinel) -> +desc(redis_sentinel) -> ?DESC(sentinel); desc(_) -> "". @@ -93,9 +94,9 @@ cmd(_) -> undefined. refs() -> [ - hoconsc:ref(?MODULE, standalone), - hoconsc:ref(?MODULE, cluster), - hoconsc:ref(?MODULE, sentinel) + hoconsc:ref(?MODULE, redis_single), + hoconsc:ref(?MODULE, redis_cluster), + hoconsc:ref(?MODULE, redis_sentinel) ]. union_member_selector(all_union_members) -> @@ -104,11 +105,11 @@ union_member_selector({value, Value}) -> refs(Value). refs(#{<<"redis_type">> := <<"single">>}) -> - [hoconsc:ref(?MODULE, standalone)]; + [hoconsc:ref(?MODULE, redis_single)]; refs(#{<<"redis_type">> := <<"cluster">>}) -> - [hoconsc:ref(?MODULE, cluster)]; + [hoconsc:ref(?MODULE, redis_cluster)]; refs(#{<<"redis_type">> := <<"sentinel">>}) -> - [hoconsc:ref(?MODULE, sentinel)]; + [hoconsc:ref(?MODULE, redis_sentinel)]; refs(_) -> throw(#{ field_name => redis_type, diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl index 65c783298..075ae5cb7 100644 --- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl @@ -107,7 +107,7 @@ t_update_with_invalid_config(_Config) -> ?assertMatch( {error, #{ kind := validation_error, - matched_type := "authn-postgresql:authentication", + matched_type := "authn:postgresql", path := "authentication.1.server", reason := required_field }}, diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl index 4f95bef93..1354e06cc 100644 --- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl @@ -162,7 +162,7 @@ t_create_invalid_config(_Config) -> ?assertMatch( {error, #{ kind := validation_error, - matched_type := "authn-redis:standalone", + matched_type := "authn:redis_single", path := "authentication.1.server", reason := required_field }}, diff --git a/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl b/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl index 7e67584ac..7a766281b 100644 --- a/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_schema_SUITE.erl @@ -53,7 +53,7 @@ t_check_schema(_Config) -> ?assertThrow( #{ path := "authentication.1.password_hash_algorithm.name", - matched_type := "authn-builtin_db:authentication/authn-hash:simple", + matched_type := "authn:builtin_db/authn-hash:simple", reason := unable_to_convert_to_enum_symbol }, Check(ConfigNotOk) @@ -72,7 +72,7 @@ t_check_schema(_Config) -> #{ path := "authentication.1.password_hash_algorithm", reason := "algorithm_name_missing", - matched_type := "authn-builtin_db:authentication" + matched_type := "authn:builtin_db" }, Check(ConfigMissingAlgoName) ). diff --git a/apps/emqx_authn/test/emqx_authn_schema_tests.erl b/apps/emqx_authn/test/emqx_authn_schema_tests.erl index 25fcd28e4..9de2c2e99 100644 --- a/apps/emqx_authn/test/emqx_authn_schema_tests.erl +++ b/apps/emqx_authn/test/emqx_authn_schema_tests.erl @@ -32,7 +32,7 @@ union_member_selector_mongo_test_() -> end}, {"single", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-mongodb:standalone"}), + ?ERR(#{matched_type := "authn:redis_single"}), Check("{mongo_type: single}") ) end}, diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index f03ae52a8..7aaa68b62 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -54,7 +54,7 @@ type_names() -> file, http_get, http_post, - mnesia, + builtin_db, mongo_single, mongo_rs, mongo_sharded, @@ -93,7 +93,7 @@ fields(http_post) -> {method, method(post)}, {headers, fun headers/1} ]; -fields(mnesia) -> +fields(builtin_db) -> authz_common_fields(built_in_database); fields(mongo_single) -> authz_common_fields(mongodb) ++ @@ -191,8 +191,8 @@ desc(http_get) -> ?DESC(http_get); desc(http_post) -> ?DESC(http_post); -desc(mnesia) -> - ?DESC(mnesia); +desc(builtin_db) -> + ?DESC(builtin_db); desc(mongo_single) -> ?DESC(mongo_single); desc(mongo_rs) -> @@ -459,7 +459,7 @@ select_union_member(#{<<"type">> := <<"http">>} = Value) -> }) end; select_union_member(#{<<"type">> := <<"built_in_database">>}) -> - ?R_REF(mnesia); + ?R_REF(builtin_db); select_union_member(#{<<"type">> := Type}) -> select_union_member_loop(Type, type_names()); select_union_member(_) -> @@ -494,7 +494,10 @@ authz_fields() -> default => [], desc => ?DESC(sources), %% doc_lift is force a root level reference instead of nesting sub-structs - extra => #{doc_lift => true} + extra => #{doc_lift => true}, + %% it is recommended to configure authz sources from dashboard + %% hance the importance level for config is low + importance => ?IMPORTANCE_LOW } )} ]. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index ae3691682..f3f014321 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -100,7 +100,7 @@ roots() -> ?R_REF("rpc"), #{ translate_to => ["gen_rpc"], - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_LOW } )} ] ++ @@ -1288,7 +1288,7 @@ emqx_schema_high_prio_roots() -> ?R_REF("authorization"), #{ desc => ?DESC(authorization), - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_HIGH } )}, lists:keyreplace("authorization", 1, Roots, Authz). From 48d8381a6df506b850d20fcbed734fb9f83b634f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 17 Apr 2023 21:22:28 +0200 Subject: [PATCH 206/279] docs: update doc refs --- rel/i18n/emqx_authn_jwt.hocon | 6 +++--- rel/i18n/emqx_authn_mnesia.hocon | 2 +- rel/i18n/emqx_authn_mongodb.hocon | 2 +- rel/i18n/emqx_authn_mysql.hocon | 2 +- rel/i18n/emqx_authn_pgsql.hocon | 2 +- rel/i18n/emqx_authn_redis.hocon | 2 +- rel/i18n/emqx_authz_schema.hocon | 6 +++--- rel/i18n/zh/emqx_authn_jwt.hocon | 6 +++--- rel/i18n/zh/emqx_authn_mnesia.hocon | 2 +- rel/i18n/zh/emqx_authn_mongodb.hocon | 2 +- rel/i18n/zh/emqx_authn_mysql.hocon | 2 +- rel/i18n/zh/emqx_authn_pgsql.hocon | 2 +- rel/i18n/zh/emqx_authn_redis.hocon | 2 +- rel/i18n/zh/emqx_authz_schema.hocon | 6 +++--- 14 files changed, 22 insertions(+), 22 deletions(-) diff --git a/rel/i18n/emqx_authn_jwt.hocon b/rel/i18n/emqx_authn_jwt.hocon index e28213e37..f947dcf5d 100644 --- a/rel/i18n/emqx_authn_jwt.hocon +++ b/rel/i18n/emqx_authn_jwt.hocon @@ -42,10 +42,10 @@ from.desc: from.label: """From Field""" -hmac-based.desc: +jwt_hmac.desc: """Configuration when the JWT for authentication is issued using the HMAC algorithm.""" -jwks.desc: +jwt_jwks.desc: """Configuration when JWTs used for authentication need to be fetched from the JWKS endpoint.""" keyfile.desc: @@ -54,7 +54,7 @@ keyfile.desc: keyfile.label: """Key File""" -public-key.desc: +jwt_public_key.desc: """Configuration when the JWT for authentication is issued using RSA or ECDSA algorithm.""" public_key.desc: diff --git a/rel/i18n/emqx_authn_mnesia.hocon b/rel/i18n/emqx_authn_mnesia.hocon index 4cfab9493..6770df090 100644 --- a/rel/i18n/emqx_authn_mnesia.hocon +++ b/rel/i18n/emqx_authn_mnesia.hocon @@ -1,6 +1,6 @@ emqx_authn_mnesia { -authentication.desc: +builtin_db.desc: """Configuration of authenticator using built-in database as data source.""" user_id_type.desc: diff --git a/rel/i18n/emqx_authn_mongodb.hocon b/rel/i18n/emqx_authn_mongodb.hocon index 97311f751..6d851f58f 100644 --- a/rel/i18n/emqx_authn_mongodb.hocon +++ b/rel/i18n/emqx_authn_mongodb.hocon @@ -39,7 +39,7 @@ salt_field.label: sharded-cluster.desc: """Configuration of authenticator using MongoDB (Sharded Cluster) as authentication data source.""" -standalone.desc: +single.desc: """Configuration of authenticator using MongoDB (Standalone) as authentication data source.""" } diff --git a/rel/i18n/emqx_authn_mysql.hocon b/rel/i18n/emqx_authn_mysql.hocon index 45634a4ad..5eb2d23e9 100644 --- a/rel/i18n/emqx_authn_mysql.hocon +++ b/rel/i18n/emqx_authn_mysql.hocon @@ -1,6 +1,6 @@ emqx_authn_mysql { -authentication.desc: +mysql.desc: """Configuration of authenticator using MySQL as authentication data source.""" query.desc: diff --git a/rel/i18n/emqx_authn_pgsql.hocon b/rel/i18n/emqx_authn_pgsql.hocon index a9d727785..696a17861 100644 --- a/rel/i18n/emqx_authn_pgsql.hocon +++ b/rel/i18n/emqx_authn_pgsql.hocon @@ -1,6 +1,6 @@ emqx_authn_pgsql { -authentication.desc: +postgresql.desc: """Configuration of authenticator using PostgreSQL as authentication data source.""" query.desc: diff --git a/rel/i18n/emqx_authn_redis.hocon b/rel/i18n/emqx_authn_redis.hocon index f39d54061..e8d98b9a1 100644 --- a/rel/i18n/emqx_authn_redis.hocon +++ b/rel/i18n/emqx_authn_redis.hocon @@ -12,7 +12,7 @@ cmd.label: sentinel.desc: """Configuration of authenticator using Redis (Sentinel) as authentication data source.""" -standalone.desc: +single.desc: """Configuration of authenticator using Redis (Standalone) as authentication data source.""" } diff --git a/rel/i18n/emqx_authz_schema.hocon b/rel/i18n/emqx_authz_schema.hocon index 9e5339b2c..5318eb769 100644 --- a/rel/i18n/emqx_authz_schema.hocon +++ b/rel/i18n/emqx_authz_schema.hocon @@ -120,11 +120,11 @@ node_error.desc: node_error.label: """Error in Node""" -mnesia.desc: +builtin_db.desc: """Authorization using a built-in database (mnesia).""" -mnesia.label: -"""mnesia""" +builtin_db.label: +"""Builtin Database""" enable.desc: """Set to true or false to disable this ACL provider""" diff --git a/rel/i18n/zh/emqx_authn_jwt.hocon b/rel/i18n/zh/emqx_authn_jwt.hocon index 2aa27c1de..76b999101 100644 --- a/rel/i18n/zh/emqx_authn_jwt.hocon +++ b/rel/i18n/zh/emqx_authn_jwt.hocon @@ -42,10 +42,10 @@ from.desc: from.label: """源字段""" -hmac-based.desc: +jwt_hmac.desc: """用于认证的 JWT 使用 HMAC 算法签发时的配置。""" -jwks.desc: +jwt_jwks.desc: """用于认证的 JWTs 需要从 JWKS 端点获取时的配置。""" keyfile.desc: @@ -54,7 +54,7 @@ keyfile.desc: keyfile.label: """私钥文件""" -public-key.desc: +jwt_public_key.desc: """用于认证的 JWT 使用 RSA 或 ECDSA 算法签发时的配置。""" public_key.desc: diff --git a/rel/i18n/zh/emqx_authn_mnesia.hocon b/rel/i18n/zh/emqx_authn_mnesia.hocon index 1ba394627..78fc96c75 100644 --- a/rel/i18n/zh/emqx_authn_mnesia.hocon +++ b/rel/i18n/zh/emqx_authn_mnesia.hocon @@ -1,6 +1,6 @@ emqx_authn_mnesia { -authentication.desc: +builtin_db.desc: """使用内置数据库作为认证数据源的认证器的配置项。""" user_id_type.desc: diff --git a/rel/i18n/zh/emqx_authn_mongodb.hocon b/rel/i18n/zh/emqx_authn_mongodb.hocon index 01419e2b9..949322a76 100644 --- a/rel/i18n/zh/emqx_authn_mongodb.hocon +++ b/rel/i18n/zh/emqx_authn_mongodb.hocon @@ -39,7 +39,7 @@ salt_field.label: sharded-cluster.desc: """使用 MongoDB (Sharded Cluster) 作为认证数据源的认证器的配置项。""" -standalone.desc: +single.desc: """使用 MongoDB (Standalone) 作为认证数据源的认证器的配置项。""" } diff --git a/rel/i18n/zh/emqx_authn_mysql.hocon b/rel/i18n/zh/emqx_authn_mysql.hocon index e718ad723..c8b72d720 100644 --- a/rel/i18n/zh/emqx_authn_mysql.hocon +++ b/rel/i18n/zh/emqx_authn_mysql.hocon @@ -1,6 +1,6 @@ emqx_authn_mysql { -authentication.desc: +mysql.desc: """使用 MySQL 作为认证数据源的认证器的配置项。""" query.desc: diff --git a/rel/i18n/zh/emqx_authn_pgsql.hocon b/rel/i18n/zh/emqx_authn_pgsql.hocon index 97bf608d2..b9d47828b 100644 --- a/rel/i18n/zh/emqx_authn_pgsql.hocon +++ b/rel/i18n/zh/emqx_authn_pgsql.hocon @@ -1,6 +1,6 @@ emqx_authn_pgsql { -authentication.desc: +postgresql.desc: """使用 PostgreSQL 作为认证数据源的认证器的配置项。""" query.desc: diff --git a/rel/i18n/zh/emqx_authn_redis.hocon b/rel/i18n/zh/emqx_authn_redis.hocon index e3d6b9d96..58137f195 100644 --- a/rel/i18n/zh/emqx_authn_redis.hocon +++ b/rel/i18n/zh/emqx_authn_redis.hocon @@ -12,7 +12,7 @@ cmd.label: sentinel.desc: """使用 Redis (Sentinel) 作为认证数据源的认证器的配置项。""" -standalone.desc: +single.desc: """使用 Redis (Standalone) 作为认证数据源的认证器的配置项。""" } diff --git a/rel/i18n/zh/emqx_authz_schema.hocon b/rel/i18n/zh/emqx_authz_schema.hocon index 3dd6d1c01..f0f973a55 100644 --- a/rel/i18n/zh/emqx_authz_schema.hocon +++ b/rel/i18n/zh/emqx_authz_schema.hocon @@ -120,11 +120,11 @@ node_error.desc: node_error.label: """节点产生的错误""" -mnesia.desc: +builtin_db.desc: """使用内部数据库授权（mnesia）。""" -mnesia.label: -"""mnesia""" +builtin_db.label: +"""Buitin Database""" enable.desc: """设为 true 或 false 以启用或禁用此访问控制数据源""" From cd1197925f8bee00c36bcfc4f95884e2bcc93962 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 09:14:02 +0200 Subject: [PATCH 207/279] test: fix unit tests after type names changed --- .../src/simple_authn/emqx_authn_jwt.erl | 2 +- .../test/emqx_authn_schema_tests.erl | 22 +++++++++---------- .../test/emqx_ee_bridge_influxdb_tests.erl | 12 ++++++++-- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index 92c6970cc..0df9014b8 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -200,7 +200,7 @@ boolean(<<"false">>) -> false; boolean(Other) -> Other. select_ref(true, _) -> - [hoconsc:ref(?MODULE, 'jwks')]; + [hoconsc:ref(?MODULE, 'jwt_jwks')]; select_ref(false, #{<<"public_key">> := _}) -> [hoconsc:ref(?MODULE, jwt_public_key)]; select_ref(false, _) -> diff --git a/apps/emqx_authn/test/emqx_authn_schema_tests.erl b/apps/emqx_authn/test/emqx_authn_schema_tests.erl index 9de2c2e99..622655b2d 100644 --- a/apps/emqx_authn/test/emqx_authn_schema_tests.erl +++ b/apps/emqx_authn/test/emqx_authn_schema_tests.erl @@ -32,19 +32,19 @@ union_member_selector_mongo_test_() -> end}, {"single", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn:redis_single"}), + ?ERR(#{matched_type := "authn:mongo_single"}), Check("{mongo_type: single}") ) end}, {"replica-set", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-mongodb:replica-set"}), + ?ERR(#{matched_type := "authn:mongo_rs"}), Check("{mongo_type: rs}") ) end}, {"sharded", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-mongodb:sharded-cluster"}), + ?ERR(#{matched_type := "authn:mongo_sharded"}), Check("{mongo_type: sharded}") ) end} @@ -61,19 +61,19 @@ union_member_selector_jwt_test_() -> end}, {"jwks", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-jwt:jwks"}), + ?ERR(#{matched_type := "authn:jwt_jwks"}), Check("{use_jwks = true}") ) end}, {"publick-key", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-jwt:public-key"}), + ?ERR(#{matched_type := "authn:jwt_public_key"}), Check("{use_jwks = false, public_key = 1}") ) end}, {"hmac-based", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-jwt:hmac-based"}), + ?ERR(#{matched_type := "authn:jwt_hmac"}), Check("{use_jwks = false}") ) end} @@ -90,19 +90,19 @@ union_member_selector_redis_test_() -> end}, {"single", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-redis:standalone"}), + ?ERR(#{matched_type := "authn:redis_single"}), Check("{redis_type = single}") ) end}, {"cluster", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-redis:cluster"}), + ?ERR(#{matched_type := "authn:redis_cluster"}), Check("{redis_type = cluster}") ) end}, {"sentinel", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-redis:sentinel"}), + ?ERR(#{matched_type := "authn:redis_sentinel"}), Check("{redis_type = sentinel}") ) end} @@ -119,13 +119,13 @@ union_member_selector_http_test_() -> end}, {"get", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-http:get"}), + ?ERR(#{matched_type := "authn:http_get"}), Check("{method = get}") ) end}, {"post", fun() -> ?assertMatch( - ?ERR(#{matched_type := "authn-http:post"}), + ?ERR(#{matched_type := "authn:http_post"}), Check("{method = post}") ) end} diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl index ce3a0b06f..1e065f6c8 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_influxdb_tests.erl @@ -5,8 +5,6 @@ -include_lib("eunit/include/eunit.hrl"). --import(emqx_ee_bridge_influxdb, [to_influx_lines/1]). - -define(INVALID_LINES, [ " ", " \n", @@ -326,3 +324,13 @@ test_pairs(PairsList) -> join(Sep, LinesList) -> lists:flatten(lists:join(Sep, LinesList)). + +to_influx_lines(RawLines) -> + OldLevel = emqx_logger:get_primary_log_level(), + try + %% mute error logs from this call + emqx_logger:set_primary_log_level(none), + emqx_ee_bridge_influxdb:to_influx_lines(RawLines) + after + emqx_logger:set_primary_log_level(OldLevel) + end. From 61f3e62ba8b2682fc133afa856e61a99d5b956fe Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 18 Apr 2023 15:49:34 +0800 Subject: [PATCH 208/279] chore: update changes --- changes/ce/perf-10430.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/perf-10430.en.md diff --git a/changes/ce/perf-10430.en.md b/changes/ce/perf-10430.en.md new file mode 100644 index 000000000..e01d8ba64 --- /dev/null +++ b/changes/ce/perf-10430.en.md @@ -0,0 +1,2 @@ +Simplify the configuration of the `retainer` feature. +- Mark `flow_control` as non-importance field. From 29584ca7215937578d969ce7dc13742df19f193d Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Fri, 14 Apr 2023 18:23:15 +0200 Subject: [PATCH 209/279] feat: add mongo_date functions to the rule engine This commit adds mong_date built-in functions to the rule engine SQL-like language. Corresponding functions already existed in EMQX 4.4 and this commit makes sure that EMQX 5.X also has these functions. Fixes: https://emqx.atlassian.net/browse/EMQX-9244 --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 28 ++++++ changes/ee/feat-10408.en.md | 1 + .../test/emqx_ee_bridge_mongodb_SUITE.erl | 75 +++++++++++++++- .../src/emqx_ee_connector_mongodb.erl | 88 ++++++++++++++++++- 4 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 changes/ee/feat-10408.en.md diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 81435d9ac..dfb79a40c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -230,6 +230,16 @@ date_to_unix_ts/4 ]). +%% MongoDB specific date functions. These functions return a date tuple. The +%% MongoDB bridge converts such date tuples to a MongoDB date type. The +%% following functions are therefore only useful for rules with at least one +%% MongoDB action. +-export([ + mongo_date/0, + mongo_date/1, + mongo_date/2 +]). + %% Proc Dict Func -export([ proc_dict_get/1, @@ -1135,3 +1145,21 @@ function_literal(Fun, [FArg | Args]) when is_atom(Fun), is_list(Args) -> ) ++ ")"; function_literal(Fun, Args) -> {invalid_func, {Fun, Args}}. + +mongo_date() -> + erlang:timestamp(). + +mongo_date(MillisecondsTimestamp) -> + convert_timestamp(MillisecondsTimestamp). + +mongo_date(Timestamp, Unit) -> + InsertedTimeUnit = time_unit(Unit), + ScaledEpoch = erlang:convert_time_unit(Timestamp, InsertedTimeUnit, millisecond), + convert_timestamp(ScaledEpoch). + +convert_timestamp(MillisecondsTimestamp) -> + MicroTimestamp = MillisecondsTimestamp * 1000, + MegaSecs = MicroTimestamp div 1000_000_000_000, + Secs = MicroTimestamp div 1000_000 - MegaSecs * 1000_000, + MicroSecs = MicroTimestamp rem 1000_000, + {MegaSecs, Secs, MicroSecs}. diff --git a/changes/ee/feat-10408.en.md b/changes/ee/feat-10408.en.md new file mode 100644 index 000000000..bc18b8a80 --- /dev/null +++ b/changes/ee/feat-10408.en.md @@ -0,0 +1 @@ +The rule engine SQL-like language has got three more built-in functions for creating values of the MongoDB date type. These functions are only useful for rules with at least one MongoDB bridge action. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 0959e3c78..1c3c4a2c3 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -27,7 +27,8 @@ group_tests() -> t_setup_via_config_and_publish, t_setup_via_http_api_and_publish, t_payload_template, - t_collection_template + t_collection_template, + t_mongo_date_rule_engine_functions ]. groups() -> @@ -140,10 +141,11 @@ start_apps() -> %% we want to make sure they are loaded before %% ekka start in emqx_common_test_helpers:start_apps/1 emqx_common_test_helpers:render_and_load_app_config(emqx_conf), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]). + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_rule_engine, emqx_bridge]). ensure_loaded() -> _ = application:load(emqx_ee_bridge), + _ = application:load(emqtt), _ = emqx_ee_bridge:module_info(), ok. @@ -282,6 +284,27 @@ find_all(Config) -> ResourceID = emqx_bridge_resource:resource_id(Type, Name), emqx_resource:simple_sync_query(ResourceID, {find, Collection, #{}, #{}}). +find_all_wait_until_non_empty(Config) -> + wait_until( + fun() -> + case find_all(Config) of + {ok, []} -> false; + _ -> true + end + end, + 5_000 + ), + find_all(Config). + +wait_until(Fun, Timeout) when Timeout >= 0 -> + case Fun() of + true -> + ok; + false -> + timer:sleep(100), + wait_until(Fun, Timeout - 100) + end. + send_message(Config, Payload) -> Name = ?config(mongo_name, Config), Type = mongo_type_bin(?config(mongo_type, Config)), @@ -376,3 +399,51 @@ t_collection_template(Config) -> find_all(Config) ), ok. + +t_mongo_date_rule_engine_functions(Config) -> + {ok, _} = + create_bridge( + Config, + #{ + <<"payload_template">> => + <<"{\"date_0\": ${date_0}, \"date_1\": ${date_1}, \"date_2\": ${date_2}}">> + } + ), + Type = mongo_type_bin(?config(mongo_type, Config)), + Name = ?config(mongo_name, Config), + SQL = + "SELECT mongo_date() as date_0, mongo_date(1000) as date_1, mongo_date(1, 'second') as date_2 FROM " + "\"t_mongo_date_rule_engine_functions/topic\"", + %% Remove rule if it already exists + RuleId = <<"rule:t_mongo_date_rule_engine_functions">>, + emqx_rule_engine:delete_rule(RuleId), + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), + {ok, Rule} = emqx_rule_engine:create_rule( + #{ + id => <<"rule:t_mongo_date_rule_engine_functions">>, + sql => SQL, + actions => [ + BridgeId, + #{function => console} + ], + description => <<"to mongo bridge">> + } + ), + %% Send a message to topic + {ok, Client} = emqtt:start_link([{clientid, <<"pub-02">>}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(Client), + emqtt:publish(Client, <<"t_mongo_date_rule_engine_functions/topic">>, #{}, <<"{\"x\":1}">>, [ + {qos, 2} + ]), + emqtt:stop(Client), + ?assertMatch( + {ok, [ + #{ + <<"date_0">> := {_, _, _}, + <<"date_1">> := {0, 1, 0}, + <<"date_2">> := {0, 1, 0} + } + ]}, + find_all_wait_until_non_empty(Config) + ), + ok. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl index 4e7adcd6e..90c422643 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl @@ -83,5 +83,89 @@ render_message(undefined = _PayloadTemplate, Message) -> render_message(PayloadTemplate, Message) -> %% Note: mongo expects a map as a document, so the rendered result %% must be JSON-serializable - Rendered = emqx_plugin_libs_rule:proc_tmpl(PayloadTemplate, Message), - emqx_utils_json:decode(Rendered, [return_maps]). + format_data(PayloadTemplate, Message). + +%% The following function was originally copied over from +%% https://github.com/emqx/emqx-enterprise/commit/50e3628129720f13f544053600ca1502731e29e0. +%% The rule engine has support for producing fields that are date tuples +%% (produced by the SQL language's built-in functions mongo_date/0, +%% mongo_date/1 and mongo_date/2) which the MongoDB driver recognizes and +%% converts to the MongoDB ISODate type +%% (https://www.compose.com/articles/understanding-dates-in-compose-mongodb/). +%% For this to work we have to replace the tuple values with references, make +%% an instance of the template, convert the instance to map with the help of +%% emqx_utils_json:decode and then finally replace the references with the +%% corresponding tuples in the resulting map. +format_data(PayloadTks, Msg) -> + % Check the Message for any tuples that need to be extracted before running the template though a json parser + PreparedTupleMap = create_mapping_of_references_to_tuple_values(Msg), + case maps:size(PreparedTupleMap) of + % If no tuples were found simply proceed with the json decoding and be done with it + 0 -> + emqx_utils_json:decode(emqx_plugin_libs_rule:proc_tmpl(PayloadTks, Msg), [return_maps]); + _ -> + % If tuples were found, replace the tuple values with the references created, run + % the modified message through the json parser, and then at the end replace the + % references with the actual tuple values. + ProcessedMessage = replace_message_values_with_references(Msg, PreparedTupleMap), + DecodedMap = emqx_utils_json:decode( + emqx_plugin_libs_rule:proc_tmpl(PayloadTks, ProcessedMessage), [return_maps] + ), + populate_map_with_tuple_values(PreparedTupleMap, DecodedMap) + end. + +replace_message_values_with_references(RawMessage, TupleMap) -> + % Iterate over every created reference/value pair and inject the reference into the message + maps:fold( + fun(Reference, OriginalValue, OriginalMessage) -> + % Iterate over the Message, which is a map, and look for the element which + % matches the Value in the map which holds the references/original values and replace + % with the reference + maps:fold( + fun(Key, Value, NewMap) -> + case Value == OriginalValue of + true -> + %% Wrap the reference in a string to make it JSON-serializable + StringRef = io_lib:format("\"~s\"", [Reference]), + WrappedRef = erlang:iolist_to_binary(StringRef), + maps:put(Key, WrappedRef, NewMap); + false -> + maps:put(Key, Value, NewMap) + end + end, + #{}, + OriginalMessage + ) + end, + RawMessage, + TupleMap + ). + +create_mapping_of_references_to_tuple_values(Message) -> + maps:fold( + fun + (_Key, Value, TupleMap) when is_tuple(Value) -> + Ref0 = emqx_guid:to_hexstr(emqx_guid:gen()), + Ref = <<"MONGO_DATE_REF_", Ref0/binary>>, + maps:put(Ref, Value, TupleMap); + (_key, _value, TupleMap) -> + TupleMap + end, + #{}, + Message + ). + +populate_map_with_tuple_values(TupleMap, MapToMap) -> + MappingFun = + fun + (_Key, Value) when is_map(Value) -> + populate_map_with_tuple_values(TupleMap, Value); + (_Key, Value) -> + case maps:is_key(Value, TupleMap) of + true -> + maps:get(Value, TupleMap); + false -> + Value + end + end, + maps:map(MappingFun, MapToMap). From 88360113106fded2af1d11899d4cb313616fbe5d Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 17 Apr 2023 18:08:03 +0200 Subject: [PATCH 210/279] docs: better changelog entry for mongo_date functions Co-authored-by: Thales Macedo Garitezi --- changes/ee/feat-10408.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ee/feat-10408.en.md b/changes/ee/feat-10408.en.md index bc18b8a80..75cc4b945 100644 --- a/changes/ee/feat-10408.en.md +++ b/changes/ee/feat-10408.en.md @@ -1 +1 @@ -The rule engine SQL-like language has got three more built-in functions for creating values of the MongoDB date type. These functions are only useful for rules with at least one MongoDB bridge action. +The rule engine SQL-like language has got three more built-in functions for creating values of the MongoDB date type. These functions are useful for rules with MongoDB bridge actions only and not supported in other actions. From d0f30d2f7ced0c0e89d056dd3854ad8f52b52123 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 18 Apr 2023 16:13:17 +0800 Subject: [PATCH 211/279] feat: delete trace/limiter config's hot update api --- apps/emqx_management/src/emqx_mgmt_api_configs.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index c94dc17b6..41bca45cf 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -42,8 +42,6 @@ <<"alarm">>, <<"sys_topics">>, <<"sysmon">>, - <<"limiter">>, - <<"trace">>, <<"log">>, <<"persistent_session_store">>, <<"zones">> From a6688ed07a7f4eb139f4535ce2df7f81ebf243f7 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 18 Apr 2023 10:24:39 +0200 Subject: [PATCH 212/279] refactor(emqx_management): Use emqx:running_node function --- apps/emqx_management/src/emqx_mgmt_api_stats.erl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_stats.erl b/apps/emqx_management/src/emqx_mgmt_api_stats.erl index 19857f267..5f4bbce65 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_stats.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_stats.erl @@ -145,12 +145,3 @@ list(get, #{query_string := Qs}) -> %%%============================================================================================== %% Internal - -running_nodes() -> - Nodes = erlang:nodes([visible, this]), - RpcResults = emqx_proto_v2:are_running(Nodes), - [ - Node - || {Node, IsRunning} <- lists:zip(Nodes, RpcResults), - IsRunning =:= {ok, true} - ]. From 8a3fccb3309b64e6ecf562580281d7262e480ce0 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Tue, 18 Apr 2023 10:32:37 +0200 Subject: [PATCH 213/279] style: fix variable name style --- lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl index 90c422643..59f763904 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl @@ -148,7 +148,7 @@ create_mapping_of_references_to_tuple_values(Message) -> Ref0 = emqx_guid:to_hexstr(emqx_guid:gen()), Ref = <<"MONGO_DATE_REF_", Ref0/binary>>, maps:put(Ref, Value, TupleMap); - (_key, _value, TupleMap) -> + (_Key, _Value, TupleMap) -> TupleMap end, #{}, From f3614b2c6577108461d7d8f44cc5281a4fb1253e Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 10 Apr 2023 10:59:46 +0800 Subject: [PATCH 214/279] chore: add README for slow subscriptions --- apps/emqx_slow_subs/README.md | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 apps/emqx_slow_subs/README.md diff --git a/apps/emqx_slow_subs/README.md b/apps/emqx_slow_subs/README.md new file mode 100644 index 000000000..e4454a56c --- /dev/null +++ b/apps/emqx_slow_subs/README.md @@ -0,0 +1,41 @@ +# EMQX Slow Subscriptions + +This application can calculate the latency (time spent) of the message to be processed and transmitted since it arrives at EMQX. + +If the latency exceeds a specified threshold, this application will add the subscriber and topic information to a slow subscriptions list or update the existing record. + +# How To Use + +You can add the below section into `emqx.conf` to enable this application + + +```yaml +slow_subs { + enable = true + threshold = "500ms" + expire_interval = "300s" + top_k_num = 10 + stats_type = whole +} +``` + +**threshold**: Latency threshold for statistics, only messages with latency exceeding this value will be collected. + +Minimum value: 100ms +Default value: 500ms + +**expire_interval**: Eviction time of the record, will start the counting since the creation of the record, and the records that are not triggered again within this specified period will be removed from the rank list. + +Default: 300s + +**top_k_num**: Maximum number of records in the slow subscription statistics record table. + +Maximum value: 1,000 +Default value: 10 + +**stats_type**: Calculation methods of the latency, which are +- **whole**: From the time the message arrives at EMQX until the message transmission completes +- **internal**: From when the message arrives at EMQX until when EMQX starts delivering the message +- **response**: From the time EMQX starts delivering the message until the message transmission completes + +Default value: whole From 69e334a77d9226c7c673e987008f53babc0b396e Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 18 Apr 2023 17:25:11 +0800 Subject: [PATCH 215/279] chore: reorganize the README to follow template --- apps/emqx_slow_subs/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/emqx_slow_subs/README.md b/apps/emqx_slow_subs/README.md index e4454a56c..cfc87f775 100644 --- a/apps/emqx_slow_subs/README.md +++ b/apps/emqx_slow_subs/README.md @@ -4,11 +4,12 @@ This application can calculate the latency (time spent) of the message to be pro If the latency exceeds a specified threshold, this application will add the subscriber and topic information to a slow subscriptions list or update the existing record. -# How To Use +More introduction: [Slow Subscriptions](https://www.emqx.io/docs/en/v5.0/observability/slow-subscribers-statistics.html) + +# Usage You can add the below section into `emqx.conf` to enable this application - ```yaml slow_subs { enable = true @@ -19,6 +20,8 @@ slow_subs { } ``` +# Configurations + **threshold**: Latency threshold for statistics, only messages with latency exceeding this value will be collected. Minimum value: 100ms @@ -28,7 +31,7 @@ Default value: 500ms Default: 300s -**top_k_num**: Maximum number of records in the slow subscription statistics record table. +**top_k_num**: Maximum number of records in the slow subscription statistics record table. Maximum value: 1,000 Default value: 10 @@ -39,3 +42,6 @@ Default value: 10 - **response**: From the time EMQX starts delivering the message until the message transmission completes Default value: whole + +# Contributing - [Mandatory] +Please see our [contributing.md](../../CONTRIBUTING.md). From 32e0a03c163f7d351bb4b8bd1bc72ddec3f22d89 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Tue, 18 Apr 2023 17:51:09 +0800 Subject: [PATCH 216/279] chore: upgrade dashboard to v1.2.2 for ce --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fe75b01bc..c6874234d 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.2.1 +export EMQX_DASHBOARD_VERSION ?= v1.2.2 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.5 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 From 21e19a33ce5bf0c240c82509454ae8467a6739ae Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 12 Apr 2023 16:37:51 +0300 Subject: [PATCH 217/279] feat(respool): switch to `emqx_resource_pool` Which was previously known as `emqx_plugin_libs_pool`. This is part of the effort to get rid of `emqx_plugin_libs` application. --- .../emqx_authn_jwks_connector.erl | 22 +++-- .../src/emqx_connector_http.erl | 5 +- .../src/emqx_connector_ldap.erl | 11 ++- .../src/emqx_connector_mongo.erl | 22 ++--- .../src/emqx_connector_mysql.erl | 27 +++--- .../src/emqx_connector_pgsql.erl | 37 ++++---- .../src/emqx_connector_redis.erl | 21 ++--- .../test/emqx_connector_mongo_SUITE.erl | 40 ++++----- .../test/emqx_connector_mysql_SUITE.erl | 44 ++++----- .../test/emqx_connector_pgsql_SUITE.erl | 40 ++++----- .../test/emqx_connector_redis_SUITE.erl | 40 ++++----- .../src/emqx_resource_pool.erl} | 28 +++--- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 10 +-- .../test/emqx_ee_bridge_mysql_SUITE.erl | 2 +- .../src/emqx_ee_connector_cassa.erl | 36 ++++---- .../src/emqx_ee_connector_clickhouse.erl | 79 +++++++--------- .../src/emqx_ee_connector_dynamo.erl | 14 +-- .../src/emqx_ee_connector_gcp_pubsub.erl | 90 +++++++------------ .../src/emqx_ee_connector_sqlserver.erl | 26 +++--- .../src/emqx_ee_connector_tdengine.erl | 14 +-- .../test/emqx_ee_connector_cassa_SUITE.erl | 38 ++++---- .../emqx_ee_connector_clickhouse_SUITE.erl | 38 ++++---- 22 files changed, 312 insertions(+), 372 deletions(-) rename apps/{emqx_plugin_libs/src/emqx_plugin_libs_pool.erl => emqx_resource/src/emqx_resource_pool.erl} (82%) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl index 1fdd7cef7..fe0349b4a 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwks_connector.erl @@ -35,18 +35,17 @@ callback_mode() -> always_sync. on_start(InstId, Opts) -> - PoolName = emqx_plugin_libs_pool:pool_name(InstId), PoolOpts = [ {pool_size, maps:get(pool_size, Opts, ?DEFAULT_POOL_SIZE)}, {connector_opts, Opts} ], - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, PoolOpts) of - ok -> {ok, #{pool_name => PoolName}}; + case emqx_resource_pool:start(InstId, ?MODULE, PoolOpts) of + ok -> {ok, #{pool_name => InstId}}; {error, Reason} -> {error, Reason} end. on_stop(_InstId, #{pool_name := PoolName}) -> - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query(InstId, get_jwks, #{pool_name := PoolName}) -> Result = ecpool:pick_and_do(PoolName, {emqx_authn_jwks_client, get_jwks, []}, no_handover), @@ -72,18 +71,17 @@ on_query(_InstId, {update, Opts}, #{pool_name := PoolName}) -> ok. on_get_status(_InstId, #{pool_name := PoolName}) -> - Func = - fun(Conn) -> - case emqx_authn_jwks_client:get_jwks(Conn) of - {ok, _} -> true; - _ -> false - end - end, - case emqx_plugin_libs_pool:health_check_ecpool_workers(PoolName, Func) of + case emqx_resource_pool:health_check_workers(PoolName, fun health_check/1) of true -> connected; false -> disconnected end. +health_check(Conn) -> + case emqx_authn_jwks_client:get_jwks(Conn) of + {ok, _} -> true; + _ -> false + end. + connect(Opts) -> ConnectorOpts = proplists:get_value(connector_opts, Opts), emqx_authn_jwks_client:start_link(ConnectorOpts). diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index ef2e11eb7..411df7899 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -231,9 +231,8 @@ on_start( {transport_opts, NTransportOpts}, {enable_pipelining, maps:get(enable_pipelining, Config, ?DEFAULT_PIPELINE_SIZE)} ], - PoolName = emqx_plugin_libs_pool:pool_name(InstId), State = #{ - pool_name => PoolName, + pool_name => InstId, pool_type => PoolType, host => Host, port => Port, @@ -241,7 +240,7 @@ on_start( base_path => BasePath, request => preprocess_request(maps:get(request, Config, undefined)) }, - case ehttpc_sup:start_pool(PoolName, PoolOpts) of + case ehttpc_sup:start_pool(InstId, PoolOpts) of {ok, _} -> {ok, State}; {error, {already_started, _}} -> {ok, State}; {error, Reason} -> {error, Reason} diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index ac2af301e..e2121de22 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -87,20 +87,19 @@ on_start( {pool_size, PoolSize}, {auto_reconnect, ?AUTO_RECONNECT_INTERVAL} ], - PoolName = emqx_plugin_libs_pool:pool_name(InstId), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ SslOpts) of - ok -> {ok, #{poolname => PoolName}}; + case emqx_resource_pool:start(InstId, ?MODULE, Opts ++ SslOpts) of + ok -> {ok, #{pool_name => InstId}}; {error, Reason} -> {error, Reason} end. -on_stop(InstId, #{poolname := PoolName}) -> +on_stop(InstId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_ldap_connector", connector => InstId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). -on_query(InstId, {search, Base, Filter, Attributes}, #{poolname := PoolName} = State) -> +on_query(InstId, {search, Base, Filter, Attributes}, #{pool_name := PoolName} = State) -> Request = {Base, Filter, Attributes}, ?TRACE( "QUERY", diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index a5873bcf6..a65a32842 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -182,12 +182,11 @@ on_start( {options, init_topology_options(maps:to_list(Topology), [])}, {worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)} ], - PoolName = emqx_plugin_libs_pool:pool_name(InstId), Collection = maps:get(collection, Config, <<"mqtt">>), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts) of + case emqx_resource_pool:start(InstId, ?MODULE, Opts) of ok -> {ok, #{ - poolname => PoolName, + pool_name => InstId, type => Type, collection => Collection }}; @@ -195,17 +194,17 @@ on_start( {error, Reason} end. -on_stop(InstId, #{poolname := PoolName}) -> +on_stop(InstId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_mongodb_connector", connector => InstId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query( InstId, {send_message, Document}, - #{poolname := PoolName, collection := Collection} = State + #{pool_name := PoolName, collection := Collection} = State ) -> Request = {insert, Collection, Document}, ?TRACE( @@ -234,7 +233,7 @@ on_query( on_query( InstId, {Action, Collection, Filter, Projector}, - #{poolname := PoolName} = State + #{pool_name := PoolName} = State ) -> Request = {Action, Collection, Filter, Projector}, ?TRACE( @@ -263,8 +262,7 @@ on_query( {ok, Result} end. --dialyzer({nowarn_function, [on_get_status/2]}). -on_get_status(InstId, #{poolname := PoolName} = _State) -> +on_get_status(InstId, #{pool_name := PoolName}) -> case health_check(PoolName) of true -> ?tp(debug, emqx_connector_mongo_health_check, #{ @@ -281,8 +279,10 @@ on_get_status(InstId, #{poolname := PoolName} = _State) -> end. health_check(PoolName) -> - emqx_plugin_libs_pool:health_check_ecpool_workers( - PoolName, fun ?MODULE:check_worker_health/1, ?HEALTH_CHECK_TIMEOUT + timer:seconds(1) + emqx_resource_pool:health_check_workers( + PoolName, + fun ?MODULE:check_worker_health/1, + ?HEALTH_CHECK_TIMEOUT + timer:seconds(1) ). %% =================================================================== diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 6600a5f77..45d459e70 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -51,7 +51,7 @@ -type sqls() :: #{atom() => binary()}. -type state() :: #{ - poolname := atom(), + pool_name := binary(), prepare_statement := prepares(), params_tokens := params_tokens(), batch_inserts := sqls(), @@ -123,13 +123,10 @@ on_start( {pool_size, PoolSize} ] ), - - PoolName = emqx_plugin_libs_pool:pool_name(InstId), - Prepares = parse_prepare_sql(Config), - State = maps:merge(#{poolname => PoolName}, Prepares), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of + State = parse_prepare_sql(Config), + case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of ok -> - {ok, init_prepare(State)}; + {ok, init_prepare(State#{pool_name => InstId})}; {error, Reason} -> ?tp( mysql_connector_start_failed, @@ -143,12 +140,12 @@ maybe_add_password_opt(undefined, Options) -> maybe_add_password_opt(Password, Options) -> [{password, Password} | Options]. -on_stop(InstId, #{poolname := PoolName}) -> +on_stop(InstId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_mysql_connector", connector => InstId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query(InstId, {TypeOrKey, SQLOrKey}, State) -> on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, State); @@ -157,7 +154,7 @@ on_query(InstId, {TypeOrKey, SQLOrKey, Params}, State) -> on_query( InstId, {TypeOrKey, SQLOrKey, Params, Timeout}, - #{poolname := PoolName, prepare_statement := Prepares} = State + #{pool_name := PoolName, prepare_statement := Prepares} = State ) -> MySqlFunction = mysql_function(TypeOrKey), {SQLOrKey2, Data} = proc_sql_params(TypeOrKey, SQLOrKey, Params, State), @@ -216,8 +213,8 @@ mysql_function(prepared_query) -> mysql_function(_) -> mysql_function(prepared_query). -on_get_status(_InstId, #{poolname := Pool} = State) -> - case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of +on_get_status(_InstId, #{pool_name := PoolName} = State) -> + case emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1) of true -> case do_check_prepares(State) of ok -> @@ -238,7 +235,7 @@ do_get_status(Conn) -> do_check_prepares(#{prepare_statement := Prepares}) when is_map(Prepares) -> ok; -do_check_prepares(State = #{poolname := PoolName, prepare_statement := {error, Prepares}}) -> +do_check_prepares(State = #{pool_name := PoolName, prepare_statement := {error, Prepares}}) -> %% retry to prepare case prepare_sql(Prepares, PoolName) of ok -> @@ -253,7 +250,7 @@ do_check_prepares(State = #{poolname := PoolName, prepare_statement := {error, P connect(Options) -> mysql:start_link(Options). -init_prepare(State = #{prepare_statement := Prepares, poolname := PoolName}) -> +init_prepare(State = #{prepare_statement := Prepares, pool_name := PoolName}) -> case maps:size(Prepares) of 0 -> State; @@ -409,7 +406,7 @@ on_sql_query( SQLOrKey, Params, Timeout, - #{poolname := PoolName} = State + #{pool_name := PoolName} = State ) -> LogMeta = #{connector => InstId, sql => SQLOrKey, state => State}, ?TRACE("QUERY", "mysql_connector_received", LogMeta), diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 8796e00a5..ddbf9491d 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -56,7 +56,7 @@ -type state() :: #{ - poolname := atom(), + pool_name := binary(), prepare_sql := prepares(), params_tokens := params_tokens(), prepare_statement := epgsql:statement() @@ -120,13 +120,10 @@ on_start( {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, {pool_size, PoolSize} ], - PoolName = emqx_plugin_libs_pool:pool_name(InstId), - Prepares = parse_prepare_sql(Config), - InitState = #{poolname => PoolName, prepare_statement => #{}}, - State = maps:merge(InitState, Prepares), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of + State = parse_prepare_sql(Config), + case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of ok -> - {ok, init_prepare(State)}; + {ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})}; {error, Reason} -> ?tp( pgsql_connector_start_failed, @@ -135,19 +132,19 @@ on_start( {error, Reason} end. -on_stop(InstId, #{poolname := PoolName}) -> +on_stop(InstId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping postgresql connector", connector => InstId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). -on_query(InstId, {TypeOrKey, NameOrSQL}, #{poolname := _PoolName} = State) -> +on_query(InstId, {TypeOrKey, NameOrSQL}, State) -> on_query(InstId, {TypeOrKey, NameOrSQL, []}, State); on_query( InstId, {TypeOrKey, NameOrSQL, Params}, - #{poolname := PoolName} = State + #{pool_name := PoolName} = State ) -> ?SLOG(debug, #{ msg => "postgresql connector received sql query", @@ -174,7 +171,7 @@ pgsql_query_type(_) -> on_batch_query( InstId, BatchReq, - #{poolname := PoolName, params_tokens := Tokens, prepare_statement := Sts} = State + #{pool_name := PoolName, params_tokens := Tokens, prepare_statement := Sts} = State ) -> case BatchReq of [{Key, _} = Request | _] -> @@ -258,8 +255,8 @@ on_sql_query(InstId, PoolName, Type, NameOrSQL, Data) -> {error, {unrecoverable_error, invalid_request}} end. -on_get_status(_InstId, #{poolname := Pool} = State) -> - case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of +on_get_status(_InstId, #{pool_name := PoolName} = State) -> + case emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1) of true -> case do_check_prepares(State) of ok -> @@ -280,7 +277,7 @@ do_get_status(Conn) -> do_check_prepares(#{prepare_sql := Prepares}) when is_map(Prepares) -> ok; -do_check_prepares(State = #{poolname := PoolName, prepare_sql := {error, Prepares}}) -> +do_check_prepares(State = #{pool_name := PoolName, prepare_sql := {error, Prepares}}) -> %% retry to prepare case prepare_sql(Prepares, PoolName) of {ok, Sts} -> @@ -358,7 +355,7 @@ parse_prepare_sql([], Prepares, Tokens) -> params_tokens => Tokens }. -init_prepare(State = #{prepare_sql := Prepares, poolname := PoolName}) -> +init_prepare(State = #{prepare_sql := Prepares, pool_name := PoolName}) -> case maps:size(Prepares) of 0 -> State; @@ -389,17 +386,17 @@ prepare_sql(Prepares, PoolName) -> end. do_prepare_sql(Prepares, PoolName) -> - do_prepare_sql(ecpool:workers(PoolName), Prepares, PoolName, #{}). + do_prepare_sql(ecpool:workers(PoolName), Prepares, #{}). -do_prepare_sql([{_Name, Worker} | T], Prepares, PoolName, _LastSts) -> +do_prepare_sql([{_Name, Worker} | T], Prepares, _LastSts) -> {ok, Conn} = ecpool_worker:client(Worker), case prepare_sql_to_conn(Conn, Prepares) of {ok, Sts} -> - do_prepare_sql(T, Prepares, PoolName, Sts); + do_prepare_sql(T, Prepares, Sts); Error -> Error end; -do_prepare_sql([], _Prepares, _PoolName, LastSts) -> +do_prepare_sql([], _Prepares, LastSts) -> {ok, LastSts}. prepare_sql_to_conn(Conn, Prepares) -> diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 4ef778e6b..e2155eb49 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -153,11 +153,10 @@ on_start( false -> [{ssl, false}] end ++ [{sentinel, maps:get(sentinel, Config, undefined)}], - PoolName = InstId, - State = #{poolname => PoolName, type => Type}, + State = #{pool_name => InstId, type => Type}, case Type of cluster -> - case eredis_cluster:start_pool(PoolName, Opts ++ [{options, Options}]) of + case eredis_cluster:start_pool(InstId, Opts ++ [{options, Options}]) of {ok, _} -> {ok, State}; {ok, _, _} -> @@ -166,22 +165,20 @@ on_start( {error, Reason} end; _ -> - case - emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts ++ [{options, Options}]) - of + case emqx_resource_pool:start(InstId, ?MODULE, Opts ++ [{options, Options}]) of ok -> {ok, State}; {error, Reason} -> {error, Reason} end end. -on_stop(InstId, #{poolname := PoolName, type := Type}) -> +on_stop(InstId, #{pool_name := PoolName, type := Type}) -> ?SLOG(info, #{ msg => "stopping_redis_connector", connector => InstId }), case Type of cluster -> eredis_cluster:stop_pool(PoolName); - _ -> emqx_plugin_libs_pool:stop_pool(PoolName) + _ -> emqx_resource_pool:stop(PoolName) end. on_query(InstId, {cmd, _} = Query, State) -> @@ -189,7 +186,7 @@ on_query(InstId, {cmd, _} = Query, State) -> on_query(InstId, {cmds, _} = Query, State) -> do_query(InstId, Query, State). -do_query(InstId, Query, #{poolname := PoolName, type := Type} = State) -> +do_query(InstId, Query, #{pool_name := PoolName, type := Type} = State) -> ?TRACE( "QUERY", "redis_connector_received", @@ -227,7 +224,7 @@ is_unrecoverable_error({error, invalid_cluster_command}) -> is_unrecoverable_error(_) -> false. -on_get_status(_InstId, #{type := cluster, poolname := PoolName}) -> +on_get_status(_InstId, #{type := cluster, pool_name := PoolName}) -> case eredis_cluster:pool_exists(PoolName) of true -> Health = eredis_cluster:ping_all(PoolName), @@ -235,8 +232,8 @@ on_get_status(_InstId, #{type := cluster, poolname := PoolName}) -> false -> disconnected end; -on_get_status(_InstId, #{poolname := Pool}) -> - Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1), +on_get_status(_InstId, #{pool_name := PoolName}) -> + Health = emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1), status_result(Health). do_get_status(Conn) -> diff --git a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl index 2be30466c..9067c85de 100644 --- a/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl @@ -64,15 +64,15 @@ t_lifecycle(_Config) -> mongo_config() ). -perform_lifecycle_check(PoolName, InitialConfig) -> +perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?MONGO_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MONGO_RESOURCE_MOD, CheckedConfig, @@ -84,39 +84,39 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % % Perform query as further check that the resource is working as expected - ?assertMatch({ok, []}, emqx_resource:query(PoolName, test_query_find())), - ?assertMatch({ok, undefined}, emqx_resource:query(PoolName, test_query_find_one())), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertMatch({ok, []}, emqx_resource:query(ResourceId, test_query_find())), + ?assertMatch({ok, undefined}, emqx_resource:query(ResourceId, test_query_find_one())), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceId), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceId)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), - ?assertMatch({ok, []}, emqx_resource:query(PoolName, test_query_find())), - ?assertMatch({ok, undefined}, emqx_resource:query(PoolName, test_query_find_one())), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + ?assertMatch({ok, []}, emqx_resource:query(ResourceId, test_query_find())), + ?assertMatch({ok, undefined}, emqx_resource:query(ResourceId, test_query_find_one())), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). % %%------------------------------------------------------------------------------ % %% Helpers diff --git a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl index dc5826766..a0455c92c 100644 --- a/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl @@ -64,14 +64,14 @@ t_lifecycle(_Config) -> mysql_config() ). -perform_lifecycle_check(PoolName, InitialConfig) -> +perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?MYSQL_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MYSQL_RESOURCE_MOD, CheckedConfig, @@ -83,53 +83,53 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % % Perform query as further check that the resource is working as expected - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_no_params())), - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_with_params())), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_no_params())), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_with_params())), ?assertMatch( {ok, _, [[1]]}, emqx_resource:query( - PoolName, + ResourceId, test_query_with_params_and_timeout() ) ), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceId), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceId)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_no_params())), - ?assertMatch({ok, _, [[1]]}, emqx_resource:query(PoolName, test_query_with_params())), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_no_params())), + ?assertMatch({ok, _, [[1]]}, emqx_resource:query(ResourceId, test_query_with_params())), ?assertMatch( {ok, _, [[1]]}, emqx_resource:query( - PoolName, + ResourceId, test_query_with_params_and_timeout() ) ), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). % %%------------------------------------------------------------------------------ % %% Helpers diff --git a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl index 2f77ca38d..a4ac4f932 100644 --- a/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_pgsql_SUITE.erl @@ -64,15 +64,15 @@ t_lifecycle(_Config) -> pgsql_config() ). -perform_lifecycle_check(PoolName, InitialConfig) -> +perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?PGSQL_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?PGSQL_RESOURCE_MOD, CheckedConfig, @@ -84,39 +84,39 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % % Perform query as further check that the resource is working as expected - ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_no_params())), - ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_with_params())), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertMatch({ok, _, [{1}]}, emqx_resource:query(ResourceId, test_query_no_params())), + ?assertMatch({ok, _, [{1}]}, emqx_resource:query(ResourceId, test_query_with_params())), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceId), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceId)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), - ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_no_params())), - ?assertMatch({ok, _, [{1}]}, emqx_resource:query(PoolName, test_query_with_params())), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + ?assertMatch({ok, _, [{1}]}, emqx_resource:query(ResourceId, test_query_no_params())), + ?assertMatch({ok, _, [{1}]}, emqx_resource:query(ResourceId, test_query_with_params())), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). % %%------------------------------------------------------------------------------ % %% Helpers diff --git a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl index 3a134ad35..e6df4f711 100644 --- a/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_redis_SUITE.erl @@ -102,14 +102,14 @@ t_sentinel_lifecycle(_Config) -> [<<"PING">>] ). -perform_lifecycle_check(PoolName, InitialConfig, RedisCommand) -> +perform_lifecycle_check(ResourceId, InitialConfig, RedisCommand) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?REDIS_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?REDIS_RESOURCE_MOD, CheckedConfig, @@ -121,49 +121,49 @@ perform_lifecycle_check(PoolName, InitialConfig, RedisCommand) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % Perform query as further check that the resource is working as expected - ?assertEqual({ok, <<"PONG">>}, emqx_resource:query(PoolName, {cmd, RedisCommand})), + ?assertEqual({ok, <<"PONG">>}, emqx_resource:query(ResourceId, {cmd, RedisCommand})), ?assertEqual( {ok, [{ok, <<"PONG">>}, {ok, <<"PONG">>}]}, - emqx_resource:query(PoolName, {cmds, [RedisCommand, RedisCommand]}) + emqx_resource:query(ResourceId, {cmds, [RedisCommand, RedisCommand]}) ), ?assertMatch( {error, {unrecoverable_error, [{ok, <<"PONG">>}, {error, _}]}}, emqx_resource:query( - PoolName, + ResourceId, {cmds, [RedisCommand, [<<"INVALID_COMMAND">>]]}, #{timeout => 500} ) ), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceId), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceId)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), - ?assertEqual({ok, <<"PONG">>}, emqx_resource:query(PoolName, {cmd, RedisCommand})), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), + ?assertEqual({ok, <<"PONG">>}, emqx_resource:query(ResourceId, {cmd, RedisCommand})), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). % %%------------------------------------------------------------------------------ % %% Helpers diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl b/apps/emqx_resource/src/emqx_resource_pool.erl similarity index 82% rename from apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl rename to apps/emqx_resource/src/emqx_resource_pool.erl index a3048e5dd..913b29c86 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl +++ b/apps/emqx_resource/src/emqx_resource_pool.erl @@ -14,31 +14,27 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_plugin_libs_pool). +-module(emqx_resource_pool). -export([ - start_pool/3, - stop_pool/1, - pool_name/1, - health_check_ecpool_workers/2, - health_check_ecpool_workers/3 + start/3, + stop/1, + health_check_workers/2, + health_check_workers/3 ]). -include_lib("emqx/include/logger.hrl"). -define(HEALTH_CHECK_TIMEOUT, 15000). -pool_name(ID) when is_binary(ID) -> - list_to_atom(binary_to_list(ID)). - -start_pool(Name, Mod, Options) -> +start(Name, Mod, Options) -> case ecpool:start_sup_pool(Name, Mod, Options) of {ok, _} -> ?SLOG(info, #{msg => "start_ecpool_ok", pool_name => Name}), ok; {error, {already_started, _Pid}} -> - stop_pool(Name), - start_pool(Name, Mod, Options); + stop(Name), + start(Name, Mod, Options); {error, Reason} -> NReason = parse_reason(Reason), ?SLOG(error, #{ @@ -49,7 +45,7 @@ start_pool(Name, Mod, Options) -> {error, {start_pool_failed, Name, NReason}} end. -stop_pool(Name) -> +stop(Name) -> case ecpool:stop_sup_pool(Name) of ok -> ?SLOG(info, #{msg => "stop_ecpool_ok", pool_name => Name}); @@ -64,10 +60,10 @@ stop_pool(Name) -> error({stop_pool_failed, Name, Reason}) end. -health_check_ecpool_workers(PoolName, CheckFunc) -> - health_check_ecpool_workers(PoolName, CheckFunc, ?HEALTH_CHECK_TIMEOUT). +health_check_workers(PoolName, CheckFunc) -> + health_check_workers(PoolName, CheckFunc, ?HEALTH_CHECK_TIMEOUT). -health_check_ecpool_workers(PoolName, CheckFunc, Timeout) -> +health_check_workers(PoolName, CheckFunc, Timeout) -> Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)], DoPerWorker = fun(Worker) -> diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index a785924d4..d2eb9ee73 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -917,7 +917,7 @@ t_invalid_private_key(Config) -> #{<<"private_key">> => InvalidPrivateKeyPEM} } ), - #{?snk_kind := gcp_pubsub_bridge_jwt_worker_failed_to_start}, + #{?snk_kind := "gcp_pubsub_bridge_jwt_worker_failed_to_start"}, 20_000 ), Res @@ -928,7 +928,7 @@ t_invalid_private_key(Config) -> [#{reason := Reason}] when Reason =:= noproc orelse Reason =:= {shutdown, {error, empty_key}}, - ?of_kind(gcp_pubsub_bridge_jwt_worker_failed_to_start, Trace) + ?of_kind("gcp_pubsub_bridge_jwt_worker_failed_to_start", Trace) ), ?assertMatch( [#{error := empty_key}], @@ -956,14 +956,14 @@ t_jwt_worker_start_timeout(Config) -> #{<<"private_key">> => InvalidPrivateKeyPEM} } ), - #{?snk_kind := gcp_pubsub_bridge_jwt_timeout}, + #{?snk_kind := "gcp_pubsub_bridge_jwt_timeout"}, 20_000 ), Res end, fun(Res, Trace) -> ?assertMatch({ok, _}, Res), - ?assertMatch([_], ?of_kind(gcp_pubsub_bridge_jwt_timeout, Trace)), + ?assertMatch([_], ?of_kind("gcp_pubsub_bridge_jwt_timeout", Trace)), ok end ), @@ -1329,7 +1329,7 @@ t_failed_to_start_jwt_worker(Config) -> fun(Trace) -> ?assertMatch( [#{reason := {error, restarting}}], - ?of_kind(gcp_pubsub_bridge_jwt_worker_failed_to_start, Trace) + ?of_kind("gcp_pubsub_bridge_jwt_worker_failed_to_start", Trace) ), ok end diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl index be07a2bb7..e497e0a47 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl @@ -265,7 +265,7 @@ unprepare(Config, Key) -> Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - {ok, _, #{state := #{poolname := PoolName}}} = emqx_resource:get_instance(ResourceID), + {ok, _, #{state := #{pool_name := PoolName}}} = emqx_resource:get_instance(ResourceID), [ begin {ok, Conn} = ecpool_worker:client(Worker), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl index 86b908038..397532f47 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl @@ -44,7 +44,7 @@ -type state() :: #{ - poolname := atom(), + pool_name := binary(), prepare_cql := prepares(), params_tokens := params_tokens(), %% returned by ecql:prepare/2 @@ -124,14 +124,10 @@ on_start( false -> [] end, - %% use InstaId of binary type as Pool name, which is supported in ecpool. - PoolName = InstId, - Prepares = parse_prepare_cql(Config), - InitState = #{poolname => PoolName, prepare_statement => #{}}, - State = maps:merge(InitState, Prepares), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts) of + State = parse_prepare_cql(Config), + case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of ok -> - {ok, init_prepare(State)}; + {ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})}; {error, Reason} -> ?tp( cassandra_connector_start_failed, @@ -140,12 +136,12 @@ on_start( {error, Reason} end. -on_stop(InstId, #{poolname := PoolName}) -> +on_stop(InstId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_cassandra_connector", connector => InstId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). -type request() :: % emqx_bridge.erl @@ -184,7 +180,7 @@ do_single_query( InstId, Request, Async, - #{poolname := PoolName} = State + #{pool_name := PoolName} = State ) -> {Type, PreparedKeyOrSQL, Params} = parse_request_to_cql(Request), ?tp( @@ -232,7 +228,7 @@ do_batch_query( InstId, Requests, Async, - #{poolname := PoolName} = State + #{pool_name := PoolName} = State ) -> CQLs = lists:map( @@ -305,8 +301,8 @@ exec_cql_batch_query(InstId, PoolName, Async, CQLs) -> Result end. -on_get_status(_InstId, #{poolname := Pool} = State) -> - case emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1) of +on_get_status(_InstId, #{pool_name := PoolName} = State) -> + case emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1) of true -> case do_check_prepares(State) of ok -> @@ -327,7 +323,7 @@ do_get_status(Conn) -> do_check_prepares(#{prepare_cql := Prepares}) when is_map(Prepares) -> ok; -do_check_prepares(State = #{poolname := PoolName, prepare_cql := {error, Prepares}}) -> +do_check_prepares(State = #{pool_name := PoolName, prepare_cql := {error, Prepares}}) -> %% retry to prepare case prepare_cql(Prepares, PoolName) of {ok, Sts} -> @@ -397,7 +393,7 @@ parse_prepare_cql([], Prepares, Tokens) -> params_tokens => Tokens }. -init_prepare(State = #{prepare_cql := Prepares, poolname := PoolName}) -> +init_prepare(State = #{prepare_cql := Prepares, pool_name := PoolName}) -> case maps:size(Prepares) of 0 -> State; @@ -429,17 +425,17 @@ prepare_cql(Prepares, PoolName) -> end. do_prepare_cql(Prepares, PoolName) -> - do_prepare_cql(ecpool:workers(PoolName), Prepares, PoolName, #{}). + do_prepare_cql(ecpool:workers(PoolName), Prepares, #{}). -do_prepare_cql([{_Name, Worker} | T], Prepares, PoolName, _LastSts) -> +do_prepare_cql([{_Name, Worker} | T], Prepares, _LastSts) -> {ok, Conn} = ecpool_worker:client(Worker), case prepare_cql_to_conn(Conn, Prepares) of {ok, Sts} -> - do_prepare_cql(T, Prepares, PoolName, Sts); + do_prepare_cql(T, Prepares, Sts); Error -> Error end; -do_prepare_cql([], _Prepares, _PoolName, LastSts) -> +do_prepare_cql([], _Prepares, LastSts) -> {ok, LastSts}. prepare_cql_to_conn(Conn, Prepares) -> diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl index 2872e8cf0..a7afcd6d5 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl @@ -62,7 +62,8 @@ -type state() :: #{ templates := templates(), - poolname := atom() + pool_name := binary(), + connect_timeout := pos_integer() }. -type clickhouse_config() :: map(). @@ -141,7 +142,6 @@ on_start( connector => InstanceID, config => emqx_utils:redact(Config) }), - PoolName = emqx_plugin_libs_pool:pool_name(InstanceID), Options = [ {url, URL}, {user, maps:get(username, Config, "default")}, @@ -149,46 +149,43 @@ on_start( {database, DB}, {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, {pool_size, PoolSize}, - {pool, PoolName} + {pool, InstanceID} ], - InitState = #{ - poolname => PoolName, - connect_timeout => ConnectTimeout - }, try Templates = prepare_sql_templates(Config), - State = maps:merge(InitState, #{templates => Templates}), - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options) of + State = #{ + pool_name => InstanceID, + templates => Templates, + connect_timeout => ConnectTimeout + }, + case emqx_resource_pool:start(InstanceID, ?MODULE, Options) of ok -> {ok, State}; {error, Reason} -> - log_start_error(Config, Reason, none), + ?tp( + info, + "clickhouse_connector_start_failed", + #{ + error => Reason, + config => emqx_utils:redact(Config) + } + ), {error, Reason} end catch _:CatchReason:Stacktrace -> - log_start_error(Config, CatchReason, Stacktrace), + ?tp( + info, + "clickhouse_connector_start_failed", + #{ + error => CatchReason, + stacktrace => Stacktrace, + config => emqx_utils:redact(Config) + } + ), {error, CatchReason} end. -log_start_error(Config, Reason, Stacktrace) -> - StacktraceMap = - case Stacktrace of - none -> #{}; - _ -> #{stacktrace => Stacktrace} - end, - LogMessage = - #{ - msg => "clickhouse_connector_start_failed", - error_reason => Reason, - config => emqx_utils:redact(Config) - }, - ?SLOG(info, maps:merge(LogMessage, StacktraceMap)), - ?tp( - clickhouse_connector_start_failed, - #{error => Reason} - ). - %% Helper functions to prepare SQL tempaltes prepare_sql_templates(#{ @@ -240,7 +237,7 @@ split_clickhouse_insert_sql(SQL) -> end. % This is a callback for ecpool which is triggered by the call to -% emqx_plugin_libs_pool:start_pool in on_start/2 +% emqx_resource_pool:start in on_start/2 connect(Options) -> URL = iolist_to_binary(emqx_http_lib:normalize(proplists:get_value(url, Options))), @@ -277,23 +274,20 @@ connect(Options) -> -spec on_stop(resource_id(), resource_state()) -> term(). -on_stop(ResourceID, #{poolname := PoolName}) -> +on_stop(InstanceID, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping clickouse connector", - connector => ResourceID + connector => InstanceID }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). %% ------------------------------------------------------------------- %% on_get_status emqx_resouce callback and related functions %% ------------------------------------------------------------------- on_get_status( - _InstId, - #{ - poolname := PoolName, - connect_timeout := Timeout - } = State + _InstanceID, + #{pool_name := PoolName, connect_timeout := Timeout} = State ) -> case do_get_status(PoolName, Timeout) of ok -> @@ -352,7 +346,7 @@ do_get_status(PoolName, Timeout) -> on_query( ResourceID, {RequestType, DataOrSQL}, - #{poolname := PoolName} = State + #{pool_name := PoolName} = State ) -> ?SLOG(debug, #{ msg => "clickhouse connector received sql query", @@ -391,16 +385,11 @@ query_type(send_message) -> on_batch_query( ResourceID, BatchReq, - State + #{pool_name := PoolName, templates := Templates} = _State ) -> %% Currently we only support batch requests with the send_message key {Keys, ObjectsToInsert} = lists:unzip(BatchReq), ensure_keys_are_of_type_send_message(Keys), - %% Pick out the SQL template - #{ - templates := Templates, - poolname := PoolName - } = State, %% Create batch insert SQL statement SQL = objects_to_sql(ObjectsToInsert, Templates), %% Do the actual query in the database diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 85daefbb7..1d273cdd7 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -114,23 +114,23 @@ on_start( Templates = parse_template(Config), State = #{ - poolname => InstanceId, + pool_name => InstanceId, database => Database, templates => Templates }, - case emqx_plugin_libs_pool:start_pool(InstanceId, ?MODULE, Options) of + case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of ok -> {ok, State}; Error -> Error end. -on_stop(InstanceId, #{poolname := PoolName} = _State) -> +on_stop(InstanceId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_dynamo_connector", connector => InstanceId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query(InstanceId, Query, State) -> do_query(InstanceId, Query, handover, State). @@ -160,8 +160,8 @@ on_batch_query_async(InstanceId, [{send_message, _} | _] = Query, Reply, State) on_batch_query_async(_InstanceId, Query, _Reply, _State) -> {error, {unrecoverable_error, {invalid_request, Query}}}. -on_get_status(_InstanceId, #{poolname := Pool}) -> - Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1), +on_get_status(_InstanceId, #{pool_name := PoolName}) -> + Health = emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1), status_result(Health). do_get_status(_Conn) -> @@ -183,7 +183,7 @@ do_query( InstanceId, Query, ApplyMode, - #{poolname := PoolName, templates := Templates, database := Database} = State + #{pool_name := PoolName, templates := Templates, database := Database} = State ) -> ?TRACE( "QUERY", diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl index 2295f63ab..7b068ec8f 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl @@ -26,7 +26,6 @@ ]). -export([reply_delegator/3]). --type bridge_id() :: binary(). -type jwt_worker() :: binary(). -type service_account_json() :: emqx_ee_bridge_gcp_pubsub:service_account_json(). -type config() :: #{ @@ -43,7 +42,7 @@ jwt_worker_id := jwt_worker(), max_retries := non_neg_integer(), payload_template := emqx_plugin_libs_rule:tmpl_token(), - pool_name := atom(), + pool_name := binary(), project_id := binary(), pubsub_topic := binary(), request_timeout := timer:time() @@ -102,14 +101,13 @@ on_start( jwt_worker_id := JWTWorkerId, project_id := ProjectId } = ensure_jwt_worker(InstanceId, Config), - PoolName = emqx_plugin_libs_pool:pool_name(InstanceId), State = #{ connect_timeout => ConnectTimeout, instance_id => InstanceId, jwt_worker_id => JWTWorkerId, max_retries => MaxRetries, payload_template => emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate), - pool_name => PoolName, + pool_name => InstanceId, project_id => ProjectId, pubsub_topic => PubSubTopic, request_timeout => RequestTimeout @@ -118,20 +116,20 @@ on_start( gcp_pubsub_on_start_before_starting_pool, #{ instance_id => InstanceId, - pool_name => PoolName, + pool_name => InstanceId, pool_opts => PoolOpts } ), - ?tp(gcp_pubsub_starting_ehttpc_pool, #{pool_name => PoolName}), - case ehttpc_sup:start_pool(PoolName, PoolOpts) of + ?tp(gcp_pubsub_starting_ehttpc_pool, #{pool_name => InstanceId}), + case ehttpc_sup:start_pool(InstanceId, PoolOpts) of {ok, _} -> {ok, State}; {error, {already_started, _}} -> - ?tp(gcp_pubsub_ehttpc_pool_already_started, #{pool_name => PoolName}), + ?tp(gcp_pubsub_ehttpc_pool_already_started, #{pool_name => InstanceId}), {ok, State}; {error, Reason} -> ?tp(gcp_pubsub_ehttpc_pool_start_failure, #{ - pool_name => PoolName, + pool_name => InstanceId, reason => Reason }), {error, Reason} @@ -140,10 +138,7 @@ on_start( -spec on_stop(manager_id(), state()) -> ok | {error, term()}. on_stop( InstanceId, - _State = #{ - jwt_worker_id := JWTWorkerId, - pool_name := PoolName - } + _State = #{jwt_worker_id := JWTWorkerId, pool_name := PoolName} ) -> ?tp(gcp_pubsub_stop, #{instance_id => InstanceId, jwt_worker_id => JWTWorkerId}), ?SLOG(info, #{ @@ -155,7 +150,7 @@ on_stop( ehttpc_sup:stop_pool(PoolName). -spec on_query( - bridge_id(), + resource_id(), {send_message, map()}, state() ) -> @@ -163,32 +158,32 @@ on_stop( | {ok, status_code(), headers(), body()} | {error, {recoverable_error, term()}} | {error, term()}. -on_query(BridgeId, {send_message, Selected}, State) -> +on_query(ResourceId, {send_message, Selected}, State) -> Requests = [{send_message, Selected}], ?TRACE( "QUERY_SYNC", "gcp_pubsub_received", - #{requests => Requests, connector => BridgeId, state => State} + #{requests => Requests, connector => ResourceId, state => State} ), - do_send_requests_sync(State, Requests, BridgeId). + do_send_requests_sync(State, Requests, ResourceId). -spec on_query_async( - bridge_id(), + resource_id(), {send_message, map()}, {ReplyFun :: function(), Args :: list()}, state() ) -> {ok, pid()}. -on_query_async(BridgeId, {send_message, Selected}, ReplyFunAndArgs, State) -> +on_query_async(ResourceId, {send_message, Selected}, ReplyFunAndArgs, State) -> Requests = [{send_message, Selected}], ?TRACE( "QUERY_ASYNC", "gcp_pubsub_received", - #{requests => Requests, connector => BridgeId, state => State} + #{requests => Requests, connector => ResourceId, state => State} ), - do_send_requests_async(State, Requests, ReplyFunAndArgs, BridgeId). + do_send_requests_async(State, Requests, ReplyFunAndArgs, ResourceId). -spec on_batch_query( - bridge_id(), + resource_id(), [{send_message, map()}], state() ) -> @@ -196,34 +191,30 @@ on_query_async(BridgeId, {send_message, Selected}, ReplyFunAndArgs, State) -> | {ok, status_code(), headers(), body()} | {error, {recoverable_error, term()}} | {error, term()}. -on_batch_query(BridgeId, Requests, State) -> +on_batch_query(ResourceId, Requests, State) -> ?TRACE( "QUERY_SYNC", "gcp_pubsub_received", - #{requests => Requests, connector => BridgeId, state => State} + #{requests => Requests, connector => ResourceId, state => State} ), - do_send_requests_sync(State, Requests, BridgeId). + do_send_requests_sync(State, Requests, ResourceId). -spec on_batch_query_async( - bridge_id(), + resource_id(), [{send_message, map()}], {ReplyFun :: function(), Args :: list()}, state() ) -> {ok, pid()}. -on_batch_query_async(BridgeId, Requests, ReplyFunAndArgs, State) -> +on_batch_query_async(ResourceId, Requests, ReplyFunAndArgs, State) -> ?TRACE( "QUERY_ASYNC", "gcp_pubsub_received", - #{requests => Requests, connector => BridgeId, state => State} + #{requests => Requests, connector => ResourceId, state => State} ), - do_send_requests_async(State, Requests, ReplyFunAndArgs, BridgeId). + do_send_requests_async(State, Requests, ReplyFunAndArgs, ResourceId). -spec on_get_status(manager_id(), state()) -> connected | disconnected. -on_get_status(InstanceId, State) -> - #{ - connect_timeout := Timeout, - pool_name := PoolName - } = State, +on_get_status(InstanceId, #{connect_timeout := Timeout, pool_name := PoolName} = State) -> case do_get_status(InstanceId, PoolName, Timeout) of true -> connected; @@ -245,8 +236,7 @@ on_get_status(InstanceId, State) -> project_id := binary() }. ensure_jwt_worker(InstanceId, #{ - service_account_json := ServiceAccountJSON, - pubsub_topic := PubSubTopic + service_account_json := ServiceAccountJSON }) -> #{ project_id := ProjectId, @@ -276,14 +266,8 @@ ensure_jwt_worker(InstanceId, #{ {ok, Worker0} -> Worker0; Error -> - ?tp( - gcp_pubsub_bridge_jwt_worker_failed_to_start, - #{instance_id => InstanceId, reason => Error} - ), - ?SLOG(error, #{ - msg => "failed_to_start_gcp_pubsub_jwt_worker", - instance_id => InstanceId, - pubsub_topic => PubSubTopic, + ?tp(error, "gcp_pubsub_bridge_jwt_worker_failed_to_start", #{ + connector => InstanceId, reason => Error }), _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId), @@ -301,26 +285,14 @@ ensure_jwt_worker(InstanceId, #{ demonitor(MRef, [flush]), ok; {'DOWN', MRef, process, Worker, Reason} -> - ?tp( - gcp_pubsub_bridge_jwt_worker_failed_to_start, - #{ - resource_id => InstanceId, - reason => Reason - } - ), - ?SLOG(error, #{ - msg => "gcp_pubsub_bridge_jwt_worker_failed_to_start", + ?tp(error, "gcp_pubsub_bridge_jwt_worker_failed_to_start", #{ connector => InstanceId, reason => Reason }), _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId), throw(failed_to_start_jwt_worker) after 10_000 -> - ?tp(gcp_pubsub_bridge_jwt_timeout, #{resource_id => InstanceId}), - ?SLOG(warning, #{ - msg => "gcp_pubsub_bridge_jwt_timeout", - connector => InstanceId - }), + ?tp(warning, "gcp_pubsub_bridge_jwt_timeout", #{connector => InstanceId}), demonitor(MRef, [flush]), _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId), throw(timeout_creating_jwt) @@ -569,7 +541,7 @@ reply_delegator(_ResourceId, ReplyFunAndArgs, Result) -> emqx_resource:apply_reply_fun(ReplyFunAndArgs, Result) end. --spec do_get_status(manager_id(), atom(), timer:time()) -> boolean(). +-spec do_get_status(manager_id(), binary(), timer:time()) -> boolean(). do_get_status(InstanceId, PoolName, Timeout) -> Workers = [Worker || {_WorkerName, Worker} <- ehttpc:workers(PoolName)], DoPerWorker = diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl index f11441a3b..70bd76d14 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_sqlserver.erl @@ -126,7 +126,7 @@ %% -type size() :: integer(). -type state() :: #{ - poolname := binary(), + pool_name := binary(), resource_opts := map(), sql_templates := map() }. @@ -208,17 +208,16 @@ on_start( {password, Password}, {driver, Driver}, {database, Database}, - {pool_size, PoolSize}, - {poolname, PoolName} + {pool_size, PoolSize} ], State = #{ %% also InstanceId - poolname => PoolName, + pool_name => PoolName, sql_templates => parse_sql_template(Config), resource_opts => ResourceOpts }, - case emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options) of + case emqx_resource_pool:start(PoolName, ?MODULE, Options) of ok -> {ok, State}; {error, Reason} -> @@ -229,12 +228,12 @@ on_start( {error, Reason} end. -on_stop(InstanceId, #{poolname := PoolName} = _State) -> +on_stop(InstanceId, #{pool_name := PoolName} = _State) -> ?SLOG(info, #{ msg => "stopping_sqlserver_connector", connector => InstanceId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). -spec on_query( manager_id(), @@ -265,7 +264,6 @@ on_query_async( InstanceId, {?ACTION_SEND_MESSAGE, _Msg} = Query, ReplyFunAndArgs, - %% #{poolname := PoolName, sql_templates := Templates} = State State ) -> ?TRACE( @@ -306,10 +304,12 @@ on_batch_query_async(InstanceId, Requests, ReplyFunAndArgs, State) -> ), do_query(InstanceId, Requests, ?ASYNC_QUERY_MODE(ReplyFunAndArgs), State). -on_get_status(_InstanceId, #{poolname := Pool, resource_opts := ResourceOpts} = _State) -> +on_get_status(_InstanceId, #{pool_name := PoolName, resource_opts := ResourceOpts} = _State) -> RequestTimeout = ?REQUEST_TIMEOUT(ResourceOpts), - Health = emqx_plugin_libs_pool:health_check_ecpool_workers( - Pool, {?MODULE, do_get_status, [RequestTimeout]}, RequestTimeout + Health = emqx_resource_pool:health_check_workers( + PoolName, + {?MODULE, do_get_status, [RequestTimeout]}, + RequestTimeout ), status_result(Health). @@ -382,7 +382,7 @@ do_query( InstanceId, Query, ApplyMode, - #{poolname := PoolName, sql_templates := Templates} = State + #{pool_name := PoolName, sql_templates := Templates} = State ) -> ?TRACE( "SINGLE_QUERY_SYNC", @@ -425,7 +425,7 @@ do_query( end. worker_do_insert( - Conn, SQL, #{resource_opts := ResourceOpts, poolname := InstanceId} = State + Conn, SQL, #{resource_opts := ResourceOpts, pool_name := InstanceId} = State ) -> LogMeta = #{connector => InstanceId, state => State}, try diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl index 9b6718882..f9ca21ad7 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl @@ -107,20 +107,20 @@ on_start( ], Prepares = parse_prepare_sql(Config), - State = maps:merge(Prepares, #{poolname => InstanceId, query_opts => query_opts(Config)}), - case emqx_plugin_libs_pool:start_pool(InstanceId, ?MODULE, Options) of + State = Prepares#{pool_name => InstanceId, query_opts => query_opts(Config)}, + case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of ok -> {ok, State}; Error -> Error end. -on_stop(InstanceId, #{poolname := PoolName} = _State) -> +on_stop(InstanceId, #{pool_name := PoolName}) -> ?SLOG(info, #{ msg => "stopping_tdengine_connector", connector => InstanceId }), - emqx_plugin_libs_pool:stop_pool(PoolName). + emqx_resource_pool:stop(PoolName). on_query(InstanceId, {query, SQL}, State) -> do_query(InstanceId, SQL, State); @@ -150,8 +150,8 @@ on_batch_query( {error, {unrecoverable_error, invalid_request}} end. -on_get_status(_InstanceId, #{poolname := Pool}) -> - Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1), +on_get_status(_InstanceId, #{pool_name := PoolName}) -> + Health = emqx_resource_pool:health_check_workers(PoolName, fun ?MODULE:do_get_status/1), status_result(Health). do_get_status(Conn) -> @@ -171,7 +171,7 @@ do_batch_insert(InstanceId, BatchReqs, InsertPart, Tokens, State) -> SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, InsertPart, Tokens), do_query(InstanceId, SQL, State). -do_query(InstanceId, Query, #{poolname := PoolName, query_opts := Opts} = State) -> +do_query(InstanceId, Query, #{pool_name := PoolName, query_opts := Opts} = State) -> ?TRACE( "QUERY", "tdengine_connector_received", diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl index 52ed03a62..f2647d756 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl @@ -101,15 +101,15 @@ show(Label, What) -> erlang:display({Label, What}), What. -perform_lifecycle_check(PoolName, InitialConfig) -> +perform_lifecycle_check(ResourceId, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?CASSANDRA_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?CASSANDRA_RESOURCE_MOD, CheckedConfig, @@ -121,45 +121,45 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), % % Perform query as further check that the resource is working as expected (fun() -> - erlang:display({pool_name, PoolName}), - QueryNoParamsResWrapper = emqx_resource:query(PoolName, test_query_no_params()), + erlang:display({pool_name, ResourceId}), + QueryNoParamsResWrapper = emqx_resource:query(ResourceId, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper) end)(), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceId), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceId)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceId)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceId)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceId), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceId)), (fun() -> QueryNoParamsResWrapper = - emqx_resource:query(PoolName, test_query_no_params()), + emqx_resource:query(ResourceId, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper) end)(), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceId)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceId)). %%-------------------------------------------------------------------- %% utils diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl index 73018e14f..e704a2c0c 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl @@ -95,15 +95,15 @@ show(Label, What) -> erlang:display({Label, What}), What. -perform_lifecycle_check(PoolName, InitialConfig) -> +perform_lifecycle_check(ResourceID, InitialConfig) -> {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?CLICKHOUSE_RESOURCE_MOD, InitialConfig), {ok, #{ - state := #{poolname := ReturnedPoolName} = State, + state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - PoolName, + ResourceID, ?CONNECTOR_RESOURCE_GROUP, ?CLICKHOUSE_RESOURCE_MOD, CheckedConfig, @@ -115,49 +115,49 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := State, status := InitialStatus }} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceID), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceID)), % % Perform query as further check that the resource is working as expected (fun() -> - erlang:display({pool_name, PoolName}), - QueryNoParamsResWrapper = emqx_resource:query(PoolName, test_query_no_params()), + erlang:display({pool_name, ResourceID}), + QueryNoParamsResWrapper = emqx_resource:query(ResourceID, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper), {_, QueryNoParamsRes} = QueryNoParamsResWrapper, ?assertMatch(<<"1">>, string:trim(QueryNoParamsRes)) end)(), - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceID)), % Resource will be listed still, but state will be changed and healthcheck will fail % as the worker no longer exists. {ok, ?CONNECTOR_RESOURCE_GROUP, #{ state := State, status := StoppedStatus }} = - emqx_resource:get_instance(PoolName), + emqx_resource:get_instance(ResourceID), ?assertEqual(stopped, StoppedStatus), - ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(ResourceID)), % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Can call stop/1 again on an already stopped instance - ?assertEqual(ok, emqx_resource:stop(PoolName)), + ?assertEqual(ok, emqx_resource:stop(ResourceID)), % Make sure it can be restarted and the healthchecks and queries work properly - ?assertEqual(ok, emqx_resource:restart(PoolName)), + ?assertEqual(ok, emqx_resource:restart(ResourceID)), % async restart, need to wait resource timer:sleep(500), {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = - emqx_resource:get_instance(PoolName), - ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + emqx_resource:get_instance(ResourceID), + ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceID)), (fun() -> QueryNoParamsResWrapper = - emqx_resource:query(PoolName, test_query_no_params()), + emqx_resource:query(ResourceID, test_query_no_params()), ?assertMatch({ok, _}, QueryNoParamsResWrapper), {_, QueryNoParamsRes} = QueryNoParamsResWrapper, ?assertMatch(<<"1">>, string:trim(QueryNoParamsRes)) end)(), % Stop and remove the resource in one go. - ?assertEqual(ok, emqx_resource:remove_local(PoolName)), - ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + ?assertEqual(ok, emqx_resource:remove_local(ResourceID)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(PoolName)), % Should not even be able to get the resource data out of ets now unlike just stopping. - ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + ?assertEqual({error, not_found}, emqx_resource:get_instance(ResourceID)). % %%------------------------------------------------------------------------------ % %% Helpers From a08c0002229aa079af013954b8b64c9c328fa51f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 12 Apr 2023 12:37:36 +0300 Subject: [PATCH 218/279] test(mongo): do not rely on internals to clean database --- .../test/emqx_ee_bridge_mongodb_SUITE.erl | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl index 0959e3c78..fc4270fd8 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl @@ -147,6 +147,16 @@ ensure_loaded() -> _ = emqx_ee_bridge:module_info(), ok. +mongo_type(Config) -> + case ?config(mongo_type, Config) of + rs -> + {rs, maps:get(<<"replica_set_name">>, ?config(mongo_config, Config))}; + sharded -> + sharded; + single -> + single + end. + mongo_type_bin(rs) -> <<"mongodb_rs">>; mongo_type_bin(sharded) -> @@ -263,17 +273,14 @@ create_bridge_http(Params) -> end. clear_db(Config) -> - Type = mongo_type_bin(?config(mongo_type, Config)), - Name = ?config(mongo_name, Config), - #{<<"collection">> := Collection} = ?config(mongo_config, Config), - ResourceID = emqx_bridge_resource:resource_id(Type, Name), - {ok, _, #{state := #{connector_state := #{poolname := PoolName}}}} = - emqx_resource:get_instance(ResourceID), - Selector = #{}, - {true, _} = ecpool:pick_and_do( - PoolName, {mongo_api, delete, [Collection, Selector]}, no_handover - ), - ok. + Type = mongo_type(Config), + Host = ?config(mongo_host, Config), + Port = ?config(mongo_port, Config), + Server = Host ++ ":" ++ integer_to_list(Port), + #{<<"database">> := Db, <<"collection">> := Collection} = ?config(mongo_config, Config), + {ok, Client} = mongo_api:connect(Type, [Server], [], [{database, Db}, {w_mode, unsafe}]), + {true, _} = mongo_api:delete(Client, Collection, _Selector = #{}), + mongo_api:disconnect(Client). find_all(Config) -> Type = mongo_type_bin(?config(mongo_type, Config)), From e3d6fa1f21a3997da02f2af78d0a67cac187d6c8 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 18 Apr 2023 14:24:08 +0800 Subject: [PATCH 219/279] chore: update config's changelog and emqx_conf.template --- apps/emqx_conf/etc/emqx_conf.conf | 17 ++++++++++------- changes/ce/feat-10358.en.md | 2 -- changes/ce/feat-10381.en.md | 1 - changes/ce/feat-10385.en.md | 1 - changes/ce/feat-10391.en.md | 1 - changes/ce/feat-10426.en.md | 10 ++++++++++ rel/emqx_conf.template.en.md | 11 +++++------ rel/emqx_conf.template.zh.md | 6 +++--- 8 files changed, 28 insertions(+), 21 deletions(-) delete mode 100644 changes/ce/feat-10358.en.md delete mode 100644 changes/ce/feat-10381.en.md delete mode 100644 changes/ce/feat-10385.en.md delete mode 100644 changes/ce/feat-10391.en.md create mode 100644 changes/ce/feat-10426.en.md diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index 86147bf25..45733d158 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -1,11 +1,14 @@ ## NOTE: -## Configs in this file might be overridden by: -## 1. Environment variables which start with 'EMQX_' prefix -## 2. File $EMQX_NODE__DATA_DIR/configs/cluster-override.conf -## 3. File $EMQX_NODE__DATA_DIR/configs/local-override.conf -## -## The *-override.conf files are overwritten at runtime when changes -## are made from EMQX dashboard UI, management HTTP API, or CLI. +## The order of priority for configuration is: +## environment variables, 'etc/emqx.conf', and 'data/configs/cluster.hocon'. +## 1. Settings in environment variables starting with 'EMQX_' are given the highest priority. +## 2. Configuration settings in the 'etc/emqx.conf' file may be overwritten by environment variables. +## 3. Changes made through the EMQX dashboard UI, management HTTP API, or CLI +## overwrite the 'data/configs/cluster.hocon' file at runtime. +## 4. If you modify a configuration setting using the API, the change takes effect immediately. +## 5. However, if the same setting is configured differently in the 'etc/emqx.conf' file, +## 'etc/emqx.conf' takes priority after a reboot. + ## All configuration details can be found in emqx.conf.example node { diff --git a/changes/ce/feat-10358.en.md b/changes/ce/feat-10358.en.md deleted file mode 100644 index e6d05c84b..000000000 --- a/changes/ce/feat-10358.en.md +++ /dev/null @@ -1,2 +0,0 @@ -Hide `flapping_detect/conn_congestion/stats` configuration. -Deprecate `flapping_detect.enable`. diff --git a/changes/ce/feat-10381.en.md b/changes/ce/feat-10381.en.md deleted file mode 100644 index 3ea11188f..000000000 --- a/changes/ce/feat-10381.en.md +++ /dev/null @@ -1 +0,0 @@ -Hide the `auto_subscribe` configuration items so that they can be modified later only through the HTTP API. diff --git a/changes/ce/feat-10385.en.md b/changes/ce/feat-10385.en.md deleted file mode 100644 index 667e01890..000000000 --- a/changes/ce/feat-10385.en.md +++ /dev/null @@ -1 +0,0 @@ -Hide data items(rule_engine/bridge/authz/authn) from configuration files and documentation. diff --git a/changes/ce/feat-10391.en.md b/changes/ce/feat-10391.en.md deleted file mode 100644 index a64b01221..000000000 --- a/changes/ce/feat-10391.en.md +++ /dev/null @@ -1 +0,0 @@ -hide exhook/rewrite/topic_metric/persistent_session_store/overload_protection from the docs and configuration file. diff --git a/changes/ce/feat-10426.en.md b/changes/ce/feat-10426.en.md new file mode 100644 index 000000000..660d4baa2 --- /dev/null +++ b/changes/ce/feat-10426.en.md @@ -0,0 +1,10 @@ +1. Changes in configuration priority: + For new EMQX installations, configuration priority is `ENV > emqx.conf > HTTP API`. + For upgrades from an older version with cluster-override.conf, configuration priority remains the same: `HTTP API > ENV > emqx.conf`. +2. Deprecation of `data/configs/local-override.conf`. +3. Simplified configuration items, hidden some advanced items + The hidden configurations include: exhook,rewrite,topic_metric,persistent_session_store,overload_protection, + flapping_detect,conn_congestion,stats,auto_subscribe,broker_perf,rule_engine,bridge,shared_subscription_group,slow_subs + and some advance items in node/dashboard. +4. This hidden update doesn't change the functionality of the original configuration, + but it sets the stage for an improved presentation of configuration documentation in future versions. diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index 8740e4319..b63b50a14 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -9,18 +9,18 @@ From bottom up: 1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.
Changes in this layer require a full node restart to take effect. -1. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` +2. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster.hocon` When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir` is used. -The `cluster-override.conf` file is overwritten at runtime when changes +The `cluster.hocon` file is overwritten at runtime when changes are made from dashboard UI, management HTTP API, or CLI. When clustered, after EMQX restarts, it copies the file from the node which has the greatest `uptime`. :::tip Tip Some of the configs (such as `node.name`) are boot-only configs and not overridable. -Config values from `*-override.conf` are **not** mapped to boot configs for +Config values from `cluster.hocon` are **not** mapped to boot configs for the config fields attributed with `mapping: path.to.boot.config.key` ::: @@ -148,7 +148,7 @@ export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_ However this also means a string value should be quoted if it happens to contain special characters such as `=` and `:`. -For example, a string value `"localhost:1883"` would be +For example, a string value `"localhost:1883"` would be parsed into object (struct): `{"localhost": 1883}`. To keep it as a string, one should quote the value like below: @@ -226,7 +226,7 @@ Arrays in EMQX config have two different representations Dot-separated paths with number in it are parsed to indexed-maps e.g. `authentication.1={...}` is parsed as `authentication={"1": {...}}` -This feature makes it easy to override array elment values. For example: +This feature makes it easy to override array element values. For example: ``` authentication=[{enable=true, backend="built_in_database", mechanism="password_based"}] @@ -322,4 +322,3 @@ ciphers = "PSK-AES128-CBC-SHA" ] ``` - diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index 9402760a2..916eed38d 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -7,18 +7,18 @@ EMQX的配置文件可分为二层，自底向上依次是： 1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。
修改这一层的配置之后，需要重启节点来使之生效。 -1. 集群范围重载层：`$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` +2. 集群范围重载层：`$EMQX_NODE__DATA_DIR/configs/cluster.hocon` 如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置，那么该目录会从 `emqx.conf` 的 `node.data_dir` 配置中读取。 -配置文件 `cluster-override.conf` 的内容会在运行时被EMQX重写。 +配置文件 `cluster.hocon` 的内容会在运行时被EMQX重写。 这些重写发生在 dashboard UI，管理HTTP API，或者CLI对集群配置进行修改时。 当EMQX运行在集群中时，一个EMQX节点重启之后，会从集群中其他节点复制该文件内容到本地。 :::tip Tip 有些配置项是不能被重载的（例如 `node.name`）。 配置项如果有 `mapping: path.to.boot.config.key` 这个属性， -则不能被添加到重载文件 `*-override.conf` 中。 +则不能被添加到重载文件 `cluster.hocon` 中。 ::: 更多的重载规则，请参考下文 [配置重载规则](#配置重载规则)。 From 22a1d05d7b64859249b182373c10a6c6976f66c1 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 18 Apr 2023 14:38:17 +0800 Subject: [PATCH 220/279] feat: hide ssl_options.user_lookup_fun --- apps/emqx/src/emqx_schema.erl | 2 ++ changes/ce/feat-10426.en.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8073e19b5..38566f3cb 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2225,6 +2225,7 @@ common_ssl_opts_schema(Defaults) -> #{ default => AvailableVersions, desc => ?DESC(common_ssl_opts_schema_versions), + importance => ?IMPORTANCE_HIGH, validator => fun(Inputs) -> validate_tls_versions(AvailableVersions, Inputs) end } )}, @@ -2235,6 +2236,7 @@ common_ssl_opts_schema(Defaults) -> #{ default => <<"emqx_tls_psk:lookup">>, converter => fun ?MODULE:user_lookup_fun_tr/2, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(common_ssl_opts_schema_user_lookup_fun) } )}, diff --git a/changes/ce/feat-10426.en.md b/changes/ce/feat-10426.en.md index 660d4baa2..b169dd19b 100644 --- a/changes/ce/feat-10426.en.md +++ b/changes/ce/feat-10426.en.md @@ -5,6 +5,6 @@ 3. Simplified configuration items, hidden some advanced items The hidden configurations include: exhook,rewrite,topic_metric,persistent_session_store,overload_protection, flapping_detect,conn_congestion,stats,auto_subscribe,broker_perf,rule_engine,bridge,shared_subscription_group,slow_subs - and some advance items in node/dashboard. + ssl_options.user_lookup_fun and some advance items in node/dashboard. 4. This hidden update doesn't change the functionality of the original configuration, but it sets the stage for an improved presentation of configuration documentation in future versions. From e3a84e0010eb3ea07e8b4e2f1b243d666c4d8e43 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 18 Apr 2023 15:50:08 +0800 Subject: [PATCH 221/279] chore: improve the changes for configuration --- changes/ce/feat-10391.en.md | 7 +++++++ changes/ce/feat-10426.en.md | 14 ++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 changes/ce/feat-10391.en.md diff --git a/changes/ce/feat-10391.en.md b/changes/ce/feat-10391.en.md new file mode 100644 index 000000000..5c37436dd --- /dev/null +++ b/changes/ce/feat-10391.en.md @@ -0,0 +1,7 @@ +Hide a large number of advanced options to simplify the configuration file. + +That includes `exhook`, `rewrite`, `topic_metric`, `persistent_session_store`, `overload_protection`, +`flapping_detect`, `conn_congestion`, `stats,auto_subscribe`, `broker_perf`, `rule_engine`, `bridge`, +`shared_subscription_group`, `slow_subs`, `ssl_options.user_lookup_fun` and some advance items +in `node` and `dashboard` section, [#10358](https://github.com/emqx/emqx/pull/10358), +[#10381](https://github.com/emqx/emqx/pull/10381), [#10385](https://github.com/emqx/emqx/pull/10385). diff --git a/changes/ce/feat-10426.en.md b/changes/ce/feat-10426.en.md index b169dd19b..8575347d6 100644 --- a/changes/ce/feat-10426.en.md +++ b/changes/ce/feat-10426.en.md @@ -1,10 +1,4 @@ -1. Changes in configuration priority: - For new EMQX installations, configuration priority is `ENV > emqx.conf > HTTP API`. - For upgrades from an older version with cluster-override.conf, configuration priority remains the same: `HTTP API > ENV > emqx.conf`. -2. Deprecation of `data/configs/local-override.conf`. -3. Simplified configuration items, hidden some advanced items - The hidden configurations include: exhook,rewrite,topic_metric,persistent_session_store,overload_protection, - flapping_detect,conn_congestion,stats,auto_subscribe,broker_perf,rule_engine,bridge,shared_subscription_group,slow_subs - ssl_options.user_lookup_fun and some advance items in node/dashboard. -4. This hidden update doesn't change the functionality of the original configuration, - but it sets the stage for an improved presentation of configuration documentation in future versions. +Optimize the configuration priority mechanism to fix the issue where the configuration +changes made to `etc/emqx.conf` do not take effect after restarting EMQX. + +More introduction about the new mechanism: [Configure Override Rules](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html#configure-override-rules) From fbadfc06e421c749f08b33a0e09decda9e801801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Tue, 18 Apr 2023 20:15:19 +0800 Subject: [PATCH 222/279] feat: change exhook, rule_engine, bridge to low importance level instead of hidden --- apps/emqx_bridge/src/schema/emqx_bridge_schema.erl | 2 +- apps/emqx_exhook/src/emqx_exhook_schema.erl | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 2 +- changes/ce/feat-10391.en.md | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index e5def2d64..4b9b7e3fe 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -137,7 +137,7 @@ namespace() -> "bridge". tags() -> [<<"Bridge">>]. -roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_HIDDEN})}]. +roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_LOW})}]. fields(bridges) -> [ diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 708e164fc..f6cc896f3 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -32,7 +32,7 @@ namespace() -> exhook. roots() -> - [{exhook, ?HOCON(?R_REF(exhook), #{importance => ?IMPORTANCE_HIDDEN})}]. + [{exhook, ?HOCON(?R_REF(exhook), #{importance => ?IMPORTANCE_LOW})}]. fields(exhook) -> [ diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 242c86c71..bc8cae07a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -38,7 +38,7 @@ namespace() -> rule_engine. tags() -> [<<"Rule Engine">>]. -roots() -> [{"rule_engine", ?HOCON(?R_REF("rule_engine"), #{importance => ?IMPORTANCE_HIDDEN})}]. +roots() -> [{"rule_engine", ?HOCON(?R_REF("rule_engine"), #{importance => ?IMPORTANCE_LOW})}]. fields("rule_engine") -> rule_engine_settings() ++ diff --git a/changes/ce/feat-10391.en.md b/changes/ce/feat-10391.en.md index 5c37436dd..f33757404 100644 --- a/changes/ce/feat-10391.en.md +++ b/changes/ce/feat-10391.en.md @@ -1,7 +1,7 @@ Hide a large number of advanced options to simplify the configuration file. -That includes `exhook`, `rewrite`, `topic_metric`, `persistent_session_store`, `overload_protection`, -`flapping_detect`, `conn_congestion`, `stats,auto_subscribe`, `broker_perf`, `rule_engine`, `bridge`, +That includes `rewrite`, `topic_metric`, `persistent_session_store`, `overload_protection`, +`flapping_detect`, `conn_congestion`, `stats,auto_subscribe`, `broker_perf`, `shared_subscription_group`, `slow_subs`, `ssl_options.user_lookup_fun` and some advance items in `node` and `dashboard` section, [#10358](https://github.com/emqx/emqx/pull/10358), [#10381](https://github.com/emqx/emqx/pull/10381), [#10385](https://github.com/emqx/emqx/pull/10385). From 444196922cb9523e6bf8eae915ed2b49c4298ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Tue, 18 Apr 2023 20:18:46 +0800 Subject: [PATCH 223/279] chore: update emqx.conf's note --- apps/emqx_conf/etc/emqx_conf.conf | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index 45733d158..8ad84fd7c 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -1,13 +1,17 @@ ## NOTE: -## The order of priority for configuration is: -## environment variables, 'etc/emqx.conf', and 'data/configs/cluster.hocon'. -## 1. Settings in environment variables starting with 'EMQX_' are given the highest priority. -## 2. Configuration settings in the 'etc/emqx.conf' file may be overwritten by environment variables. -## 3. Changes made through the EMQX dashboard UI, management HTTP API, or CLI -## overwrite the 'data/configs/cluster.hocon' file at runtime. -## 4. If you modify a configuration setting using the API, the change takes effect immediately. -## 5. However, if the same setting is configured differently in the 'etc/emqx.conf' file, -## 'etc/emqx.conf' takes priority after a reboot. +## The EMQX configuration is prioritized (overlayed) in the following order: +## `cluster.hocon < emqx.conf < environment variables`. + +## Settings in environment variables that begin with 'EMQX_' have the highest priority +## and will override any settings in the `etc/emqx.conf` file. + +## Changes made through the EMQX dashboard UI, management HTTP API, or CLI +## will be written into the `data/configs/cluster.hocon` file at runtime and will take effect immediately. + +## However, if the same configuration items are set differently in the `etc/emqx.conf` file, +## the runtime updates will be overridden by the settings in `etc/emqx.conf` after the node restarts. + +## To avoid confusion, it is highly recommend NOT to have the same config keys in both `cluster.hocon` and `emqx.conf`. ## All configuration details can be found in emqx.conf.example From 9ca09383ba725887e2d00829fd6ae153a7a1550a Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 18 Apr 2023 21:07:02 +0800 Subject: [PATCH 224/279] chore: remove desc from emqx.conf Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_conf/etc/emqx_conf.conf | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index 8ad84fd7c..dea04e9e4 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -2,18 +2,6 @@ ## The EMQX configuration is prioritized (overlayed) in the following order: ## `cluster.hocon < emqx.conf < environment variables`. -## Settings in environment variables that begin with 'EMQX_' have the highest priority -## and will override any settings in the `etc/emqx.conf` file. - -## Changes made through the EMQX dashboard UI, management HTTP API, or CLI -## will be written into the `data/configs/cluster.hocon` file at runtime and will take effect immediately. - -## However, if the same configuration items are set differently in the `etc/emqx.conf` file, -## the runtime updates will be overridden by the settings in `etc/emqx.conf` after the node restarts. - -## To avoid confusion, it is highly recommend NOT to have the same config keys in both `cluster.hocon` and `emqx.conf`. - -## All configuration details can be found in emqx.conf.example node { name = "emqx@127.0.0.1" From 59182ee0fce6e5be75c54508af645b44268dedbd Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 18 Apr 2023 21:07:16 +0800 Subject: [PATCH 225/279] chore: update apps/emqx_conf/etc/emqx_conf.conf Co-authored-by: Zaiming (Stone) Shi --- apps/emqx_conf/etc/emqx_conf.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index dea04e9e4..a54894dcd 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -1,6 +1,6 @@ ## NOTE: ## The EMQX configuration is prioritized (overlayed) in the following order: -## `cluster.hocon < emqx.conf < environment variables`. +## `data/configs/cluster.hocon < etc/emqx.conf < environment variables`. node { From 97b8f00508a57824191d9d56833e9b032c4c94fe Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 15:07:57 +0200 Subject: [PATCH 226/279] test: pick random port number for gcp pubsub mock server --- .../test/emqx_connector_web_hook_server.erl | 26 +++++++++++++- .../test/emqx_ee_bridge_gcp_pubsub_SUITE.erl | 35 +++++++++---------- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/apps/emqx_connector/test/emqx_connector_web_hook_server.erl b/apps/emqx_connector/test/emqx_connector_web_hook_server.erl index b68ebcbba..bdc6e100c 100644 --- a/apps/emqx_connector/test/emqx_connector_web_hook_server.erl +++ b/apps/emqx_connector/test/emqx_connector_web_hook_server.erl @@ -29,7 +29,14 @@ start_link(Port, Path) -> start_link(Port, Path, false). start_link(Port, Path, SSLOpts) -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [Port, Path, SSLOpts]). + case Port of + random -> + PickedPort = pick_port_number(56000), + {ok, Pid} = supervisor:start_link({local, ?MODULE}, ?MODULE, [PickedPort, Path, SSLOpts]), + {ok, {PickedPort, Pid}}; + _ -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [Port, Path, SSLOpts]) + end. stop() -> try @@ -103,3 +110,20 @@ default_handler(Req0, State) -> Req0 ), {ok, Req, State}. + +pick_port_number(Port) -> + case is_port_in_use(Port) of + true -> + pick_port_number(Port + 1); + false -> + Port + end. + +is_port_in_use(Port) -> + case gen_tcp:listen(Port, [{reuseaddr, true}, {active, false}]) of + {ok, ListenSocket} -> + gen_tcp:close(ListenSocket), + false; + {error, eaddrinuse} -> + true + end. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl index a785924d4..b8377d814 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl @@ -70,22 +70,13 @@ init_per_suite(Config) -> ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), - HTTPHost = "localhost", - HTTPPort = 56000, - HostPort = HTTPHost ++ ":" ++ integer_to_list(HTTPPort), - true = os:putenv("PUBSUB_EMULATOR_HOST", HostPort), - [ - {http_host, HTTPHost}, - {http_port, HTTPPort} - | Config - ]. + Config. end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), ok = emqx_common_test_helpers:stop_apps([emqx_conf]), ok = emqx_connector_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_rule_engine]), _ = application:stop(emqx_connector), - os:unsetenv("PUBSUB_EMULATOR_HOST"), ok. init_per_group(sync_query, Config) -> @@ -113,26 +104,26 @@ init_per_testcase(TestCase, Config0) when 1 -> [{skip_due_to_no_batching, true}]; _ -> - {ok, _} = start_echo_http_server(), delete_all_bridges(), Tid = install_telemetry_handler(TestCase), Config = generate_config(Config0), put(telemetry_table, Tid), - [{telemetry_table, Tid} | Config] + {ok, HttpServer} = start_echo_http_server(), + [{telemetry_table, Tid}, {http_server, HttpServer} | Config] end; init_per_testcase(TestCase, Config0) -> ct:timetrap({seconds, 30}), - {ok, _} = start_echo_http_server(), + {ok, HttpServer} = start_echo_http_server(), delete_all_bridges(), Tid = install_telemetry_handler(TestCase), Config = generate_config(Config0), put(telemetry_table, Tid), - [{telemetry_table, Tid} | Config]. + [{telemetry_table, Tid}, {http_server, HttpServer} | Config]. end_per_testcase(_TestCase, _Config) -> ok = snabbkaffe:stop(), delete_all_bridges(), - ok = emqx_connector_web_hook_server:stop(), + ok = stop_echo_http_server(), emqx_common_test_helpers:call_janitor(), ok. @@ -242,7 +233,6 @@ success_http_handler() -> start_echo_http_server() -> HTTPHost = "localhost", - HTTPPort = 56000, HTTPPath = <<"/v1/projects/myproject/topics/mytopic:publish">>, ServerSSLOpts = [ @@ -250,14 +240,23 @@ start_echo_http_server() -> {versions, ['tlsv1.2', 'tlsv1.3']}, {ciphers, ["ECDHE-RSA-AES256-GCM-SHA384", "TLS_CHACHA20_POLY1305_SHA256"]} ] ++ certs(), - {ok, _} = emqx_connector_web_hook_server:start_link(HTTPPort, HTTPPath, ServerSSLOpts), + {ok, {HTTPPort, _Pid}} = emqx_connector_web_hook_server:start_link( + random, HTTPPath, ServerSSLOpts + ), ok = emqx_connector_web_hook_server:set_handler(success_http_handler()), + HTTPHost = "localhost", + HostPort = HTTPHost ++ ":" ++ integer_to_list(HTTPPort), + true = os:putenv("PUBSUB_EMULATOR_HOST", HostPort), {ok, #{ - host_port => HTTPHost ++ ":" ++ integer_to_list(HTTPPort), + host_port => HostPort, host => HTTPHost, port => HTTPPort }}. +stop_echo_http_server() -> + os:unsetenv("PUBSUB_EMULATOR_HOST"), + ok = emqx_connector_web_hook_server:stop(). + certs() -> CertsPath = emqx_common_test_helpers:deps_path(emqx, "etc/certs"), [ From 3cd9ca706795b2ccd5c3a14f0270a77a031450c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=90=E6=96=87?= Date: Tue, 18 Apr 2023 21:10:40 +0800 Subject: [PATCH 227/279] chore: change config seq --- rel/emqx_conf.template.en.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index b63b50a14..bded20da6 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -7,9 +7,10 @@ and a superset of JSON. EMQX configuration consists of two layers. From bottom up: -1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.
+1. Cluster configs: `$EMQX_NODE__DATA_DIR/configs/cluster.hocon` +2. `emqx.conf` + `EMQX_` prefixed environment variables.
Changes in this layer require a full node restart to take effect. -2. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster.hocon` + When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir` is used. @@ -144,8 +145,7 @@ For example, this environment variable sets an array value. ``` export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]' ``` - -However this also means a string value should be quoted if it happens to contain special +However, this also means a string value should be quoted if it happens to contain special characters such as `=` and `:`. For example, a string value `"localhost:1883"` would be From 87c517069608724d72f08a199ff11ac9f2342d13 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 18 Apr 2023 21:33:41 +0800 Subject: [PATCH 228/279] chore: fix flaky test --- .../test/emqx_ee_bridge_cassa_SUITE.erl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl index 3e442a926..4711d1981 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl @@ -530,15 +530,16 @@ t_write_failure(Config) -> fun(Trace0) -> ct:pal("trace: ~p", [Trace0]), Trace = ?of_kind(buffer_worker_flush_nack, Trace0), - ?assertMatch([#{result := {async_return, {error, _}}} | _], Trace), - [#{result := {async_return, {error, Error}}} | _] = Trace, - case Error of - {resource_error, _} -> + [#{result := Result} | _] = Trace, + case Result of + {async_return, {error, {resource_error, _}}} -> ok; - {recoverable_error, disconnected} -> + {async_return, {error, {recoverable_error, disconnected}}} -> + ok; + {error, {resource_error, _}} -> ok; _ -> - ct:fail("unexpected error: ~p", [Error]) + ct:fail("unexpected error: ~p", [Result]) end end ), From 89cd6cfede44b08b6e28d26bdbb6ff6f2001e746 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 17 Apr 2023 17:01:39 -0300 Subject: [PATCH 229/279] fix(plugins): attempt to extract plugin from current node on startup Fixes https://emqx.atlassian.net/browse/EMQX-9605 Fixes https://github.com/emqx/emqx-elixir-plugin/issues/25 If an user happens to configure a plugin in a lone-node cluster via environment variables, it would fail to start up as there are no other nodes to copy the plugin from. Here, we attempt to check if the package is present in the current node but not yet extracted. --- apps/emqx/test/emqx_common_test_helpers.erl | 1 + apps/emqx_plugins/src/emqx_plugins.app.src | 2 +- apps/emqx_plugins/src/emqx_plugins.erl | 37 ++++++--- apps/emqx_plugins/test/emqx_plugins_SUITE.erl | 77 ++++++++++++++++++- changes/ce/fix-10422.en.md | 1 + 5 files changed, 106 insertions(+), 12 deletions(-) create mode 100644 changes/ce/fix-10422.en.md diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index e9ddc61a8..8603be879 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -764,6 +764,7 @@ setup_node(Node, Opts) when is_map(Opts) -> load_apps => LoadApps, apps => Apps, env => Env, + join_to => JoinTo, start_apps => StartApps } ] diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index c0372c003..d5c16ea59 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {modules, []}, {mod, {emqx_plugins_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 264247086..04faa44e9 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -479,22 +479,39 @@ ensure_exists_and_installed(NameVsn) -> case filelib:is_dir(dir(NameVsn)) of true -> ok; - _ -> - Nodes = [N || N <- mria:running_nodes(), N /= node()], - case get_from_any_node(Nodes, NameVsn, []) of + false -> + %% Do we have the package, but it's not extracted yet? + case get_tar(NameVsn) of {ok, TarContent} -> ok = file:write_file(pkg_file(NameVsn), TarContent), ok = do_ensure_installed(NameVsn); - {error, NodeErrors} -> - ?SLOG(error, #{ - msg => "failed_to_copy_plugin_from_other_nodes", - name_vsn => NameVsn, - node_errors => NodeErrors - }), - {error, plugin_not_found} + _ -> + %% If not, try to get it from the cluster. + do_get_from_cluster(NameVsn) end end. +do_get_from_cluster(NameVsn) -> + Nodes = [N || N <- mria:running_nodes(), N /= node()], + case get_from_any_node(Nodes, NameVsn, []) of + {ok, TarContent} -> + ok = file:write_file(pkg_file(NameVsn), TarContent), + ok = do_ensure_installed(NameVsn); + {error, NodeErrors} when Nodes =/= [] -> + ?SLOG(error, #{ + msg => "failed_to_copy_plugin_from_other_nodes", + name_vsn => NameVsn, + node_errors => NodeErrors + }), + {error, plugin_not_found}; + {error, _} -> + ?SLOG(error, #{ + msg => "no_nodes_to_copy_plugin_from", + name_vsn => NameVsn + }), + {error, plugin_not_found} + end. + get_from_any_node([], _NameVsn, Errors) -> {error, Errors}; get_from_any_node([Node | T], NameVsn, Errors) -> diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index 260ad1681..14d6d06fc 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -45,7 +45,10 @@ all() -> groups() -> [ - {copy_plugin, [sequence], [group_t_copy_plugin_to_a_new_node]}, + {copy_plugin, [sequence], [ + group_t_copy_plugin_to_a_new_node, + group_t_copy_plugin_to_a_new_node_single_node + ]}, {create_tar_copy_plugin, [sequence], [group_t_copy_plugin_to_a_new_node]} ]. @@ -601,6 +604,78 @@ group_t_copy_plugin_to_a_new_node(Config) -> rpc:call(CopyToNode, emqx_plugins, describe, [NameVsn]) ). +%% checks that we can start a cluster with a lone node. +group_t_copy_plugin_to_a_new_node_single_node({init, Config}) -> + PrivDataDir = ?config(priv_dir, Config), + ToInstallDir = filename:join(PrivDataDir, "plugins_copy_to"), + file:del_dir_r(ToInstallDir), + ok = filelib:ensure_path(ToInstallDir), + #{package := Package, release_name := PluginName} = get_demo_plugin_package(ToInstallDir), + NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX), + [{CopyTo, CopyToOpts}] = + emqx_common_test_helpers:emqx_cluster( + [ + {core, plugins_copy_to} + ], + #{ + apps => [emqx_conf, emqx_plugins], + env => [ + {emqx, init_config_load_done, false}, + {emqx, boot_modules, []} + ], + env_handler => fun + (emqx_plugins) -> + ok = emqx_plugins:put_config(install_dir, ToInstallDir), + %% this is to simulate an user setting the state + %% via environment variables before starting the node + ok = emqx_plugins:put_config( + states, + [#{name_vsn => NameVsn, enable => true}] + ), + ok; + (_) -> + ok + end, + priv_data_dir => PrivDataDir, + schema_mod => emqx_conf_schema, + peer_mod => slave, + load_schema => true + } + ), + [ + {to_install_dir, ToInstallDir}, + {copy_to_node_name, CopyTo}, + {copy_to_opts, CopyToOpts}, + {name_vsn, NameVsn}, + {plugin_name, PluginName} + | Config + ]; +group_t_copy_plugin_to_a_new_node_single_node({'end', Config}) -> + CopyToNode = proplists:get_value(copy_to_node, Config), + ok = emqx_common_test_helpers:stop_slave(CopyToNode), + ok = file:del_dir_r(proplists:get_value(to_install_dir, Config)), + ok; +group_t_copy_plugin_to_a_new_node_single_node(Config) -> + CopyTo = ?config(copy_to_node_name, Config), + CopyToOpts = ?config(copy_to_opts, Config), + ToInstallDir = ?config(to_install_dir, Config), + NameVsn = proplists:get_value(name_vsn, Config), + %% Start the node for the first time. The plugin should start + %% successfully even if it's not extracted yet. Simply starting + %% the node would crash if not working properly. + CopyToNode = emqx_common_test_helpers:start_slave(CopyTo, CopyToOpts), + ct:pal("~p config:\n ~p", [ + CopyToNode, erpc:call(CopyToNode, emqx_plugins, get_config, [[], #{}]) + ]), + ct:pal("~p install_dir:\n ~p", [ + CopyToNode, erpc:call(CopyToNode, file, list_dir, [ToInstallDir]) + ]), + ?assertMatch( + {ok, #{running_status := running, config_status := enabled}}, + rpc:call(CopyToNode, emqx_plugins, describe, [NameVsn]) + ), + ok. + make_tar(Cwd, NameWithVsn) -> make_tar(Cwd, NameWithVsn, NameWithVsn). diff --git a/changes/ce/fix-10422.en.md b/changes/ce/fix-10422.en.md new file mode 100644 index 000000000..7c18ccf32 --- /dev/null +++ b/changes/ce/fix-10422.en.md @@ -0,0 +1 @@ +Fixed a bug where external plugins could not be configured via environment variables in a lone-node cluster. From acea64790be45101ec8c70dbcc583721d914023b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 18 Apr 2023 17:00:37 +0800 Subject: [PATCH 230/279] chore: add README for emqx_modoules --- apps/emqx_modules/README.md | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 apps/emqx_modules/README.md diff --git a/apps/emqx_modules/README.md b/apps/emqx_modules/README.md new file mode 100644 index 000000000..7cb1db0c5 --- /dev/null +++ b/apps/emqx_modules/README.md @@ -0,0 +1,54 @@ +# EMQX Modules + +The application provide some minor functional modules that are not included in the MQTT +protocol standard, including "Delayed Publish", "Topic Rewrite", "Topic Metrics" and "Telemetry". + + +## Delayed Publish + +After enabling this module, messages sent by the user with the prefix +`$delayed/{Interval}/{Topic}` will be delayed by `{Interval}` seconds before +being published to the `{Topic}`. + +More introduction about [Delayed Publish](https://www.emqx.io/docs/en/v5.0/mqtt/mqtt-delayed-publish.html). + +See [Enabling/Disabling Delayed Publish via HTTP API](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1delayed/put). + + +## Topic Rewrite + +Topic Rewrite allows users to configure rules to change the topic strings that +the client requests to subscribe or publish. + +This feature is very useful when designing topics that are compatible with different +client versions. For example, an old device that has already been issued and cannot +be upgraded may use old topic rules, but the production environment need to apply +a new design rules for the topics. + +More introduction about [Topic Rewrite](https://www.emqx.io/docs/en/v5.0/mqtt/mqtt-topic-rewrite.html). + +See [List all rewrite rules](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_rewrite/get) +and [Create or Update rewrite rules](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_rewrite/put). + + +## Topic Metrics + +Topic Metrics is used for users to specify monitoring of certain topics and to +count the number of messages, QoS distribution, and rate for all messages on that topic. + +More introduction about [Topic Metrics](https://www.emqx.io/docs/en/v5.0/dashboard/diagnose.html#topic-metrics). + +See HTTP API docs to [List all monitored topics](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_metrics/get), +[Create topic metrics](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_metrics/post) +and [Get the monitored result](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/MQTT/paths/~1mqtt~1topic_metrics~1%7Btopic%7D/get). + + +## Telemetry + +Telemetry is used for collecting non-sensitive information about the EMQX cluster. + +More introduction about [Telemetry](https://www.emqx.io/docs/en/v5.0/telemetry/telemetry.html#telemetry). + +See HTTP API docs to [Enable/Disable telemetry](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/put), +[Get the enabled status](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1status/get) +and [Get the data of the module collected](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Telemetry/paths/~1telemetry~1data/get). From 7de6c5b3921a2c6d4ead2a7557760af48e5a2a24 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 18 Apr 2023 17:38:36 +0800 Subject: [PATCH 231/279] chore: update prometheus README --- apps/emqx_prometheus/README.md | 283 ++------------------------------- 1 file changed, 10 insertions(+), 273 deletions(-) diff --git a/apps/emqx_prometheus/README.md b/apps/emqx_prometheus/README.md index ddff9774c..c008adc81 100644 --- a/apps/emqx_prometheus/README.md +++ b/apps/emqx_prometheus/README.md @@ -1,279 +1,16 @@ -# emqx-prometheus +# EMQX Prometheus Agent -EMQX Prometheus Agent - -## push emqx stats/metrics to prometheus PushGateway - -``` -prometheus.push.gateway.server = http://127.0.0.1:9091 - -prometheus.interval = 15000 -``` - -## pull emqx stats/metrics - -``` -Method: GET -Path: api/v4/emqx_prometheus?type=prometheus -params: type: [prometheus| json] - -prometheus data - -# TYPE erlang_vm_ets_limit gauge -erlang_vm_ets_limit 256000 -# TYPE erlang_vm_logical_processors gauge -erlang_vm_logical_processors 4 -# TYPE erlang_vm_logical_processors_available gauge -erlang_vm_logical_processors_available NaN -# TYPE erlang_vm_logical_processors_online gauge -erlang_vm_logical_processors_online 4 -# TYPE erlang_vm_port_count gauge -erlang_vm_port_count 17 -# TYPE erlang_vm_port_limit gauge -erlang_vm_port_limit 1048576 +This application provides the ability to integrate with Prometheus. It provides +an HTTP API for collecting metrics of the current node +and also supports configuring a Push Gateway URL address for pushing these metrics. -json data +More introduction about [Integrate with Prometheus](https://www.emqx.io/docs/en/v5.0/observability/prometheus.html#integrate-with-prometheus) -{ - "stats": {key:value}, - "metrics": {key:value}, - "packets": {key:value}, - "messages": {key:value}, - "delivery": {key:value}, - "client": {key:value}, - "session": {key:value} -} - -``` +See HTTP API docs to learn how to +[Update Prometheus config](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Monitor/paths/~1prometheus/put) +and [Get all metrics data](https://www.emqx.io/docs/en/v5.0/admin/api-docs.html#tag/Monitor/paths/~1prometheus~1stats/get). -## Before EMQX v4.0.0 -The prometheus data simple is: - - -```bash -# TYPE erlang_vm_ets_limit gauge -erlang_vm_ets_limit 2097152 -# TYPE erlang_vm_logical_processors gauge -erlang_vm_logical_processors 2 -# TYPE erlang_vm_logical_processors_available gauge -erlang_vm_logical_processors_available 2 -# TYPE erlang_vm_logical_processors_online gauge -erlang_vm_logical_processors_online 2 -# TYPE erlang_vm_port_count gauge -erlang_vm_port_count 19 -# TYPE erlang_vm_port_limit gauge -erlang_vm_port_limit 1048576 -# TYPE erlang_vm_process_count gauge -erlang_vm_process_count 460 -# TYPE erlang_vm_process_limit gauge -erlang_vm_process_limit 2097152 -# TYPE erlang_vm_schedulers gauge -erlang_vm_schedulers 2 -# TYPE erlang_vm_schedulers_online gauge -erlang_vm_schedulers_online 2 -# TYPE erlang_vm_smp_support untyped -erlang_vm_smp_support 1 -# TYPE erlang_vm_threads untyped -erlang_vm_threads 1 -# TYPE erlang_vm_thread_pool_size gauge -erlang_vm_thread_pool_size 32 -# TYPE erlang_vm_time_correction untyped -erlang_vm_time_correction 1 -# TYPE erlang_vm_statistics_context_switches counter -erlang_vm_statistics_context_switches 39850 -# TYPE erlang_vm_statistics_garbage_collection_number_of_gcs counter -erlang_vm_statistics_garbage_collection_number_of_gcs 17116 -# TYPE erlang_vm_statistics_garbage_collection_words_reclaimed counter -erlang_vm_statistics_garbage_collection_words_reclaimed 55711819 -# TYPE erlang_vm_statistics_garbage_collection_bytes_reclaimed counter -erlang_vm_statistics_garbage_collection_bytes_reclaimed 445694552 -# TYPE erlang_vm_statistics_bytes_received_total counter -erlang_vm_statistics_bytes_received_total 400746 -# TYPE erlang_vm_statistics_bytes_output_total counter -erlang_vm_statistics_bytes_output_total 337197 -# TYPE erlang_vm_statistics_reductions_total counter -erlang_vm_statistics_reductions_total 21157980 -# TYPE erlang_vm_statistics_run_queues_length_total gauge -erlang_vm_statistics_run_queues_length_total 0 -# TYPE erlang_vm_statistics_runtime_milliseconds counter -erlang_vm_statistics_runtime_milliseconds 6559 -# TYPE erlang_vm_statistics_wallclock_time_milliseconds counter -erlang_vm_statistics_wallclock_time_milliseconds 261243 -# TYPE erlang_vm_memory_atom_bytes_total gauge -erlang_vm_memory_atom_bytes_total{usage="used"} 1814822 -erlang_vm_memory_atom_bytes_total{usage="free"} 22459 -# TYPE erlang_vm_memory_bytes_total gauge -erlang_vm_memory_bytes_total{kind="system"} 109820104 -erlang_vm_memory_bytes_total{kind="processes"} 44983656 -# TYPE erlang_vm_dets_tables gauge -erlang_vm_dets_tables 1 -# TYPE erlang_vm_ets_tables gauge -erlang_vm_ets_tables 139 -# TYPE erlang_vm_memory_processes_bytes_total gauge -erlang_vm_memory_processes_bytes_total{usage="used"} 44983656 -erlang_vm_memory_processes_bytes_total{usage="free"} 0 -# TYPE erlang_vm_memory_system_bytes_total gauge -erlang_vm_memory_system_bytes_total{usage="atom"} 1837281 -erlang_vm_memory_system_bytes_total{usage="binary"} 595872 -erlang_vm_memory_system_bytes_total{usage="code"} 40790577 -erlang_vm_memory_system_bytes_total{usage="ets"} 37426896 -erlang_vm_memory_system_bytes_total{usage="other"} 29169478 -# TYPE erlang_mnesia_held_locks gauge -erlang_mnesia_held_locks 0 -# TYPE erlang_mnesia_lock_queue gauge -erlang_mnesia_lock_queue 0 -# TYPE erlang_mnesia_transaction_participants gauge -erlang_mnesia_transaction_participants 0 -# TYPE erlang_mnesia_transaction_coordinators gauge -erlang_mnesia_transaction_coordinators 0 -# TYPE erlang_mnesia_failed_transactions counter -erlang_mnesia_failed_transactions 2 -# TYPE erlang_mnesia_committed_transactions counter -erlang_mnesia_committed_transactions 239 -# TYPE erlang_mnesia_logged_transactions counter -erlang_mnesia_logged_transactions 60 -# TYPE erlang_mnesia_restarted_transactions counter -erlang_mnesia_restarted_transactions 0 -# TYPE emqx_packets_auth_received counter -emqx_packets_auth_received 0 -# TYPE emqx_packets_auth_sent counter -emqx_packets_auth_sent 0 -# TYPE emqx_packets_received counter -emqx_packets_received 0 -# TYPE emqx_packets_sent counter -emqx_packets_sent 0 -# TYPE emqx_packets_connect counter -emqx_packets_connect 0 -# TYPE emqx_packets_connack_sent counter -emqx_packets_connack_sent 0 -# TYPE emqx_packets_connack_error counter -emqx_packets_connack_error 0 -# TYPE emqx_packets_connack_auth_error counter -emqx_packets_connack_auth_error 0 -# TYPE emqx_packets_disconnect_received counter -emqx_packets_disconnect_received 0 -# TYPE emqx_packets_disconnect_sent counter -emqx_packets_disconnect_sent 0 -# TYPE emqx_packets_subscribe counter -emqx_packets_subscribe 0 -# TYPE emqx_packets_subscribe_error counter -emqx_packets_subscribe_error 0 -# TYPE emqx_packets_subscribe_auth_error counter -emqx_packets_subscribe_auth_error 0 -# TYPE emqx_packets_suback counter -emqx_packets_suback 0 -# TYPE emqx_packets_unsubscribe counter -emqx_packets_unsubscribe 0 -# TYPE emqx_packets_unsubscribe_error counter -emqx_packets_unsubscribe_error 0 -# TYPE emqx_packets_unsuback counter -emqx_packets_unsuback 0 -# TYPE emqx_packets_publish_received counter -emqx_packets_publish_received 0 -# TYPE emqx_packets_publish_sent counter -emqx_packets_publish_sent 0 -# TYPE emqx_packets_publish_auth_error counter -emqx_packets_publish_auth_error 0 -# TYPE emqx_packets_publish_error counter -emqx_packets_publish_error 0 -# TYPE emqx_packets_puback_received counter -emqx_packets_puback_received 0 -# TYPE emqx_packets_puback_sent counter -emqx_packets_puback_sent 0 -# TYPE emqx_packets_puback_missed counter -emqx_packets_puback_missed 0 -# TYPE emqx_packets_pubrec_received counter -emqx_packets_pubrec_received 0 -# TYPE emqx_packets_pubrec_sent counter -emqx_packets_pubrec_sent 0 -# TYPE emqx_packets_pubrec_missed counter -emqx_packets_pubrec_missed 0 -# TYPE emqx_packets_pubrel_received counter -emqx_packets_pubrel_received 0 -# TYPE emqx_packets_pubrel_sent counter -emqx_packets_pubrel_sent 0 -# TYPE emqx_packets_pubrel_missed counter -emqx_packets_pubrel_missed 0 -# TYPE emqx_packets_pubcomp_received counter -emqx_packets_pubcomp_received 0 -# TYPE emqx_packets_pubcomp_sent counter -emqx_packets_pubcomp_sent 0 -# TYPE emqx_packets_pubcomp_missed counter -emqx_packets_pubcomp_missed 0 -# TYPE emqx_packets_pingreq counter -emqx_packets_pingreq 0 -# TYPE emqx_packets_pingresp counter -emqx_packets_pingresp 0 -# TYPE emqx_bytes_received counter -emqx_bytes_received 0 -# TYPE emqx_bytes_sent counter -emqx_bytes_sent 0 -# TYPE emqx_connections_count gauge -emqx_connections_count 0 -# TYPE emqx_connections_max gauge -emqx_connections_max 0 -# TYPE emqx_retained_count gauge -emqx_retained_count 3 -# TYPE emqx_retained_max gauge -emqx_retained_max 3 -# TYPE emqx_sessions_count gauge -emqx_sessions_count 0 -# TYPE emqx_sessions_max gauge -emqx_sessions_max 0 -# TYPE emqx_subscriptions_count gauge -emqx_subscriptions_count 0 -# TYPE emqx_subscriptions_max gauge -emqx_subscriptions_max 0 -# TYPE emqx_topics_count gauge -emqx_topics_count 0 -# TYPE emqx_topics_max gauge -emqx_topics_max 0 -# TYPE emqx_vm_cpu_use gauge -emqx_vm_cpu_use 100.0 -# TYPE emqx_vm_cpu_idle gauge -emqx_vm_cpu_idle 0.0 -# TYPE emqx_vm_run_queue gauge -emqx_vm_run_queue 1 -# TYPE emqx_vm_process_messages_in_queues gauge -emqx_vm_process_messages_in_queues 0 -# TYPE emqx_messages_received counter -emqx_messages_received 0 -# TYPE emqx_messages_sent counter -emqx_messages_sent 0 -# TYPE emqx_messages_dropped counter -emqx_messages_dropped 0 -# TYPE emqx_messages_retained counter -emqx_messages_retained 3 -# TYPE emqx_messages_qos0_received counter -emqx_messages_qos0_received 0 -# TYPE emqx_messages_qos0_sent counter -emqx_messages_qos0_sent 0 -# TYPE emqx_messages_qos1_received counter -emqx_messages_qos1_received 0 -# TYPE emqx_messages_qos1_sent counter -emqx_messages_qos1_sent 0 -# TYPE emqx_messages_qos2_received counter -emqx_messages_qos2_received 0 -# TYPE emqx_messages_qos2_expired counter -emqx_messages_qos2_expired 0 -# TYPE emqx_messages_qos2_sent counter -emqx_messages_qos2_sent 0 -# TYPE emqx_messages_qos2_dropped counter -emqx_messages_qos2_dropped 0 -# TYPE emqx_messages_forward counter -emqx_messages_forward 0 -``` - - -License -------- - -Apache License Version 2.0 - -Author ------- - -EMQX Team. - +Correspondingly, we have also provided a [Grafana template](https://grafana.com/grafana/dashboards/17446-emqx/) +for visualizing these metrics. From c858fba9d47eb41268d534fb88025b7004a4ab3a Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 18 Apr 2023 17:34:28 +0200 Subject: [PATCH 232/279] chore: v5.0.23 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index ea79dcd0e..ec3883c77 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.22"). +-define(EMQX_RELEASE_CE, "5.0.23"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.3-alpha.1"). diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index ecc211e35..312a9dfbe 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.22 +version: 5.0.23 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.22 +appVersion: 5.0.23 From f8ff2d6bc97bb81d1751a2e3a86c7d77cb869cc8 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Tue, 18 Apr 2023 18:09:24 +0200 Subject: [PATCH 233/279] docs: Generate changelog for v5.0.23 --- changes/v5.0.23.en.md | 62 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 changes/v5.0.23.en.md diff --git a/changes/v5.0.23.en.md b/changes/v5.0.23.en.md new file mode 100644 index 000000000..6d016d2da --- /dev/null +++ b/changes/v5.0.23.en.md @@ -0,0 +1,62 @@ +# v5.0.23 + +## Enhancements + +- [#10156](https://github.com/emqx/emqx/pull/10156) Change the priority of the configuration: + 1. If it is a new installation of EMQX, the priority of configuration is `ENV > emqx.conf > HTTP API`. + 2. If EMQX is upgraded from an old version (i.e., the cluster-override.conf file still exists in EMQX's data directory), then the configuration priority remains the same as before. That is, `HTTP API > ENV > emqx.conf`. + + Deprecated data/configs/local-override.conf. + + Stabilizing the HTTP API for hot updates. + +- [#10354](https://github.com/emqx/emqx/pull/10354) More specific error messages when configure with bad max_heap_size value. + Log current value and the max value when the `message_queue_too_long` error is thrown. + +- [#10359](https://github.com/emqx/emqx/pull/10359) Metrics now are not implicitly collected in places where API handlers don't make any use of them. Instead, a separate backplane RPC gathers cluster-wide metrics. + +- [#10373](https://github.com/emqx/emqx/pull/10373) Deprecate the trace.payload_encode configuration. + Add payload_encode=[text,hidden,hex] option when creating a trace via HTTP API. + +- [#10389](https://github.com/emqx/emqx/pull/10389) Unify the config formats for `cluster.core_nodes` and `cluster.statics.seeds`. + Now they both support formats in array `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` or semicolon-separated string `"emqx1@127.0.0.1,emqx2@127.0.0.1"`. + +- [#10391](https://github.com/emqx/emqx/pull/10391) Hide a large number of advanced options to simplify the configuration file. + + That includes `rewrite`, `topic_metric`, `persistent_session_store`, `overload_protection`, + `flapping_detect`, `conn_congestion`, `stats,auto_subscribe`, `broker_perf`, + `shared_subscription_group`, `slow_subs`, `ssl_options.user_lookup_fun` and some advance items + in `node` and `dashboard` section, [#10358](https://github.com/emqx/emqx/pull/10358), + [#10381](https://github.com/emqx/emqx/pull/10381), [#10385](https://github.com/emqx/emqx/pull/10385). + +- [#10392](https://github.com/emqx/emqx/pull/10392) A new function to convert a formatted date to an integer timestamp has been added: date_to_unix_ts/3 + +- [#10404](https://github.com/emqx/emqx/pull/10404) Change the default queue mode for buffer workers to `memory_only`. + Before this change, the default queue mode was `volatile_offload`. When under high message rate pressure and when the resource is not keeping up with such rate, the buffer performance degraded a lot due to the constant disk operations. + +- [#10426](https://github.com/emqx/emqx/pull/10426) Optimize the configuration priority mechanism to fix the issue where the configuration + changes made to `etc/emqx.conf` do not take effect after restarting EMQX. + + More introduction about the new mechanism: [Configure Override Rules](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html#configure-override-rules) + +- [#10376](https://github.com/emqx/emqx/pull/10376) Simplify the configuration of the limiter feature and optimize some codes + - Rename `message_in` to `messages` + - Rename `bytes_in` to `bytes` + - Use `burst` instead of `capacity` + - Hide non-importance fields + - Optimize limiter instances in different rate settings + +- [#10430](https://github.com/emqx/emqx/pull/10430) Simplify the configuration of the `retainer` feature. + - Mark `flow_control` as non-importance field. + +## Bug Fixes + +- [#10369](https://github.com/emqx/emqx/pull/10369) Fix error in `/api/v5/monitor_current` API endpoint that happens when some EMQX nodes are down. + + Prior to this fix, sometimes the request returned HTTP code 500 and the following message: + ``` + {"code":"INTERNAL_ERROR","message":"error, badarg, [{erlang,'++',[{error,nodedown},[{node,'emqx@10.42.0.150'}]], ... + ``` + +- [#10410](https://github.com/emqx/emqx/pull/10410) Fix config check failed when gateways are configured in emqx.conf. + This issue was first introduced in v5.0.22 via [#10278](https://github.com/emqx/emqx/pull/10278), the boot-time config check was missing. From 9a4af6bd7663e667da0bec3254b1e53e7c8dcc54 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:23:21 +0200 Subject: [PATCH 234/279] docs: update config docs --- rel/emqx_conf.template.en.md | 19 ++++++++++++------- rel/emqx_conf.template.zh.md | 16 ++++++++++------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index bded20da6..c1259869c 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -7,22 +7,27 @@ and a superset of JSON. EMQX configuration consists of two layers. From bottom up: -1. Cluster configs: `$EMQX_NODE__DATA_DIR/configs/cluster.hocon` -2. `emqx.conf` + `EMQX_` prefixed environment variables.
- Changes in this layer require a full node restart to take effect. +1. Cluster-synced configs: `$EMQX_NODE__DATA_DIR/configs/cluster.hocon`. +2. Local node configs: `emqx.conf` + `EMQX_` prefixed environment variables. +:::tip Tip +Prior to v5.0.23 and e5.0.3, the cluster-synced configs are stored in +`cluster-override.conf` which is applied on top of the local configs. + +If upgraded from an earlier version, as long as `cluster-override.conf` exists, +`cluster.hocon` will not be created, and `cluster-override.conf` will stay on +top of the overriding layers. +::: When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir` is used. The `cluster.hocon` file is overwritten at runtime when changes -are made from dashboard UI, management HTTP API, or CLI. When clustered, +are made from Dashboard, management HTTP API, or CLI. When clustered, after EMQX restarts, it copies the file from the node which has the greatest `uptime`. :::tip Tip -Some of the configs (such as `node.name`) are boot-only configs and not overridable. -Config values from `cluster.hocon` are **not** mapped to boot configs for -the config fields attributed with `mapping: path.to.boot.config.key` +To avoid confusion, don't add the same keys in both `cluster.hocon` and `emqx.conf`. ::: For detailed override rules, see [Config Overlay Rules](#config-overlay-rules). diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index 916eed38d..a9df27f63 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -5,9 +5,15 @@ HOCON（Human-Optimized Config Object Notation）是一个JSON的超集，非常 EMQX的配置文件可分为二层，自底向上依次是： -1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。
- 修改这一层的配置之后，需要重启节点来使之生效。 -2. 集群范围重载层：`$EMQX_NODE__DATA_DIR/configs/cluster.hocon` +1. 集群同步配置：`$EMQX_NODE__DATA_DIR/configs/cluster.hocon`。 +2. 本地节点配置：`emqx.conf` 加上 `EMQX_` 前缀的环境变量。 + +:::tip Tip +在 v5.0.23 或 e5.0.3 之前，集群同步配置保存在文件 `cluster-override.conf` 中，并且它覆盖在配置的最上层。 + +如果从之前的版本升级上来，只要 `cluster-override.conf` 文件存在， +EMQX 就不会创建 `cluster.hocon`，并且 `cluster-override.conf` 会继续覆盖在配置的最上层。 +::: 如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置，那么该目录会从 `emqx.conf` 的 `node.data_dir` 配置中读取。 @@ -16,9 +22,7 @@ EMQX的配置文件可分为二层，自底向上依次是： 当EMQX运行在集群中时，一个EMQX节点重启之后，会从集群中其他节点复制该文件内容到本地。 :::tip Tip -有些配置项是不能被重载的（例如 `node.name`）。 -配置项如果有 `mapping: path.to.boot.config.key` 这个属性， -则不能被添加到重载文件 `cluster.hocon` 中。 +为避免歧义，应尽量避免让 `cluster.hocon` 和 `emqx.conf` 出现配置交集。 ::: 更多的重载规则，请参考下文 [配置重载规则](#配置重载规则)。 From a30e08d06a4e1217c103913d08f334e548b6edd7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:36:43 +0200 Subject: [PATCH 235/279] refactor: set authn config at 'low' prio instead of 'hidden' --- apps/emqx/src/emqx_schema.erl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8335d69b8..6cece7568 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2762,10 +2762,16 @@ str(S) when is_list(S) -> S. authentication(Which) -> - Desc = + {Importance, Desc} = case Which of - global -> ?DESC(global_authentication); - listener -> ?DESC(listener_authentication) + global -> + %% For root level authentication, it is recommended to configure + %% from the dashboard or API. + %% Hence it's considered a low-importance when it comes to + %% configuration importance. + {?IMPORTANCE_LOW, ?DESC(global_authentication)}; + listener -> + {?IMPORTANCE_HIDDEN, ?DESC(listener_authentication)} end, %% poor man's dependency injection %% this is due to the fact that authn is implemented outside of 'emqx' app. @@ -2781,7 +2787,7 @@ authentication(Which) -> hoconsc:mk(Type, #{ desc => Desc, converter => fun ensure_array/2, - importance => ?IMPORTANCE_HIDDEN + importance => Importance }). %% the older version schema allows individual element (instead of a chain) in config From 6bc33e86be0520b641cc0d3d26c4058dd483c9d7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:37:36 +0200 Subject: [PATCH 236/279] refactor: set rpc config at 'low' importance level instead of 'hidden' --- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 362fc1587..7aeec68f0 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -100,7 +100,7 @@ roots() -> ?R_REF("rpc"), #{ translate_to => ["gen_rpc"], - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_LOW } )} ] ++ From 4d67312baba61c2603f0b5237843247c56df16fa Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:40:32 +0200 Subject: [PATCH 237/279] refactor: set authz config at 'high' importance level and authorization.sources at 'low' level prior to this commit, the root was set to 'hidden' which is not ideal because some may still want to configure the sources from files --- apps/emqx_authz/src/emqx_authz_schema.erl | 5 ++++- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index f03ae52a8..39bbcc360 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -494,7 +494,10 @@ authz_fields() -> default => [], desc => ?DESC(sources), %% doc_lift is force a root level reference instead of nesting sub-structs - extra => #{doc_lift => true} + extra => #{doc_lift => true}, + %% it is recommended to configure authz sources from dashboard + %% hance the importance level for config is low + importance => ?IMPORTANCE_LOW } )} ]. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 7aeec68f0..8e5e6937f 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1288,7 +1288,7 @@ emqx_schema_high_prio_roots() -> ?R_REF("authorization"), #{ desc => ?DESC(authorization), - importance => ?IMPORTANCE_HIDDEN + importance => ?IMPORTANCE_HIGH } )}, lists:keyreplace("authorization", 1, Roots, Authz). From d51cc750dec275460e26cc7ec3a06eb66be05858 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:44:31 +0200 Subject: [PATCH 238/279] refactor: hide psk user_lookup_fun --- apps/emqx/src/emqx_schema.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 6cece7568..f264b3e8a 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2235,6 +2235,7 @@ common_ssl_opts_schema(Defaults) -> #{ default => <<"emqx_tls_psk:lookup">>, converter => fun ?MODULE:user_lookup_fun_tr/2, + importance => ?IMPORTANCE_HIDDEN, desc => ?DESC(common_ssl_opts_schema_user_lookup_fun) } )}, From 30fd9b10f78db09afe9c0f41a631ddec7a7ef5e0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:45:37 +0200 Subject: [PATCH 239/279] refactor: change bridges config importance level from hidden to low --- apps/emqx_bridge/src/schema/emqx_bridge_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index e5def2d64..4b9b7e3fe 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -137,7 +137,7 @@ namespace() -> "bridge". tags() -> [<<"Bridge">>]. -roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_HIDDEN})}]. +roots() -> [{bridges, ?HOCON(?R_REF(bridges), #{importance => ?IMPORTANCE_LOW})}]. fields(bridges) -> [ From be70c7d3850e8a85de8d056f888e7ddbf5116d4b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:47:16 +0200 Subject: [PATCH 240/279] refactor: change rule_engine config importance level from hidden to low --- apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 242c86c71..bc8cae07a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -38,7 +38,7 @@ namespace() -> rule_engine. tags() -> [<<"Rule Engine">>]. -roots() -> [{"rule_engine", ?HOCON(?R_REF("rule_engine"), #{importance => ?IMPORTANCE_HIDDEN})}]. +roots() -> [{"rule_engine", ?HOCON(?R_REF("rule_engine"), #{importance => ?IMPORTANCE_LOW})}]. fields("rule_engine") -> rule_engine_settings() ++ From 12e549a3a20818af2dd750fb42bbe8a6f17638c4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:48:14 +0200 Subject: [PATCH 241/279] refactor: change exhook config importance level from hidden to low --- apps/emqx_exhook/src/emqx_exhook_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 708e164fc..f6cc896f3 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -32,7 +32,7 @@ namespace() -> exhook. roots() -> - [{exhook, ?HOCON(?R_REF(exhook), #{importance => ?IMPORTANCE_HIDDEN})}]. + [{exhook, ?HOCON(?R_REF(exhook), #{importance => ?IMPORTANCE_LOW})}]. fields(exhook) -> [ From f361870ca7cd5165f644970aed671e781fbfa961 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:53:49 +0200 Subject: [PATCH 242/279] refactor: hide sysmon.top config --- apps/emqx/src/emqx_schema.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index f264b3e8a..b3c5e1778 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1633,7 +1633,9 @@ fields("sysmon") -> {"top", sc( ref("sysmon_top"), - #{} + %% Userful monitoring solution when benchmarking, + %% but hardly common enough for regular users. + #{importance => ?IMPORTANCE_HIDDEN} )} ]; fields("sysmon_vm") -> From 3e72d6c4a35d14fda9b6e260350e62a86f05f099 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 19:03:27 +0200 Subject: [PATCH 243/279] docs: sync changelogs --- changes/ce/feat-10391.en.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/changes/ce/feat-10391.en.md b/changes/ce/feat-10391.en.md index a64b01221..f33757404 100644 --- a/changes/ce/feat-10391.en.md +++ b/changes/ce/feat-10391.en.md @@ -1 +1,7 @@ -hide exhook/rewrite/topic_metric/persistent_session_store/overload_protection from the docs and configuration file. +Hide a large number of advanced options to simplify the configuration file. + +That includes `rewrite`, `topic_metric`, `persistent_session_store`, `overload_protection`, +`flapping_detect`, `conn_congestion`, `stats,auto_subscribe`, `broker_perf`, +`shared_subscription_group`, `slow_subs`, `ssl_options.user_lookup_fun` and some advance items +in `node` and `dashboard` section, [#10358](https://github.com/emqx/emqx/pull/10358), +[#10381](https://github.com/emqx/emqx/pull/10381), [#10385](https://github.com/emqx/emqx/pull/10385). From 6c352fd525ff6b45b65d433e13cd8bf1bc7d4c8f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 18 Apr 2023 18:23:21 +0200 Subject: [PATCH 244/279] docs: update config docs --- rel/emqx_conf.template.en.md | 22 ++++++++++++++-------- rel/emqx_conf.template.zh.md | 16 ++++++++++------ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index 8740e4319..bf5120046 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -7,21 +7,27 @@ and a superset of JSON. EMQX configuration consists of two layers. From bottom up: -1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.
- Changes in this layer require a full node restart to take effect. -1. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` +1. Cluster-synced configs: `$EMQX_NODE__DATA_DIR/configs/cluster.hocon`. +2. Local node configs: `emqx.conf` + `EMQX_` prefixed environment variables. + +:::tip Tip +Prior to v5.0.23 and e5.0.3, the cluster-synced configs are stored in +`cluster-override.conf` which is applied on top of the local configs. + +If upgraded from an earlier version, as long as `cluster-override.conf` exists, +`cluster.hocon` will not be created, and `cluster-override.conf` will stay on +top of the overriding layers. +::: When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir` is used. -The `cluster-override.conf` file is overwritten at runtime when changes -are made from dashboard UI, management HTTP API, or CLI. When clustered, +The `cluster.hocon` file is overwritten at runtime when changes +are made from Dashboard, management HTTP API, or CLI. When clustered, after EMQX restarts, it copies the file from the node which has the greatest `uptime`. :::tip Tip -Some of the configs (such as `node.name`) are boot-only configs and not overridable. -Config values from `*-override.conf` are **not** mapped to boot configs for -the config fields attributed with `mapping: path.to.boot.config.key` +To avoid confusion, don't add the same keys in both `cluster.hocon` and `emqx.conf`. ::: For detailed override rules, see [Config Overlay Rules](#config-overlay-rules). diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index 9402760a2..a96c9ee48 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -5,9 +5,15 @@ HOCON（Human-Optimized Config Object Notation）是一个JSON的超集，非常 EMQX的配置文件可分为二层，自底向上依次是： -1. 不可变的基础层 `emqx.conf` 加上 `EMQX_` 前缀的环境变量。
- 修改这一层的配置之后，需要重启节点来使之生效。 -1. 集群范围重载层：`$EMQX_NODE__DATA_DIR/configs/cluster-override.conf` +1. 集群同步配置：`$EMQX_NODE__DATA_DIR/configs/cluster.hocon`。 +2. 本地节点配置：`emqx.conf` 加上 `EMQX_` 前缀的环境变量。 + +:::tip Tip +在 v5.0.23 或 e5.0.3 之前，集群同步配置保存在文件 `cluster-override.conf` 中，并且它覆盖在配置的最上层。 + +如果从之前的版本升级上来，只要 `cluster-override.conf` 文件存在， +EMQX 就不会创建 `cluster.hocon`，并且 `cluster-override.conf` 会继续覆盖在配置的最上层。 +::: 如果环境变量 `$EMQX_NODE__DATA_DIR` 没有设置，那么该目录会从 `emqx.conf` 的 `node.data_dir` 配置中读取。 @@ -16,9 +22,7 @@ EMQX的配置文件可分为二层，自底向上依次是： 当EMQX运行在集群中时，一个EMQX节点重启之后，会从集群中其他节点复制该文件内容到本地。 :::tip Tip -有些配置项是不能被重载的（例如 `node.name`）。 -配置项如果有 `mapping: path.to.boot.config.key` 这个属性， -则不能被添加到重载文件 `*-override.conf` 中。 +为避免歧义，应尽量避免让 `cluster.hocon` 和 `emqx.conf` 出现配置交集。 ::: 更多的重载规则，请参考下文 [配置重载规则](#配置重载规则)。 From 199cbc60d910022e48e57dc546faeb4fa70d42c9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 18 Apr 2023 10:19:09 -0300 Subject: [PATCH 245/279] chore: un-hide ocsp stapling config Undoing https://github.com/emqx/emqx/pull/10160 --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx/src/emqx_schema.erl | 3 +-- apps/emqx/test/emqx_ocsp_cache_SUITE.erl | 8 ++++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index b2dfca9e1..d42478fea 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.23"}, + {vsn, "5.0.24"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 38566f3cb..ace6d3332 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2322,8 +2322,6 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> ref("ocsp"), #{ required => false, - %% TODO: remove after e5.0.2 - importance => ?IMPORTANCE_HIDDEN, validator => fun ocsp_inner_validator/1 } )}, @@ -2332,6 +2330,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> boolean(), #{ default => false, + importance => ?IMPORTANCE_MEDIUM, desc => ?DESC("server_ssl_opts_schema_enable_crl_check") } )} diff --git a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl index dff8ce5a7..15ca29853 100644 --- a/apps/emqx/test/emqx_ocsp_cache_SUITE.erl +++ b/apps/emqx/test/emqx_ocsp_cache_SUITE.erl @@ -677,8 +677,12 @@ do_t_update_listener(Config) -> %% no ocsp at first ListenerId = "ssl:default", {ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId), - ?assertEqual( - undefined, + ?assertMatch( + #{ + <<"enable_ocsp_stapling">> := false, + <<"refresh_http_timeout">> := _, + <<"refresh_interval">> := _ + }, emqx_utils_maps:deep_get([<<"ssl_options">>, <<"ocsp">>], ListenerData0, undefined) ), assert_no_http_get(), From d947b663271890d4ceafde55254e83afa935add2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 19 Apr 2023 09:51:19 +0800 Subject: [PATCH 246/279] chore: update apps/emqx_slow_subs/README.md --- apps/emqx_slow_subs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_slow_subs/README.md b/apps/emqx_slow_subs/README.md index cfc87f775..8b83508c2 100644 --- a/apps/emqx_slow_subs/README.md +++ b/apps/emqx_slow_subs/README.md @@ -43,5 +43,5 @@ Default value: 10 Default value: whole -# Contributing - [Mandatory] +# Contributing Please see our [contributing.md](../../CONTRIBUTING.md). From 5455500647785e0d780590d26835df92f8f3c162 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 19 Apr 2023 14:17:43 +0800 Subject: [PATCH 247/279] fix(limiter): fix compatibility problem of configuration --- .../emqx_limiter/src/emqx_limiter_schema.erl | 76 +++++++++--------- apps/emqx/test/emqx_ratelimiter_SUITE.erl | 80 +++++++++++++++++-- .../src/emqx_dashboard_swagger.erl | 2 +- 3 files changed, 111 insertions(+), 47 deletions(-) diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index 730559f80..c762a0f1d 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -24,6 +24,7 @@ fields/1, to_rate/1, to_capacity/1, + to_burst/1, default_period/0, to_burst_rate/1, to_initial/1, @@ -54,8 +55,10 @@ -type bucket_name() :: atom(). -type rate() :: infinity | float(). -type burst_rate() :: 0 | float(). +%% this is a compatible type for the deprecated field and type `capacity`. +-type burst() :: burst_rate(). %% the capacity of the token bucket --type capacity() :: non_neg_integer(). +%%-type capacity() :: non_neg_integer(). %% initial capacity of the token bucket -type initial() :: non_neg_integer(). -type bucket_path() :: list(atom()). @@ -72,13 +75,13 @@ -typerefl_from_string({rate/0, ?MODULE, to_rate}). -typerefl_from_string({burst_rate/0, ?MODULE, to_burst_rate}). --typerefl_from_string({capacity/0, ?MODULE, to_capacity}). +-typerefl_from_string({burst/0, ?MODULE, to_burst}). -typerefl_from_string({initial/0, ?MODULE, to_initial}). -reflect_type([ rate/0, burst_rate/0, - capacity/0, + burst/0, initial/0, failure_strategy/0, bucket_name/0 @@ -130,39 +133,9 @@ fields(node_opts) -> fields(client_fields) -> client_fields(types(), #{default => #{}}); fields(bucket_infinity) -> - [ - {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})}, - {burst, - ?HOCON(capacity(), #{ - desc => ?DESC(capacity), - default => <<"0">>, - importance => ?IMPORTANCE_HIDDEN, - aliases => [capacity] - })}, - {initial, - ?HOCON(initial(), #{ - default => <<"0">>, - desc => ?DESC(initial), - importance => ?IMPORTANCE_HIDDEN - })} - ]; + fields_of_bucket(<<"infinity">>); fields(bucket_limit) -> - [ - {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"1000/s">>})}, - {burst, - ?HOCON(capacity(), #{ - desc => ?DESC(burst), - default => <<"0">>, - importance => ?IMPORTANCE_HIDDEN, - aliases => [capacity] - })}, - {initial, - ?HOCON(initial(), #{ - default => <<"0">>, - desc => ?DESC(initial), - importance => ?IMPORTANCE_HIDDEN - })} - ]; + fields_of_bucket(<<"1000/s">>); fields(client_opts) -> [ {rate, ?HOCON(rate(), #{default => <<"infinity">>, desc => ?DESC(rate)})}, @@ -186,7 +159,7 @@ fields(client_opts) -> } )}, {burst, - ?HOCON(capacity(), #{ + ?HOCON(burst(), #{ desc => ?DESC(burst), default => <<"0">>, importance => ?IMPORTANCE_HIDDEN, @@ -265,8 +238,6 @@ types() -> calc_capacity(#{rate := infinity}) -> infinity; -calc_capacity(#{burst := infinity}) -> - infinity; calc_capacity(#{rate := Rate, burst := Burst}) -> erlang:floor(1000 * Rate / default_period()) + Burst. @@ -277,6 +248,17 @@ calc_capacity(#{rate := Rate, burst := Burst}) -> to_burst_rate(Str) -> to_rate(Str, false, true). +%% The default value of `capacity` is `infinity`, +%% but we have changed `capacity` to `burst` which should not be `infinity` +%% and its default value is 0, so we should convert `infinity` to 0 +to_burst(Str) -> + case to_rate(Str, true, true) of + {ok, infinity} -> + {ok, 0}; + Any -> + Any + end. + %% rate can be: 10 10MB 10MB/s 10MB/2s infinity %% e.g. the bytes_in regex tree is: %% @@ -415,6 +397,24 @@ composite_bucket_fields(Types, ClientRef) -> )} ]. +fields_of_bucket(Default) -> + [ + {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => Default})}, + {burst, + ?HOCON(burst(), #{ + desc => ?DESC(burst), + default => <<"0">>, + importance => ?IMPORTANCE_HIDDEN, + aliases => [capacity] + })}, + {initial, + ?HOCON(initial(), #{ + default => <<"0">>, + desc => ?DESC(initial), + importance => ?IMPORTANCE_HIDDEN + })} + ]. + client_fields(Types, Meta) -> [ {Type, diff --git a/apps/emqx/test/emqx_ratelimiter_SUITE.erl b/apps/emqx/test/emqx_ratelimiter_SUITE.erl index 7288dcf7c..26048873e 100644 --- a/apps/emqx/test/emqx_ratelimiter_SUITE.erl +++ b/apps/emqx/test/emqx_ratelimiter_SUITE.erl @@ -220,7 +220,7 @@ t_try_restore_agg(_) -> }, Cli2 = Cli#{ rate := infinity, - burst := infinity, + burst := 0, divisible := true, max_retry_time := 100, failure_strategy := force @@ -264,11 +264,11 @@ t_rate(_) -> Bucket2 = Bucket#{ rate := ?RATE("100/100ms"), initial := 0, - burst := infinity + burst := 0 }, Cli2 = Cli#{ rate := infinity, - burst := infinity, + burst := 0, initial := 0 }, Bucket2#{client := Cli2} @@ -295,7 +295,7 @@ t_capacity(_) -> }, Cli2 = Cli#{ rate := infinity, - burst := infinity, + burst := 0, initial := 0 }, Bucket2#{client := Cli2} @@ -403,11 +403,11 @@ t_limit_global_with_unlimit_other(_) -> Bucket2 = Bucket#{ rate := infinity, initial := 0, - burst := infinity + burst := 0 }, Cli2 = Cli#{ rate := infinity, - burst := infinity, + burst := 0, initial := 0 }, Bucket2#{client := Cli2} @@ -574,6 +574,66 @@ t_schema_unit(_) -> ?assertEqual({ok, 100 * 1024 * 1024 * 1024}, M:to_capacity("100GB")), ok. +compatibility_for_capacity(_) -> + CfgStr = << + "" + "\n" + "listeners.tcp.default {\n" + " bind = \"0.0.0.0:1883\"\n" + " max_connections = 1024000\n" + " limiter.messages.capacity = infinity\n" + " limiter.client.messages.capacity = infinity\n" + "}\n" + "" + >>, + ?assertMatch( + #{ + messages := #{burst := 0}, + client := #{messages := #{burst := 0}} + }, + parse_and_check(CfgStr) + ). + +compatibility_for_message_in(_) -> + CfgStr = << + "" + "\n" + "listeners.tcp.default {\n" + " bind = \"0.0.0.0:1883\"\n" + " max_connections = 1024000\n" + " limiter.message_in.rate = infinity\n" + " limiter.client.message_in.rate = infinity\n" + "}\n" + "" + >>, + ?assertMatch( + #{ + messages := #{rate := infinity}, + client := #{messages := #{rate := infinity}} + }, + parse_and_check(CfgStr) + ). + +compatibility_for_bytes_in(_) -> + CfgStr = << + "" + "\n" + "listeners.tcp.default {\n" + " bind = \"0.0.0.0:1883\"\n" + " max_connections = 1024000\n" + " limiter.bytes_in.rate = infinity\n" + " limiter.client.bytes_in.rate = infinity\n" + "}\n" + "" + >>, + ?assertMatch( + #{ + bytes := #{rate := infinity}, + client := #{bytes := #{rate := infinity}} + }, + parse_and_check(CfgStr) + ). + %%-------------------------------------------------------------------- %%% Internal functions %%-------------------------------------------------------------------- @@ -753,13 +813,13 @@ make_limiter_cfg() -> Client = #{ rate => infinity, initial => 0, - burst => infinity, + burst => 0, low_watermark => 0, divisible => false, max_retry_time => timer:seconds(5), failure_strategy => force }, - #{client => Client, rate => infinity, initial => 0, burst => infinity}. + #{client => Client, rate => infinity, initial => 0, burst => 0}. add_bucket(Cfg) -> add_bucket(?MODULE, Cfg). @@ -813,3 +873,7 @@ apply_modifier(Pairs, #{default := Template}) -> Acc#{N => M(Template)} end, lists:foldl(Fun, #{}, Pairs). + +parse_and_check(ConfigString) -> + ok = emqx_common_test_helpers:load_config(emqx_schema, ConfigString), + emqx:get_config([listeners, tcp, default, limiter]). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index f700ec146..e471486e5 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -768,7 +768,7 @@ typename_to_spec("log_level()", _Mod) -> }; typename_to_spec("rate()", _Mod) -> #{type => string, example => <<"10MB">>}; -typename_to_spec("capacity()", _Mod) -> +typename_to_spec("burst()", _Mod) -> #{type => string, example => <<"100MB">>}; typename_to_spec("burst_rate()", _Mod) -> %% 0/0s = no burst From 4f0c891aa6f34c2d623961ad25b72a1e4c150664 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 19 Apr 2023 15:23:19 +0800 Subject: [PATCH 248/279] chore: bump version && update changes --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- changes/ce/fix-10448.en.md | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changes/ce/fix-10448.en.md diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index b2dfca9e1..d42478fea 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.23"}, + {vsn, "5.0.24"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index b810f9c5f..8c7e424e0 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.18"}, + {vsn, "5.0.19"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/changes/ce/fix-10448.en.md b/changes/ce/fix-10448.en.md new file mode 100644 index 000000000..eaa0dc656 --- /dev/null +++ b/changes/ce/fix-10448.en.md @@ -0,0 +1,3 @@ +Fix a compatibility issue of limiter configuration introduced by v5.0.23 which broke the upgrade from previous versions if the `capacity` is `infinity`. + +Since v5.0.23, we had instead of `capacity` by `burst`, after this fix, a `capacity` with `infinity` value will be converted to a `burst` with a zero value. From bc353b0a06e39e574866cfa1447d4fa54be6318f Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 19 Apr 2023 15:56:29 +0800 Subject: [PATCH 249/279] fix(dynamo): change `database` to `table` in the schema of the DynamoDB bridge there is no term like `database` in DynamoDB, the correct concept should be `table` --- .../src/emqx_ee_bridge_dynamo.erl | 2 +- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 2 +- .../src/emqx_ee_connector_dynamo.erl | 52 ++++++++++--------- rel/i18n/emqx_ee_connector_dynamo.hocon | 6 +++ 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl index ba1fd0c70..3157db18e 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl @@ -43,7 +43,7 @@ values(_Method) -> type => dynamo, name => <<"foo">>, url => <<"http://127.0.0.1:8000">>, - database => <<"mqtt">>, + table => <<"mqtt">>, pool_size => 8, username => <<"root">>, password => <<"******">>, diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index 5ebd9a89d..a63a69052 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -158,7 +158,7 @@ dynamo_config(BridgeType, Config) -> "bridges.~s.~s {\n" " enable = true\n" " url = ~p\n" - " database = ~p\n" + " table = ~p\n" " username = ~p\n" " password = ~p\n" " resource_opts = {\n" diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 85daefbb7..1e7203279 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -51,20 +51,24 @@ roots() -> fields(config) -> [ - {url, mk(binary(), #{required => true, desc => ?DESC("url")})} - | add_default_username( + {url, mk(binary(), #{required => true, desc => ?DESC("url")})}, + {table, mk(binary(), #{required => true, desc => ?DESC("table")})} + | override_schemas( emqx_connector_schema_lib:relational_db_fields() ) ]. -add_default_username(Fields) -> - lists:map( +override_schemas(Fields) -> + lists:foldr( fun - ({username, OrigUsernameFn}) -> - {username, add_default_fn(OrigUsernameFn, <<"root">>)}; - (Field) -> - Field + ({username, OrigUsernameFn}, Acc) -> + [{username, add_default_fn(OrigUsernameFn, <<"root">>)} | Acc]; + ({database, _}, Acc) -> + Acc; + (Field, Acc) -> + [Field | Acc] end, + [], Fields ). @@ -88,7 +92,7 @@ on_start( url := Url, username := Username, password := Password, - database := Database, + table := Table, pool_size := PoolSize } = Config ) -> @@ -115,7 +119,7 @@ on_start( Templates = parse_template(Config), State = #{ poolname => InstanceId, - database => Database, + table => Table, templates => Templates }, case emqx_plugin_libs_pool:start_pool(InstanceId, ?MODULE, Options) of @@ -183,7 +187,7 @@ do_query( InstanceId, Query, ApplyMode, - #{poolname := PoolName, templates := Templates, database := Database} = State + #{poolname := PoolName, templates := Templates, table := Table} = State ) -> ?TRACE( "QUERY", @@ -192,7 +196,7 @@ do_query( ), Result = ecpool:pick_and_do( PoolName, - {?MODULE, worker_do_query, [Database, Query, Templates]}, + {?MODULE, worker_do_query, [Table, Query, Templates]}, ApplyMode ), @@ -217,33 +221,33 @@ do_query( Result end. -worker_do_query(_Client, Database, Query0, Templates) -> +worker_do_query(_Client, Table, Query0, Templates) -> try Query = apply_template(Query0, Templates), - execute(Query, Database) + execute(Query, Table) catch _Type:Reason -> {error, {unrecoverable_error, {invalid_request, Reason}}} end. %% some simple query commands for authn/authz or test -execute({insert_item, Msg}, Database) -> +execute({insert_item, Msg}, Table) -> Item = convert_to_item(Msg), - erlcloud_ddb2:put_item(Database, Item); -execute({delete_item, Key}, Database) -> - erlcloud_ddb2:delete_item(Database, Key); -execute({get_item, Key}, Database) -> - erlcloud_ddb2:get_item(Database, Key); + erlcloud_ddb2:put_item(Table, Item); +execute({delete_item, Key}, Table) -> + erlcloud_ddb2:delete_item(Table, Key); +execute({get_item, Key}, Table) -> + erlcloud_ddb2:get_item(Table, Key); %% commands for data bridge query or batch query -execute({send_message, Msg}, Database) -> +execute({send_message, Msg}, Table) -> Item = convert_to_item(Msg), - erlcloud_ddb2:put_item(Database, Item); -execute([{put, _} | _] = Msgs, Database) -> + erlcloud_ddb2:put_item(Table, Item); +execute([{put, _} | _] = Msgs, Table) -> %% type of batch_write_item argument :: batch_write_item_request_items() %% batch_write_item_request_items() :: maybe_list(batch_write_item_request_item()) %% batch_write_item_request_item() :: {table_name(), list(batch_write_item_request())} %% batch_write_item_request() :: {put, item()} | {delete, key()} - erlcloud_ddb2:batch_write_item({Database, Msgs}). + erlcloud_ddb2:batch_write_item({Table, Msgs}). connect(Opts) -> #{ diff --git a/rel/i18n/emqx_ee_connector_dynamo.hocon b/rel/i18n/emqx_ee_connector_dynamo.hocon index 295929a72..6d3c69be4 100644 --- a/rel/i18n/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/emqx_ee_connector_dynamo.hocon @@ -11,4 +11,10 @@ emqx_ee_connector_dynamo { } } +table.desc: +"""DynamoDB Table.""" + +table.label: +"""DynamoDB Table""" + } From b112f544a98e4c3b77d52b9ff444d679688b32b2 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 18 Apr 2023 19:00:35 +0800 Subject: [PATCH 250/279] chore: update changes --- changes/ee/fix-10438.en.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/ee/fix-10438.en.md diff --git a/changes/ee/fix-10438.en.md b/changes/ee/fix-10438.en.md new file mode 100644 index 000000000..859a35499 --- /dev/null +++ b/changes/ee/fix-10438.en.md @@ -0,0 +1,3 @@ +Fix a configuration item name error in the DynamoDB data bridge. + +Changed `database` to `table`, because there is no term like `database` in DynamoDB, the correct concept should be `table` From e89f4d45658f33865deea046d352402661d7508d Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 19 Apr 2023 15:56:46 +0800 Subject: [PATCH 251/279] fix(dynamo): fix terminology erros - Changed `username` to `aws_access_key_id` - Changed `password` to `aws_secret_access_key` --- changes/ee/fix-10438.en.md | 6 +- .../src/emqx_ee_bridge_dynamo.erl | 4 +- .../test/emqx_ee_bridge_dynamo_SUITE.erl | 16 +++--- .../src/emqx_ee_connector_dynamo.erl | 56 ++++++++----------- rel/i18n/emqx_ee_connector_dynamo.hocon | 34 +++++++++-- 5 files changed, 68 insertions(+), 48 deletions(-) diff --git a/changes/ee/fix-10438.en.md b/changes/ee/fix-10438.en.md index 859a35499..6394bc3cf 100644 --- a/changes/ee/fix-10438.en.md +++ b/changes/ee/fix-10438.en.md @@ -1,3 +1,5 @@ -Fix a configuration item name error in the DynamoDB data bridge. +Fix some configuration item terminology errors in the DynamoDB data bridge: -Changed `database` to `table`, because there is no term like `database` in DynamoDB, the correct concept should be `table` +- Changed `database` to `table` +- Changed `username` to `aws_access_key_id` +- Changed `password` to `aws_secret_access_key` diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl index 3157db18e..cbfa5b6b1 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl @@ -45,8 +45,8 @@ values(_Method) -> url => <<"http://127.0.0.1:8000">>, table => <<"mqtt">>, pool_size => 8, - username => <<"root">>, - password => <<"******">>, + aws_access_key_id => <<"root">>, + aws_secret_access_key => <<"******">>, template => ?DEFAULT_TEMPLATE, local_topic => <<"local/topic/#">>, resource_opts => #{ diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl index a63a69052..9cf7eb8f4 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl @@ -14,8 +14,8 @@ % DB defaults -define(TABLE, "mqtt"). -define(TABLE_BIN, to_bin(?TABLE)). --define(USERNAME, "root"). --define(PASSWORD, "public"). +-define(ACCESS_KEY_ID, "root"). +-define(SECRET_ACCESS_KEY, "public"). -define(HOST, "dynamo"). -define(PORT, 8000). -define(SCHEMA, "http://"). @@ -159,8 +159,8 @@ dynamo_config(BridgeType, Config) -> " enable = true\n" " url = ~p\n" " table = ~p\n" - " username = ~p\n" - " password = ~p\n" + " aws_access_key_id = ~p\n" + " aws_secret_access_key = ~p\n" " resource_opts = {\n" " request_timeout = 500ms\n" " batch_size = ~b\n" @@ -172,8 +172,8 @@ dynamo_config(BridgeType, Config) -> Name, Url, ?TABLE, - ?USERNAME, - ?PASSWORD, + ?ACCESS_KEY_ID, + ?SECRET_ACCESS_KEY, BatchSize, QueryMode ] @@ -244,10 +244,10 @@ delete_table(_Config) -> setup_dynamo(Config) -> Host = ?GET_CONFIG(host, Config), Port = ?GET_CONFIG(port, Config), - erlcloud_ddb2:configure(?USERNAME, ?PASSWORD, Host, Port, ?SCHEMA). + erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, Host, Port, ?SCHEMA). directly_setup_dynamo() -> - erlcloud_ddb2:configure(?USERNAME, ?PASSWORD, ?HOST, ?PORT, ?SCHEMA). + erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, ?HOST, ?PORT, ?SCHEMA). directly_query(Query) -> directly_setup_dynamo(), diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index 1e7203279..9a149b6f7 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -52,32 +52,21 @@ roots() -> fields(config) -> [ {url, mk(binary(), #{required => true, desc => ?DESC("url")})}, - {table, mk(binary(), #{required => true, desc => ?DESC("table")})} - | override_schemas( - emqx_connector_schema_lib:relational_db_fields() - ) + {table, mk(binary(), #{required => true, desc => ?DESC("table")})}, + {aws_access_key_id, + mk( + binary(), + #{required => true, desc => ?DESC("aws_access_key_id")} + )}, + {aws_secret_access_key, + mk( + binary(), + #{required => true, desc => ?DESC("aws_secret_access_key")} + )}, + {pool_size, fun emqx_connector_schema_lib:pool_size/1}, + {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} ]. -override_schemas(Fields) -> - lists:foldr( - fun - ({username, OrigUsernameFn}, Acc) -> - [{username, add_default_fn(OrigUsernameFn, <<"root">>)} | Acc]; - ({database, _}, Acc) -> - Acc; - (Field, Acc) -> - [Field | Acc] - end, - [], - Fields - ). - -add_default_fn(OrigFn, Default) -> - fun - (default) -> Default; - (Field) -> OrigFn(Field) - end. - %%======================================================================================== %% `emqx_resource' API %%======================================================================================== @@ -90,8 +79,8 @@ on_start( InstanceId, #{ url := Url, - username := Username, - password := Password, + aws_access_key_id := AccessKeyID, + aws_secret_access_key := SecretAccessKey, table := Table, pool_size := PoolSize } = Config @@ -99,7 +88,7 @@ on_start( ?SLOG(info, #{ msg => "starting_dynamo_connector", connector => InstanceId, - config => emqx_utils:redact(Config) + config => redact(Config) }), {Schema, Server} = get_host_schema(to_str(Url)), @@ -109,8 +98,8 @@ on_start( {config, #{ host => Host, port => Port, - username => to_str(Username), - password => to_str(Password), + aws_access_key_id => to_str(AccessKeyID), + aws_secret_access_key => to_str(SecretAccessKey), schema => Schema }}, {pool_size, PoolSize} @@ -251,13 +240,13 @@ execute([{put, _} | _] = Msgs, Table) -> connect(Opts) -> #{ - username := Username, - password := Password, + aws_access_key_id := AccessKeyID, + aws_secret_access_key := SecretAccessKey, host := Host, port := Port, schema := Schema } = proplists:get_value(config, Opts), - erlcloud_ddb2:configure(Username, Password, Host, Port, Schema), + erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema), %% The dynamodb driver uses caller process as its connection process %% so at here, the connection process is the ecpool worker self @@ -342,3 +331,6 @@ convert2binary(Value) when is_map(Value) -> do_async_reply(Result, {ReplyFun, [Context]}) -> ReplyFun(Context, Result). + +redact(Data) -> + emqx_utils:redact(Data, fun(Any) -> Any =:= aws_secret_access_key end). diff --git a/rel/i18n/emqx_ee_connector_dynamo.hocon b/rel/i18n/emqx_ee_connector_dynamo.hocon index 6d3c69be4..a1f3f7b02 100644 --- a/rel/i18n/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/emqx_ee_connector_dynamo.hocon @@ -11,10 +11,36 @@ emqx_ee_connector_dynamo { } } -table.desc: -"""DynamoDB Table.""" + table { + desc { + en: """DynamoDB Table.""" + zh: """DynamoDB 的表。""" + } + label: { + en: "Table " + zh: "表" + } + } -table.label: -"""DynamoDB Table""" + aws_access_key_id { + desc { + en: """Access Key ID for connecting to DynamoDB.""" + zh: """DynamoDB 的访问 ID。""" + } + label: { + en: "AWS Access Key ID" + zh: "连接访问 ID" + } + } + aws_secret_access_key { + desc { + en: """AWS Secret Access Key for connecting to DynamoDB.""" + zh: """DynamoDB 的访问密钥。""" + } + label: { + en: "AWS Secret Access Key" + zh: "连接访问密钥" + } + } } From 8219af2fd439acb1a75c3f3dca0399361082b31e Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 19 Apr 2023 16:45:53 +0800 Subject: [PATCH 252/279] chore: improve changelog --- changes/ce/fix-10448.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/fix-10448.en.md b/changes/ce/fix-10448.en.md index eaa0dc656..c35ecdd8b 100644 --- a/changes/ce/fix-10448.en.md +++ b/changes/ce/fix-10448.en.md @@ -1,3 +1,3 @@ Fix a compatibility issue of limiter configuration introduced by v5.0.23 which broke the upgrade from previous versions if the `capacity` is `infinity`. -Since v5.0.23, we had instead of `capacity` by `burst`, after this fix, a `capacity` with `infinity` value will be converted to a `burst` with a zero value. +In v5.0.23 we have replaced `capacity` with `burst`. After this fix, a `capacity = infinity` config will be automatically converted to equivalent `burst = 0`. From 39c213d2d7f0f9e684426beee19da30a7e996f87 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Wed, 19 Apr 2023 18:12:43 +0800 Subject: [PATCH 253/279] chore: upgrade dashboard to v1.2.3 for ce --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c874b81ce..10e6d1424 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.2.2 +export EMQX_DASHBOARD_VERSION ?= v1.2.3 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.6-beta.1 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 From bd935e34e67283ebc0f0f4a1924df152138bdea3 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 19 Apr 2023 21:12:24 +0800 Subject: [PATCH 254/279] chore: update apps/emqx_modules/README.md --- apps/emqx_modules/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_modules/README.md b/apps/emqx_modules/README.md index 7cb1db0c5..65ceb12e1 100644 --- a/apps/emqx_modules/README.md +++ b/apps/emqx_modules/README.md @@ -6,7 +6,7 @@ protocol standard, including "Delayed Publish", "Topic Rewrite", "Topic Metrics" ## Delayed Publish -After enabling this module, messages sent by the user with the prefix +After enabling this module, messages sent by the clients with the topic prefixed with `$delayed/{Interval}/{Topic}` will be delayed by `{Interval}` seconds before being published to the `{Topic}`. From eda2f0819d1560226038cd01a64eca77f65325b3 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 19 Apr 2023 21:22:01 +0800 Subject: [PATCH 255/279] chore: apply suggestions from code review --- apps/emqx_modules/README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/emqx_modules/README.md b/apps/emqx_modules/README.md index 65ceb12e1..567b1ac71 100644 --- a/apps/emqx_modules/README.md +++ b/apps/emqx_modules/README.md @@ -20,10 +20,9 @@ See [Enabling/Disabling Delayed Publish via HTTP API](https://www.emqx.io/docs/e Topic Rewrite allows users to configure rules to change the topic strings that the client requests to subscribe or publish. -This feature is very useful when designing topics that are compatible with different -client versions. For example, an old device that has already been issued and cannot -be upgraded may use old topic rules, but the production environment need to apply -a new design rules for the topics. +This feature is very useful when it need to compatibility with different versions of topic designs. +For example, an old device that has already been issued and cannot +be upgraded may use old topic designs, but for some reason, we adjusted the format of topics. We can use this feature to rewrite the old topics as the new format to eliminate these differences. More introduction about [Topic Rewrite](https://www.emqx.io/docs/en/v5.0/mqtt/mqtt-topic-rewrite.html). From a9bd91fcfffa2d6cb44fc7772e2937bff3bbc6a8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 19 Apr 2023 10:23:04 -0300 Subject: [PATCH 256/279] refactor(gcp_pubsub): move GCP PubSub Bridge to its own app Fixes https://emqx.atlassian.net/browse/EMQX-9536 Note: since GCP PubSub is not shared by any authn/authz backend, there's no need to separate its connector into another app. --- apps/emqx_bridge_gcp_pubsub/rebar.config | 10 ++++++++++ .../src/emqx_bridge_gcp_pubsub.app.src | 6 +++++- .../src/emqx_bridge_gcp_pubsub.erl | 2 +- .../src/emqx_bridge_gcp_pubsub_connector.erl | 4 ++-- .../test/emqx_bridge_gcp_pubsub_SUITE.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 5 +++-- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 8 ++++---- lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +- mix.exs | 4 +++- rebar.config.erl | 2 ++ ...e_gcp_pubsub.hocon => emqx_bridge_gcp_pubsub.hocon} | 2 +- ...e_gcp_pubsub.hocon => emqx_bridge_gcp_pubsub.hocon} | 2 +- scripts/find-apps.sh | 3 +++ 13 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 apps/emqx_bridge_gcp_pubsub/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl => apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl (99%) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl => apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl (99%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl => apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl (99%) rename rel/i18n/{emqx_ee_bridge_gcp_pubsub.hocon => emqx_bridge_gcp_pubsub.hocon} (98%) rename rel/i18n/zh/{emqx_ee_bridge_gcp_pubsub.hocon => emqx_bridge_gcp_pubsub.hocon} (98%) diff --git a/apps/emqx_bridge_gcp_pubsub/rebar.config b/apps/emqx_bridge_gcp_pubsub/rebar.config new file mode 100644 index 000000000..2fd264fc0 --- /dev/null +++ b/apps/emqx_bridge_gcp_pubsub/rebar.config @@ -0,0 +1,10 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_gcp_pubsub]} +]}. diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index 0e1427888..86627eb2a 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -2,7 +2,11 @@ {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [ + kernel, + stdlib, + ehttpc + ]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl rename to apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl index 180640d65..70109a0ea 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_gcp_pubsub). +-module(emqx_bridge_gcp_pubsub). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl rename to apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl index 7b068ec8f..a3f0ef36b 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_gcp_pubsub). +-module(emqx_bridge_gcp_pubsub_connector). -behaviour(emqx_resource). @@ -27,7 +27,7 @@ -export([reply_delegator/3]). -type jwt_worker() :: binary(). --type service_account_json() :: emqx_ee_bridge_gcp_pubsub:service_account_json(). +-type service_account_json() :: emqx_bridge_gcp_pubsub:service_account_json(). -type config() :: #{ connect_timeout := emqx_schema:duration_ms(), max_retries := non_neg_integer(), diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl rename to apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl index 57c06fa7f..55527bf1f 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_gcp_pubsub_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_gcp_pubsub_SUITE). +-module(emqx_bridge_gcp_pubsub_SUITE). -compile(nowarn_export_all). -compile(export_all). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index d6c59c716..440889d02 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,13 +1,14 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.10"}, + {vsn, "0.1.11"}, {registered, []}, {applications, [ kernel, stdlib, emqx_ee_connector, telemetry, - emqx_bridge_kafka + emqx_bridge_kafka, + emqx_bridge_gcp_pubsub ]}, {env, []}, {modules, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 3ad5cbbb4..9465464d9 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -14,7 +14,7 @@ api_schemas(Method) -> [ - ref(emqx_ee_bridge_gcp_pubsub, Method), + ref(emqx_bridge_gcp_pubsub, Method), ref(emqx_bridge_kafka, Method ++ "_consumer"), ref(emqx_bridge_kafka, Method ++ "_producer"), ref(emqx_ee_bridge_mysql, Method), @@ -42,7 +42,7 @@ schema_modules() -> [ emqx_bridge_kafka, emqx_ee_bridge_hstreamdb, - emqx_ee_bridge_gcp_pubsub, + emqx_bridge_gcp_pubsub, emqx_ee_bridge_influxdb, emqx_ee_bridge_mongodb, emqx_ee_bridge_mysql, @@ -76,7 +76,7 @@ resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer; %% to hocon; keeping this as just `kafka' for backwards compatibility. resource_type(kafka) -> emqx_bridge_kafka_impl_producer; resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; -resource_type(gcp_pubsub) -> emqx_ee_connector_gcp_pubsub; +resource_type(gcp_pubsub) -> emqx_bridge_gcp_pubsub_connector; resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; resource_type(mongodb_sharded) -> emqx_ee_connector_mongodb; resource_type(mongodb_single) -> emqx_ee_connector_mongodb; @@ -108,7 +108,7 @@ fields(bridges) -> )}, {gcp_pubsub, mk( - hoconsc:map(name, ref(emqx_ee_bridge_gcp_pubsub, "config")), + hoconsc:map(name, ref(emqx_bridge_gcp_pubsub, "config")), #{ desc => <<"EMQX Enterprise Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 7ebc320e5..ced7ae86a 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.10"}, + {vsn, "0.1.11"}, {registered, []}, {applications, [ kernel, diff --git a/mix.exs b/mix.exs index 1dbde8980..f3149a584 100644 --- a/mix.exs +++ b/mix.exs @@ -154,7 +154,8 @@ defmodule EMQXUmbrella.MixProject do # need to remove those when listing `/apps/`... defp enterprise_umbrella_apps() do MapSet.new([ - :emqx_bridge_kafka + :emqx_bridge_kafka, + :emqx_bridge_gcp_pubsub ]) end @@ -344,6 +345,7 @@ defmodule EMQXUmbrella.MixProject do emqx_ee_connector: :permanent, emqx_ee_bridge: :permanent, emqx_bridge_kafka: :permanent, + emqx_bridge_gcp_pubsub: :permanent, emqx_ee_schema_registry: :permanent ], else: [] diff --git a/rebar.config.erl b/rebar.config.erl index 80f126096..9bdbfb848 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -79,6 +79,7 @@ is_enterprise(ce) -> false; is_enterprise(ee) -> true. is_community_umbrella_app("apps/emqx_bridge_kafka") -> false; +is_community_umbrella_app("apps/emqx_bridge_gcp_pubsub") -> false; is_community_umbrella_app(_) -> true. is_jq_supported() -> @@ -439,6 +440,7 @@ relx_apps_per_edition(ee) -> emqx_ee_connector, emqx_ee_bridge, emqx_bridge_kafka, + emqx_bridge_gcp_pubsub, emqx_ee_schema_registry ]; relx_apps_per_edition(ce) -> diff --git a/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon b/rel/i18n/emqx_bridge_gcp_pubsub.hocon similarity index 98% rename from rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon rename to rel/i18n/emqx_bridge_gcp_pubsub.hocon index 6f864a524..cc255aec3 100644 --- a/rel/i18n/emqx_ee_bridge_gcp_pubsub.hocon +++ b/rel/i18n/emqx_bridge_gcp_pubsub.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_gcp_pubsub { +emqx_bridge_gcp_pubsub { connect_timeout.desc: """The timeout when connecting to the HTTP server.""" diff --git a/rel/i18n/zh/emqx_ee_bridge_gcp_pubsub.hocon b/rel/i18n/zh/emqx_bridge_gcp_pubsub.hocon similarity index 98% rename from rel/i18n/zh/emqx_ee_bridge_gcp_pubsub.hocon rename to rel/i18n/zh/emqx_bridge_gcp_pubsub.hocon index 4318211c9..19bd7058e 100644 --- a/rel/i18n/zh/emqx_ee_bridge_gcp_pubsub.hocon +++ b/rel/i18n/zh/emqx_bridge_gcp_pubsub.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_gcp_pubsub { +emqx_bridge_gcp_pubsub { connect_timeout.desc: """连接 HTTP 服务器的超时时间。""" diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 66990ae12..eed2acdb8 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -75,6 +75,9 @@ describe_app() { apps/emqx_bridge_kafka) profile='emqx-enterprise' ;; + apps/emqx_bridge_gcp_pubsub) + profile='emqx-enterprise' + ;; apps/*) profile='emqx' ;; From cb995e20330d89aa44270c925058318b5a9e128b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 19 Apr 2023 17:42:27 -0300 Subject: [PATCH 257/279] fix(buffer_worker): avoid sending late reply messages to callers Fixes https://emqx.atlassian.net/browse/EMQX-9635 During a sync call from process `A` to a buffer worker `B`, its call to the underlying resource `C` can be very slow. In those cases, `A` will receive a timeout response and expect no more messages from `B` nor `C`. However, prior to this fix, if `B` is stuck in a long sync call to `C` and then gets its response after `A` timed out, `B` would still send the late response to `A`, polluting its mailbox. --- apps/emqx_resource/src/emqx_resource.app.src | 2 +- .../src/emqx_resource_buffer_worker.erl | 17 +++++-- .../test/emqx_connector_demo.erl | 6 ++- .../test/emqx_resource_SUITE.erl | 45 +++++++++++++++++++ changes/ce/fix-10455.en.md | 9 ++++ 5 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 changes/ce/fix-10455.en.md diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 00c315714..2553e6dd8 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.13"}, + {vsn, "0.1.14"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index e34cf5d0a..2e2cd5631 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -52,7 +52,7 @@ -export([queue_item_marshaller/1, estimate_size/1]). --export([handle_async_reply/2, handle_async_batch_reply/2]). +-export([handle_async_reply/2, handle_async_batch_reply/2, reply_call/2]). -export([clear_disk_queue_dir/2]). @@ -293,10 +293,8 @@ code_change(_OldVsn, State, _Extra) -> pick_call(Id, Key, Query, Timeout) -> ?PICK(Id, Key, Pid, begin - Caller = self(), MRef = erlang:monitor(process, Pid, [{alias, reply_demonitor}]), - From = {Caller, MRef}, - ReplyTo = {fun gen_statem:reply/2, [From]}, + ReplyTo = {fun ?MODULE:reply_call/2, [MRef]}, erlang:send(Pid, ?SEND_REQ(ReplyTo, Query)), receive {MRef, Response} -> @@ -1703,6 +1701,17 @@ default_resume_interval(_RequestTimeout = infinity, HealthCheckInterval) -> default_resume_interval(RequestTimeout, HealthCheckInterval) -> max(1, min(HealthCheckInterval, RequestTimeout div 3)). +-spec reply_call(reference(), term()) -> ok. +reply_call(Alias, Response) -> + %% Since we use a reference created with `{alias, + %% reply_demonitor}', after we `demonitor' it in case of a + %% timeout, we won't send any more messages that the caller is not + %% expecting anymore. Using `gen_statem:reply({pid(), + %% reference()}, _)' would still send a late reply even after the + %% demonitor. + erlang:send(Alias, {Alias, Response}), + ok. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). adjust_batch_time_test_() -> diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index a1393c574..5be854e93 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -144,7 +144,11 @@ on_query(_InstId, {sleep_before_reply, For}, #{pid := Pid}) -> Result after 1000 -> {error, timeout} - end. + end; +on_query(_InstId, {sync_sleep_before_reply, SleepFor}, _State) -> + %% This simulates a slow sync call + timer:sleep(SleepFor), + {ok, slept}. on_query_async(_InstId, block, ReplyFun, #{pid := Pid}) -> Pid ! {block, ReplyFun}, diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 385b4cb91..e098c2e1c 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -2751,6 +2751,51 @@ t_volatile_offload_mode(_Config) -> end ). +t_late_call_reply(_Config) -> + emqx_connector_demo:set_callback_mode(always_sync), + RequestTimeout = 500, + ?assertMatch( + {ok, _}, + emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{ + buffer_mode => memory_only, + request_timeout => RequestTimeout, + query_mode => sync + } + ) + ), + ?check_trace( + begin + %% Sleep for longer than the request timeout; the call reply will + %% have been already returned (a timeout), but the resource will + %% still send a message with the reply. + %% The demo connector will reply with `{error, timeout}' after 1 s. + SleepFor = RequestTimeout + 500, + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + emqx_resource:query( + ?ID, + {sync_sleep_before_reply, SleepFor}, + #{timeout => RequestTimeout} + ) + ), + %% Our process shouldn't receive any late messages. + receive + LateReply -> + ct:fail("received late reply: ~p", [LateReply]) + after SleepFor -> + ok + end, + ok + end, + [] + ), + ok. + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/changes/ce/fix-10455.en.md b/changes/ce/fix-10455.en.md new file mode 100644 index 000000000..07d8c71db --- /dev/null +++ b/changes/ce/fix-10455.en.md @@ -0,0 +1,9 @@ +Fixed an issue that could cause (otherwise harmless) noise in the logs. + +During some particularly slow synchronous calls to bridges, some late replies could be sent to connections processes that were no longer expecting a reply, and then emit an error log like: + +``` +2023-04-19T18:24:35.350233+00:00 [error] msg: unexpected_info, mfa: emqx_channel:handle_info/2, line: 1278, peername: 172.22.0.1:36384, clientid: caribdis_bench_sub_1137967633_4788, info: {#Ref<0.408802983.1941504010.189402>,{ok,200,[{<<"cache-control">>,<<"max-age=0, ...">>}} +``` + +Those logs are harmless, but they could flood and worry the users without need. From 712f7b74540af5e0c0b1e3352b836feaa2083665 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 20 Apr 2023 14:15:09 +0800 Subject: [PATCH 258/279] chore: deprecate statsd --- apps/emqx_statsd/src/emqx_statsd_api.erl | 2 ++ apps/emqx_statsd/src/emqx_statsd_schema.erl | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index e65c93432..4ee144d57 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -49,6 +49,7 @@ schema("/statsd") -> 'operationId' => statsd, get => #{ + deprecated => true, description => ?DESC(get_statsd_config_api), tags => ?API_TAG_STATSD, responses => @@ -56,6 +57,7 @@ schema("/statsd") -> }, put => #{ + deprecated => true, description => ?DESC(update_statsd_config_api), tags => ?API_TAG_STATSD, 'requestBody' => statsd_config_schema(), diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl index e44f94954..01decc6f7 100644 --- a/apps/emqx_statsd/src/emqx_statsd_schema.erl +++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl @@ -32,7 +32,8 @@ namespace() -> "statsd". -roots() -> ["statsd"]. +roots() -> + [{"statsd", hoconsc:mk(hoconsc:ref(?MODULE, "statsd"), #{importance => ?IMPORTANCE_HIDDEN})}]. fields("statsd") -> [ From 794ddd8d73ab62fade2c2998d470db7efc3b03c8 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 20 Apr 2023 14:23:55 +0800 Subject: [PATCH 259/279] chore: update changes --- apps/emqx_statsd/src/emqx_statsd.app.src | 2 +- changes/ce/feat-10457.en.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changes/ce/feat-10457.en.md diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 412e0b685..87fc8c596 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_statsd, [ {description, "EMQX Statsd"}, - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, [ diff --git a/changes/ce/feat-10457.en.md b/changes/ce/feat-10457.en.md new file mode 100644 index 000000000..d6a44bd53 --- /dev/null +++ b/changes/ce/feat-10457.en.md @@ -0,0 +1,4 @@ +Deprecates the integration with StatsD. + +Since StatsD is not used a lot. So we will deprecate it in the next release +and plan to remove it in 5.1 From adc422d0635a8ecebfb1220bd10cea29f786f94a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 20 Apr 2023 11:11:35 +0200 Subject: [PATCH 260/279] chore: re-split dynamo i18n file --- rel/i18n/emqx_ee_connector_dynamo.hocon | 50 ++++++++-------------- rel/i18n/zh/emqx_ee_connector_dynamo.hocon | 18 ++++++++ scripts/split-i18n-files.escript | 12 ++++-- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/rel/i18n/emqx_ee_connector_dynamo.hocon b/rel/i18n/emqx_ee_connector_dynamo.hocon index a0c7aacbe..29b6bf99e 100644 --- a/rel/i18n/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/emqx_ee_connector_dynamo.hocon @@ -1,41 +1,27 @@ emqx_ee_connector_dynamo { +aws_access_key_id.desc: +"""Access Key ID for connecting to DynamoDB.""" + +aws_access_key_id.label: +"""AWS Access Key ID""" + +aws_secret_access_key.desc: +"""AWS Secret Access Key for connecting to DynamoDB.""" + +aws_secret_access_key.label: +"""AWS Secret Access Key""" + +table.desc: +"""DynamoDB Table.""" + +table.label: +"""Table """ + url.desc: """The url of DynamoDB endpoint.""" url.label: """DynamoDB Endpoint""" - table { - desc { - en: """DynamoDB Table.""" - zh: """DynamoDB 的表。""" - } - label: { - en: "Table " - zh: "表" - } - } - - aws_access_key_id { - desc { - en: """Access Key ID for connecting to DynamoDB.""" - zh: """DynamoDB 的访问 ID。""" - } - label: { - en: "AWS Access Key ID" - zh: "连接访问 ID" - } - } - - aws_secret_access_key { - desc { - en: """AWS Secret Access Key for connecting to DynamoDB.""" - zh: """DynamoDB 的访问密钥。""" - } - label: { - en: "AWS Secret Access Key" - zh: "连接访问密钥" - } - } } diff --git a/rel/i18n/zh/emqx_ee_connector_dynamo.hocon b/rel/i18n/zh/emqx_ee_connector_dynamo.hocon index 540d79dd0..e7b911c1e 100644 --- a/rel/i18n/zh/emqx_ee_connector_dynamo.hocon +++ b/rel/i18n/zh/emqx_ee_connector_dynamo.hocon @@ -1,5 +1,23 @@ emqx_ee_connector_dynamo { +aws_access_key_id.desc: +"""DynamoDB 的访问 ID。""" + +aws_access_key_id.label: +"""连接访问 ID""" + +aws_secret_access_key.desc: +"""DynamoDB 的访问密钥。""" + +aws_secret_access_key.label: +"""连接访问密钥""" + +table.desc: +"""DynamoDB 的表。""" + +table.label: +"""表""" + url.desc: """DynamoDB 的地址。""" diff --git a/scripts/split-i18n-files.escript b/scripts/split-i18n-files.escript index 5910db667..b9f558925 100755 --- a/scripts/split-i18n-files.escript +++ b/scripts/split-i18n-files.escript @@ -27,9 +27,13 @@ add_ebin(Dir) -> split_file(Path) -> {ok, DescMap} = hocon:load(Path), [{Module, Descs}] = maps:to_list(DescMap), - ok = split(Path, Module, <<"en">>, Descs), - ok = split(Path, Module, <<"zh">>, Descs), - ok. + try + ok = split(Path, Module, <<"en">>, Descs), + ok = split(Path, Module, <<"zh">>, Descs) + catch + throw : already_done -> + ok + end. split(Path, Module, Lang, Fields) when is_map(Fields) -> split(Path, Module, Lang, maps:to_list(Fields)); @@ -54,6 +58,8 @@ rename(FilePath, Lang) -> BaseName = filename:basename(FilePath), filename:join([Dir, Lang, BaseName]). +do_split(_Path, _Name, _Lang, #{<<"desc">> := Desc}) when is_binary(Desc) -> + throw(already_done); do_split(Path, Name, Lang, #{<<"desc">> := Desc} = D) -> try Label = maps:get(<<"label">>, D, #{}), From bcc8f4313bf0f26e43dd5148f43ef465337bc149 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 20 Apr 2023 18:00:09 +0800 Subject: [PATCH 261/279] chore: make plugins config to low level --- apps/emqx_plugins/src/emqx_plugins_schema.erl | 15 +++++++++------ .../emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- changes/ce/feat-10458.en.md | 3 +++ .../src/emqx_ee_connector.app.src | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 changes/ce/feat-10458.en.md diff --git a/apps/emqx_plugins/src/emqx_plugins_schema.erl b/apps/emqx_plugins/src/emqx_plugins_schema.erl index 9d9d045de..b86f6b6c1 100644 --- a/apps/emqx_plugins/src/emqx_plugins_schema.erl +++ b/apps/emqx_plugins/src/emqx_plugins_schema.erl @@ -29,7 +29,7 @@ namespace() -> "plugin". -roots() -> [?CONF_ROOT]. +roots() -> [{?CONF_ROOT, ?HOCON(?R_REF(?CONF_ROOT), #{importance => ?IMPORTANCE_LOW})}]. fields(?CONF_ROOT) -> #{ @@ -73,16 +73,19 @@ states(type) -> ?ARRAY(?R_REF(state)); states(required) -> false; states(default) -> []; states(desc) -> ?DESC(states); +states(importance) -> ?IMPORTANCE_HIGH; states(_) -> undefined. install_dir(type) -> string(); install_dir(required) -> false; -%% runner's root dir +%% runner's root dir todo move to data dir in 5.1 install_dir(default) -> <<"plugins">>; -install_dir(T) when T =/= desc -> undefined; -install_dir(desc) -> ?DESC(install_dir). +install_dir(desc) -> ?DESC(install_dir); +install_dir(importance) -> ?IMPORTANCE_LOW; +install_dir(_) -> undefined. check_interval(type) -> emqx_schema:duration(); check_interval(default) -> <<"5s">>; -check_interval(T) when T =/= desc -> undefined; -check_interval(desc) -> ?DESC(check_interval). +check_interval(desc) -> ?DESC(check_interval); +check_interval(deprecated) -> {since, "5.0.24"}; +check_interval(_) -> undefined. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 133138252..932ebc5ed 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.14"}, + {vsn, "5.0.15"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, diff --git a/changes/ce/feat-10458.en.md b/changes/ce/feat-10458.en.md new file mode 100644 index 000000000..655885145 --- /dev/null +++ b/changes/ce/feat-10458.en.md @@ -0,0 +1,3 @@ +Set the level of plugin configuration options to low level, +in most cases, users only need to manage plugins on the dashboard +without the need for manual modification, so we lowered the level. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 7ebc320e5..ced7ae86a 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.10"}, + {vsn, "0.1.11"}, {registered, []}, {applications, [ kernel, From 4464a31fabe99846df88f60d8cd307fc26a70018 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 20 Apr 2023 18:01:27 +0800 Subject: [PATCH 262/279] chore: remove *_collector for prometheus api's example --- apps/emqx_prometheus/src/emqx_prometheus.app.src | 2 +- apps/emqx_prometheus/src/emqx_prometheus_api.erl | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index ae879da8f..f94b22d81 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.9"}, + {vsn, "5.0.10"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index 945c6eba9..d3bfc0224 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -122,13 +122,7 @@ prometheus_config_example() -> interval => "15s", push_gateway_server => <<"http://127.0.0.1:9091">>, headers => #{'header-name' => 'header-value'}, - job_name => <<"${name}/instance/${name}~${host}">>, - vm_dist_collector => enabled, - mnesia_collector => enabled, - vm_statistics_collector => enabled, - vm_system_info_collector => enabled, - vm_memory_collector => enabled, - vm_msacc_collector => enabled + job_name => <<"${name}/instance/${name}~${host}">> }. prometheus_data_schema() -> From 21473e7ca59aa60eef38b7ef99f371ffff591c3e Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 20 Apr 2023 18:16:04 +0800 Subject: [PATCH 263/279] fix(dynamo): fix field name errors --- lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl index f132438b6..ebb86f577 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl @@ -107,7 +107,7 @@ on_start( Templates = parse_template(Config), State = #{ - poolname => InstanceId, + pool_name => InstanceId, table => Table, templates => Templates }, @@ -176,7 +176,7 @@ do_query( InstanceId, Query, ApplyMode, - #{poolname := PoolName, templates := Templates, table := Table} = State + #{pool_name := PoolName, templates := Templates, table := Table} = State ) -> ?TRACE( "QUERY", From a55017ffaae40fda1eefb653b134e9d4e110f439 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 19 Apr 2023 10:39:42 -0300 Subject: [PATCH 264/279] ci: check if Elixir files are formatted in pre-commit hook --- scripts/git-hook-pre-commit.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/git-hook-pre-commit.sh b/scripts/git-hook-pre-commit.sh index b0e0699b9..cebd92b22 100755 --- a/scripts/git-hook-pre-commit.sh +++ b/scripts/git-hook-pre-commit.sh @@ -4,6 +4,9 @@ set -euo pipefail OPT="${1:--c}" +# mix format check is quite fast +mix format --check-formatted + files_dirty="$(git diff --name-only | grep -E '.*\.erl' || true)" files_cached="$(git diff --cached --name-only | grep -E '.*\.erl' || true)" if [[ "${files_dirty}" == '' ]] && [[ "${files_cached}" == '' ]]; then From b5eda9f0d1112d9343fcc85f01b4aa059beaddc3 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Fri, 14 Apr 2023 19:09:07 +0300 Subject: [PATCH 265/279] perf(emqx_resource): don't reactivate alarms on reoccurring errors Avoid unnecessary calls to activate an alarm if it has been already activated. Fixes: EMQX-9529/#10357 --- .../src/emqx_resource_manager.erl | 47 ++++++++++++------- .../test/emqx_connector_demo.erl | 2 + .../test/emqx_resource_SUITE.erl | 37 ++++++++++++++- 3 files changed, 68 insertions(+), 18 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 7ecf56c18..b35bade77 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -511,10 +511,10 @@ start_resource(Data, From) -> id => Data#data.id, reason => Reason }), - _ = maybe_alarm(disconnected, Data#data.id, Data#data.error), + _ = maybe_alarm(disconnected, Data#data.id, Err, Data#data.error), %% Keep track of the error reason why the connection did not work %% so that the Reason can be returned when the verification call is made. - UpdatedData = Data#data{status = disconnected, error = Reason}, + UpdatedData = Data#data{status = disconnected, error = Err}, Actions = maybe_reply(retry_actions(UpdatedData), From, Err), {next_state, disconnected, update_state(UpdatedData, Data), Actions} end. @@ -582,11 +582,11 @@ handle_connected_health_check(Data) -> with_health_check(#data{state = undefined} = Data, Func) -> Func(disconnected, Data); -with_health_check(Data, Func) -> +with_health_check(#data{error = PrevError} = Data, Func) -> ResId = Data#data.id, HCRes = emqx_resource:call_health_check(Data#data.manager_id, Data#data.mod, Data#data.state), {Status, NewState, Err} = parse_health_check_result(HCRes, Data), - _ = maybe_alarm(Status, ResId, Err), + _ = maybe_alarm(Status, ResId, Err, PrevError), ok = maybe_resume_resource_workers(ResId, Status), UpdatedData = Data#data{ state = NewState, status = Status, error = Err @@ -605,21 +605,25 @@ update_state(Data, _DataWas) -> health_check_interval(Opts) -> maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL). -maybe_alarm(connected, _ResId, _Error) -> +maybe_alarm(connected, _ResId, _Error, _PrevError) -> ok; -maybe_alarm(_Status, <>, _Error) -> +maybe_alarm(_Status, <>, _Error, _PrevError) -> ok; -maybe_alarm(_Status, ResId, Error) -> +%% Assume that alarm is already active +maybe_alarm(_Status, _ResId, Error, Error) -> + ok; +maybe_alarm(_Status, ResId, Error, _PrevError) -> HrError = case Error of - undefined -> <<"Unknown reason">>; - _Else -> emqx_utils:readable_error_msg(Error) + {error, undefined} -> <<"Unknown reason">>; + {error, Reason} -> emqx_utils:readable_error_msg(Reason) end, emqx_alarm:activate( ResId, #{resource_id => ResId, reason => resource_down}, <<"resource down: ", HrError/binary>> - ). + ), + ?tp(resource_activate_alarm, #{resource_id => ResId}). maybe_resume_resource_workers(ResId, connected) -> lists:foreach( @@ -635,11 +639,11 @@ maybe_clear_alarm(ResId) -> emqx_alarm:deactivate(ResId). parse_health_check_result(Status, Data) when ?IS_STATUS(Status) -> - {Status, Data#data.state, undefined}; + {Status, Data#data.state, status_to_error(Status)}; parse_health_check_result({Status, NewState}, _Data) when ?IS_STATUS(Status) -> - {Status, NewState, undefined}; + {Status, NewState, status_to_error(Status)}; parse_health_check_result({Status, NewState, Error}, _Data) when ?IS_STATUS(Status) -> - {Status, NewState, Error}; + {Status, NewState, {error, Error}}; parse_health_check_result({error, Error}, Data) -> ?SLOG( error, @@ -649,7 +653,16 @@ parse_health_check_result({error, Error}, Data) -> reason => Error } ), - {disconnected, Data#data.state, Error}. + {disconnected, Data#data.state, {error, Error}}. + +status_to_error(connected) -> + undefined; +status_to_error(_) -> + {error, undefined}. + +%% Compatibility +external_error({error, Reason}) -> Reason; +external_error(Other) -> Other. maybe_reply(Actions, undefined, _Reply) -> Actions; @@ -660,7 +673,7 @@ maybe_reply(Actions, From, Reply) -> data_record_to_external_map(Data) -> #{ id => Data#data.id, - error => Data#data.error, + error => external_error(Data#data.error), mod => Data#data.mod, callback_mode => Data#data.callback_mode, query_mode => Data#data.query_mode, @@ -679,8 +692,8 @@ do_wait_for_ready(ResId, Retry) -> case read_cache(ResId) of {_Group, #data{status = connected}} -> ok; - {_Group, #data{status = disconnected, error = Reason}} -> - {error, Reason}; + {_Group, #data{status = disconnected, error = Err}} -> + {error, external_error(Err)}; _ -> timer:sleep(?WAIT_FOR_RESOURCE_DELAY), do_wait_for_ready(ResId, Retry - 1) diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index 5be854e93..96e22c6b6 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -62,6 +62,7 @@ set_callback_mode(Mode) -> persistent_term:put(?CM_KEY, Mode). on_start(_InstId, #{create_error := true}) -> + ?tp(connector_demo_start_error, #{}), error("some error"); on_start(InstId, #{name := Name} = Opts) -> Register = maps:get(register, Opts, false), @@ -243,6 +244,7 @@ batch_big_payload({async, ReplyFunAndArgs}, InstId, Batch, State = #{pid := Pid} {ok, Pid}. on_get_status(_InstId, #{health_check_error := true}) -> + ?tp(connector_demo_health_check_error, #{}), disconnected; on_get_status(_InstId, #{pid := Pid}) -> timer:sleep(300), diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index e098c2e1c..f8ddd56b5 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -2635,7 +2635,6 @@ t_call_mode_uncoupled_from_query_mode(_Config) -> Trace2 ) ), - ok end ). @@ -2796,6 +2795,42 @@ t_late_call_reply(_Config) -> ), ok. +t_resource_create_error_activate_alarm_once(_) -> + do_t_resource_activate_alarm_once( + #{name => test_resource, create_error => true}, + connector_demo_start_error + ). + +t_resource_health_check_error_activate_alarm_once(_) -> + do_t_resource_activate_alarm_once( + #{name => test_resource, health_check_error => true}, + connector_demo_health_check_error + ). + +do_t_resource_activate_alarm_once(ResourceConfig, SubscribeEvent) -> + ?check_trace( + begin + ?wait_async_action( + emqx_resource:create_local( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + ResourceConfig, + #{auto_restart_interval => 100, health_check_interval => 100} + ), + #{?snk_kind := resource_activate_alarm, resource_id := ?ID} + ), + ?assertMatch([#{activated := true, name := ?ID}], emqx_alarm:get_alarms(activated)), + {ok, SubRef} = snabbkaffe:subscribe( + ?match_event(#{?snk_kind := SubscribeEvent}), 4, 7000 + ), + ?assertMatch({ok, [_, _, _, _]}, snabbkaffe:receive_events(SubRef)) + end, + fun(Trace) -> + ?assertMatch([_], ?of_kind(resource_activate_alarm, Trace)) + end + ). + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ From 662206fd33fbbf3e7114980430b36db8994f51cf Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 20 Apr 2023 10:53:12 -0300 Subject: [PATCH 266/279] ci: simplify find-apps.sh for ee apps --- scripts/find-apps.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index eed2acdb8..bfb6ba2cc 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -72,14 +72,12 @@ describe_app() { runner="docker" fi case "${app}" in - apps/emqx_bridge_kafka) - profile='emqx-enterprise' - ;; - apps/emqx_bridge_gcp_pubsub) - profile='emqx-enterprise' - ;; apps/*) - profile='emqx' + if [[ -f "${app}/BSL.txt" ]]; then + profile='emqx-enterprise' + else + profile='emqx' + fi ;; lib-ee/*) profile='emqx-enterprise' From b960d2ecb39100b17141a5409e7ef6c35cc674be Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Wed, 19 Apr 2023 20:42:14 +0300 Subject: [PATCH 267/279] perf(emqx_alarm): use dirty Mnesia operations to activate an alarm Alarms are stored in a local content shard and all 'activate' operations are serialized as they are called by one process ('emqx_alarm' gen_server), so using dirty operations gives performance gain without sacrificing consistency. Fixes: EMQX-9529/#10357 --- apps/emqx/src/emqx_alarm.erl | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 6aa3cb95d..7255e96da 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -57,7 +57,6 @@ %% Internal exports (RPC) -export([ - create_activate_alarm/3, do_get_alarms/0 ]). @@ -218,17 +217,12 @@ init([]) -> {ok, #{}, get_validity_period()}. handle_call({activate_alarm, Name, Details, Message}, _From, State) -> - Res = mria:transaction( - mria:local_content_shard(), - fun ?MODULE:create_activate_alarm/3, - [Name, Details, Message] - ), - case Res of - {atomic, Alarm} -> + case create_activate_alarm(Name, Details, Message) of + {ok, Alarm} -> do_actions(activate, Alarm, emqx:get_config([alarm, actions])), {reply, ok, State, get_validity_period()}; - {aborted, Reason} -> - {reply, Reason, State, get_validity_period()} + Err -> + {reply, Err, State, get_validity_period()} end; handle_call({deactivate_alarm, Name, Details, Message}, _From, State) -> case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of @@ -283,9 +277,9 @@ get_validity_period() -> emqx:get_config([alarm, validity_period]). create_activate_alarm(Name, Details, Message) -> - case mnesia:read(?ACTIVATED_ALARM, Name) of + case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of [#activated_alarm{name = Name}] -> - mnesia:abort({error, already_existed}); + {error, already_existed}; [] -> Alarm = #activated_alarm{ name = Name, @@ -293,8 +287,8 @@ create_activate_alarm(Name, Details, Message) -> message = normalize_message(Name, iolist_to_binary(Message)), activate_at = erlang:system_time(microsecond) }, - ok = mnesia:write(?ACTIVATED_ALARM, Alarm, write), - Alarm + ok = mria:dirty_write(?ACTIVATED_ALARM, Alarm), + {ok, Alarm} end. do_get_alarms() -> From 423a30fbb3392538807b1289a401e0f4b7b2ded2 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Wed, 19 Apr 2023 21:35:56 +0300 Subject: [PATCH 268/279] fix(emqx_alarm): add safe call API to activate/deactivate alarms and use it in resource_manager Don't let 'emqx_resource_manager' crash because of emqx_alarm timeouts. Fixes: EMQX-9529/#10357 --- apps/emqx/src/emqx_alarm.erl | 26 ++++++++++++++++++- .../src/emqx_resource_manager.erl | 6 ++--- changes/ce/fix-10407.en.md | 7 +++++ 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 changes/ce/fix-10407.en.md diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 7255e96da..056f36050 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -42,7 +42,9 @@ get_alarms/0, get_alarms/1, format/1, - format/2 + format/2, + safe_activate/3, + safe_deactivate/1 ]). %% gen_server callbacks @@ -122,6 +124,9 @@ activate(Name, Details) -> activate(Name, Details, Message) -> gen_server:call(?MODULE, {activate_alarm, Name, Details, Message}). +safe_activate(Name, Details, Message) -> + safe_call({activate_alarm, Name, Details, Message}). + -spec ensure_deactivated(binary() | atom()) -> ok. ensure_deactivated(Name) -> ensure_deactivated(Name, no_details). @@ -154,6 +159,9 @@ deactivate(Name, Details) -> deactivate(Name, Details, Message) -> gen_server:call(?MODULE, {deactivate_alarm, Name, Details, Message}). +safe_deactivate(Name) -> + safe_call({deactivate_alarm, Name, no_details, <<"">>}). + -spec delete_all_deactivated_alarms() -> ok. delete_all_deactivated_alarms() -> gen_server:call(?MODULE, delete_all_deactivated_alarms). @@ -468,3 +476,19 @@ normalize_message(Name, <<"">>) -> list_to_binary(io_lib:format("~p", [Name])); normalize_message(_Name, Message) -> Message. + +safe_call(Req) -> + try + gen_server:call(?MODULE, Req) + catch + _:{timeout, _} = Reason -> + ?SLOG(warning, #{msg => "emqx_alarm_safe_call_timeout", reason => Reason}), + {error, timeout}; + _:Reason:St -> + ?SLOG(error, #{ + msg => "emqx_alarm_safe_call_exception", + reason => Reason, + stacktrace => St + }), + {error, Reason} + end. diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index b35bade77..877b35fff 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -375,7 +375,7 @@ handle_event(state_timeout, health_check, connecting, Data) -> %% and successful health_checks handle_event(enter, _OldState, connected = State, Data) -> ok = log_state_consistency(State, Data), - _ = emqx_alarm:deactivate(Data#data.id), + _ = emqx_alarm:safe_deactivate(Data#data.id), ?tp(resource_connected_enter, #{}), {keep_state_and_data, health_check_actions(Data)}; handle_event(state_timeout, health_check, connected, Data) -> @@ -618,7 +618,7 @@ maybe_alarm(_Status, ResId, Error, _PrevError) -> {error, undefined} -> <<"Unknown reason">>; {error, Reason} -> emqx_utils:readable_error_msg(Reason) end, - emqx_alarm:activate( + emqx_alarm:safe_activate( ResId, #{resource_id => ResId, reason => resource_down}, <<"resource down: ", HrError/binary>> @@ -636,7 +636,7 @@ maybe_resume_resource_workers(_, _) -> maybe_clear_alarm(<>) -> ok; maybe_clear_alarm(ResId) -> - emqx_alarm:deactivate(ResId). + emqx_alarm:safe_deactivate(ResId). parse_health_check_result(Status, Data) when ?IS_STATUS(Status) -> {Status, Data#data.state, status_to_error(Status)}; diff --git a/changes/ce/fix-10407.en.md b/changes/ce/fix-10407.en.md new file mode 100644 index 000000000..d9df9ce69 --- /dev/null +++ b/changes/ce/fix-10407.en.md @@ -0,0 +1,7 @@ +Improve 'emqx_alarm' performance by using Mnesia dirty operations and avoiding +unnecessary calls from 'emqx_resource_manager' to reactivate alarms that have been already activated. +Use new safe 'emqx_alarm' API to activate/deactivate alarms to ensure that emqx_resource_manager +doesn't crash because of alarm timeouts. +The crashes were possible when the following conditions co-occurred: + - a relatively high number of failing resources, e.g. bridges tried to activate alarms on re-occurring errors; + - the system experienced a very high load. From 45254d7d85c9dd2c0f75052b5216563718c73d96 Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Thu, 20 Apr 2023 16:30:43 +0300 Subject: [PATCH 269/279] fix(emqx_bridge): validate Webhook bad URL and return 'BAD_REQUEST' if it's invalid Fixes: EMQX-9310 --- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge_api.erl | 4 +- apps/emqx_bridge/src/emqx_bridge_resource.erl | 47 ++++++++++------ .../test/emqx_bridge_api_SUITE.erl | 56 +++++++++++++++++++ changes/ce/fix-10463.en.md | 2 + 5 files changed, 93 insertions(+), 18 deletions(-) create mode 100644 changes/ce/fix-10463.en.md diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 04f2b58a7..d6c140fef 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.16"}, + {vsn, "0.1.17"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index c87ea8ea6..09d1159bd 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -64,7 +64,7 @@ {BridgeType, BridgeName} -> EXPR catch - throw:{invalid_bridge_id, Reason} -> + throw:#{reason := Reason} -> ?NOT_FOUND(<<"Invalid bridge ID, ", Reason/binary>>) end ). @@ -546,6 +546,8 @@ schema("/bridges_probe") -> case emqx_bridge_resource:create_dry_run(ConnType, maps:remove(<<"type">>, Params1)) of ok -> ?NO_CONTENT; + {error, #{kind := validation_error} = Reason} -> + ?BAD_REQUEST('TEST_FAILED', map_to_json(Reason)); {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' -> ?BAD_REQUEST('TEST_FAILED', Reason) end; diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 347f9d973..1ad024c40 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -87,7 +87,7 @@ parse_bridge_id(BridgeId) -> [Type, Name] -> {to_type_atom(Type), validate_name(Name)}; _ -> - invalid_bridge_id( + invalid_data( <<"should be of pattern {type}:{name}, but got ", BridgeId/binary>> ) end. @@ -108,14 +108,14 @@ validate_name(Name0) -> true -> Name0; false -> - invalid_bridge_id(<<"bad name: ", Name0/binary>>) + invalid_data(<<"bad name: ", Name0/binary>>) end; false -> - invalid_bridge_id(<<"only 0-9a-zA-Z_-. is allowed in name: ", Name0/binary>>) + invalid_data(<<"only 0-9a-zA-Z_-. is allowed in name: ", Name0/binary>>) end. --spec invalid_bridge_id(binary()) -> no_return(). -invalid_bridge_id(Reason) -> throw({?FUNCTION_NAME, Reason}). +-spec invalid_data(binary()) -> no_return(). +invalid_data(Reason) -> throw(#{kind => validation_error, reason => Reason}). is_id_char(C) when C >= $0 andalso C =< $9 -> true; is_id_char(C) when C >= $a andalso C =< $z -> true; @@ -130,7 +130,7 @@ to_type_atom(Type) -> erlang:binary_to_existing_atom(Type, utf8) catch _:_ -> - invalid_bridge_id(<<"unknown type: ", Type/binary>>) + invalid_data(<<"unknown bridge type: ", Type/binary>>) end. reset_metrics(ResourceId) -> @@ -243,12 +243,19 @@ create_dry_run(Type, Conf0) -> {error, Reason} -> {error, Reason}; {ok, ConfNew} -> - ParseConf = parse_confs(bin(Type), TmpPath, ConfNew), - Res = emqx_resource:create_dry_run_local( - bridge_to_resource_type(Type), ParseConf - ), - _ = maybe_clear_certs(TmpPath, ConfNew), - Res + try + ParseConf = parse_confs(bin(Type), TmpPath, ConfNew), + Res = emqx_resource:create_dry_run_local( + bridge_to_resource_type(Type), ParseConf + ), + Res + catch + %% validation errors + throw:Reason -> + {error, Reason} + after + _ = maybe_clear_certs(TmpPath, ConfNew) + end end. remove(BridgeId) -> @@ -300,10 +307,18 @@ parse_confs( max_retries := Retry } = Conf ) -> - {BaseUrl, Path} = parse_url(Url), - {ok, BaseUrl2} = emqx_http_lib:uri_parse(BaseUrl), + Url1 = bin(Url), + {BaseUrl, Path} = parse_url(Url1), + BaseUrl1 = + case emqx_http_lib:uri_parse(BaseUrl) of + {ok, BUrl} -> + BUrl; + {error, Reason} -> + Reason1 = emqx_utils:readable_error_msg(Reason), + invalid_data(<<"Invalid URL: ", Url1/binary, ", details: ", Reason1/binary>>) + end, Conf#{ - base_url => BaseUrl2, + base_url => BaseUrl1, request => #{ path => Path, @@ -338,7 +353,7 @@ parse_url(Url) -> {iolist_to_binary([Scheme, "//", HostPort]), <<>>} end; [Url] -> - error({invalid_url, Url}) + invalid_data(<<"Missing scheme in URL: ", Url/binary>>) end. str(Bin) when is_binary(Bin) -> binary_to_list(Bin); diff --git a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl index 3afe17080..d55b92138 100644 --- a/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl @@ -414,6 +414,18 @@ t_http_crud_apis(Config) -> }, json(maps:get(<<"message">>, PutFail2)) ), + {ok, 400, _} = request_json( + put, + uri(["bridges", BridgeID]), + ?HTTP_BRIDGE(<<"localhost:1234/foo">>, Name), + Config + ), + {ok, 400, _} = request_json( + put, + uri(["bridges", BridgeID]), + ?HTTP_BRIDGE(<<"htpp://localhost:12341234/foo">>, Name), + Config + ), %% delete the bridge {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config), @@ -498,6 +510,22 @@ t_http_crud_apis(Config) -> %% Try create bridge with bad characters as name {ok, 400, _} = request(post, uri(["bridges"]), ?HTTP_BRIDGE(URL1, <<"隋达"/utf8>>), Config), + %% Missing scheme in URL + {ok, 400, _} = request( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(<<"localhost:1234/foo">>, <<"missing_url_scheme">>), + Config + ), + + %% Invalid port + {ok, 400, _} = request( + post, + uri(["bridges"]), + ?HTTP_BRIDGE(<<"http://localhost:12341234/foo">>, <<"invalid_port">>), + Config + ), + {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), Config). t_http_bridges_local_topic(Config) -> @@ -1016,6 +1044,34 @@ t_bridges_probe(Config) -> ) ), + %% Missing scheme in URL + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"TEST_FAILED">>, + <<"message">> := _ + }}, + request_json( + post, + uri(["bridges_probe"]), + ?HTTP_BRIDGE(<<"203.0.113.3:1234/foo">>), + Config + ) + ), + + %% Invalid port + ?assertMatch( + {ok, 400, #{ + <<"code">> := <<"TEST_FAILED">>, + <<"message">> := _ + }}, + request_json( + post, + uri(["bridges_probe"]), + ?HTTP_BRIDGE(<<"http://203.0.113.3:12341234/foo">>), + Config + ) + ), + {ok, 204, _} = request( post, uri(["bridges_probe"]), diff --git a/changes/ce/fix-10463.en.md b/changes/ce/fix-10463.en.md new file mode 100644 index 000000000..9d57bc1b0 --- /dev/null +++ b/changes/ce/fix-10463.en.md @@ -0,0 +1,2 @@ +Improve bridges API error handling. +If Webhook bridge URL is not valid, bridges API will return '400' error instead of '500'. From 5a7685a341b6466e9c1e18c84aa8b70df67004c5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 20 Apr 2023 22:26:29 +0800 Subject: [PATCH 270/279] chore: apply suggestions from code review Co-authored-by: ieQu1 <99872536+ieQu1@users.noreply.github.com> Co-authored-by: Thales Macedo Garitezi --- apps/emqx_modules/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_modules/README.md b/apps/emqx_modules/README.md index 567b1ac71..dfa349514 100644 --- a/apps/emqx_modules/README.md +++ b/apps/emqx_modules/README.md @@ -1,6 +1,6 @@ # EMQX Modules -The application provide some minor functional modules that are not included in the MQTT +The application provides some minor functional modules that are not included in the MQTT protocol standard, including "Delayed Publish", "Topic Rewrite", "Topic Metrics" and "Telemetry". @@ -20,7 +20,7 @@ See [Enabling/Disabling Delayed Publish via HTTP API](https://www.emqx.io/docs/e Topic Rewrite allows users to configure rules to change the topic strings that the client requests to subscribe or publish. -This feature is very useful when it need to compatibility with different versions of topic designs. +This feature is very useful when there's a need to transform between different topic structures. For example, an old device that has already been issued and cannot be upgraded may use old topic designs, but for some reason, we adjusted the format of topics. We can use this feature to rewrite the old topics as the new format to eliminate these differences. From 8e8ba6ce7e94e84ae7b3e6f3e9a5aee0d3c601db Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 19 Apr 2023 16:42:54 +0800 Subject: [PATCH 271/279] fix: always check authn_http's header and ssl_option --- apps/emqx_authn/src/emqx_authn.app.src | 2 +- apps/emqx_authn/src/emqx_authn_schema.erl | 26 +++++ .../src/simple_authn/emqx_authn_http.erl | 52 ++++++--- .../test/emqx_authn_https_SUITE.erl | 17 +++ .../emqx_conf/test/emqx_conf_schema_tests.erl | 100 +++++++++++++++--- 5 files changed, 163 insertions(+), 34 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 063771e24..c1d48909c 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.17"}, + {vsn, "0.1.18"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index 112ea2076..a7cdaac5f 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -18,10 +18,12 @@ -elvis([{elvis_style, invalid_dynamic_call, disable}]). -include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_authn.hrl"). -export([ common_fields/0, roots/0, + validations/0, tags/0, fields/1, authenticator_type/0, @@ -207,3 +209,27 @@ array(Name) -> array(Name, DescId) -> {Name, ?HOCON(?R_REF(Name), #{desc => ?DESC(DescId)})}. + +validations() -> + [ + {check_http_ssl_opts, fun(Conf) -> + CheckFun = fun emqx_authn_http:check_ssl_opts/1, + validation(Conf, CheckFun) + end}, + {check_http_headers, fun(Conf) -> + CheckFun = fun emqx_authn_http:check_headers/1, + validation(Conf, CheckFun) + end} + ]. + +validation(Conf, CheckFun) when is_map(Conf) -> + validation(hocon_maps:get(?CONF_NS, Conf), CheckFun); +validation(undefined, _) -> + ok; +validation([], _) -> + ok; +validation([AuthN | Tail], CheckFun) -> + case CheckFun(#{?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY => AuthN}) of + ok -> validation(Tail, CheckFun); + Error -> Error + end. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 33587a2db..33bd07efc 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -38,6 +38,8 @@ headers/1 ]). +-export([check_headers/1, check_ssl_opts/1]). + -export([ refs/0, union_member_selector/1, @@ -107,8 +109,8 @@ common_fields() -> validations() -> [ - {check_ssl_opts, fun check_ssl_opts/1}, - {check_headers, fun check_headers/1} + {check_ssl_opts, fun ?MODULE:check_ssl_opts/1}, + {check_headers, fun ?MODULE:check_headers/1} ]. url(type) -> binary(); @@ -262,21 +264,39 @@ transform_header_name(Headers) -> ). check_ssl_opts(Conf) -> - {BaseUrl, _Path, _Query} = parse_url(get_conf_val("url", Conf)), - case BaseUrl of - <<"https://", _/binary>> -> - case get_conf_val("ssl.enable", Conf) of - true -> ok; - false -> false - end; - <<"http://", _/binary>> -> - ok + case get_conf_val("url", Conf) of + undefined -> + ok; + Url -> + {BaseUrl, _Path, _Query} = parse_url(Url), + case BaseUrl of + <<"https://", _/binary>> -> + case get_conf_val("ssl.enable", Conf) of + true -> + ok; + false -> + <<"it's required to enable the TLS option to establish a https connection">> + end; + <<"http://", _/binary>> -> + ok + end end. check_headers(Conf) -> - Method = to_bin(get_conf_val("method", Conf)), - Headers = get_conf_val("headers", Conf), - Method =:= <<"post">> orelse (not maps:is_key(<<"content-type">>, Headers)). + case get_conf_val("headers", Conf) of + undefined -> + ok; + Headers -> + case to_bin(get_conf_val("method", Conf)) of + <<"post">> -> + ok; + <<"get">> -> + case maps:is_key(<<"content-type">>, Headers) of + false -> ok; + true -> <<"HTTP GET requests cannot include content-type header.">> + end + end + end. parse_url(Url) -> case string:split(Url, "//", leading) of @@ -311,7 +331,7 @@ parse_config( method => Method, path => Path, headers => ensure_header_name_type(Headers), - base_path_templete => emqx_authn_utils:parse_str(Path), + base_path_template => emqx_authn_utils:parse_str(Path), base_query_template => emqx_authn_utils:parse_deep( cow_qs:parse_qs(to_bin(Query)) ), @@ -324,7 +344,7 @@ parse_config( generate_request(Credential, #{ method := Method, headers := Headers0, - base_path_templete := BasePathTemplate, + base_path_template := BasePathTemplate, base_query_template := BaseQueryTemplate, body_template := BodyTemplate }) -> diff --git a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl index c4315b69f..f23a160d1 100644 --- a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl @@ -114,6 +114,22 @@ t_create_invalid_version(_Config) -> emqx_access_control:authenticate(?CREDENTIALS) ). +t_create_disable_ssl_opts_when_https(_Config) -> + {ok, _} = create_https_auth_with_ssl_opts( + #{ + <<"server_name_indication">> => <<"authn-server">>, + <<"verify">> => <<"verify_peer">>, + <<"versions">> => [<<"tlsv1.2">>], + <<"ciphers">> => [<<"ECDHE-RSA-AES256-GCM-SHA384">>], + <<"enable">> => <<"false">> + } + ), + + ?assertEqual( + {error, not_authorized}, + emqx_access_control:authenticate(?CREDENTIALS) + ). + t_create_invalid_ciphers(_Config) -> {ok, _} = create_https_auth_with_ssl_opts( #{ @@ -135,6 +151,7 @@ t_create_invalid_ciphers(_Config) -> create_https_auth_with_ssl_opts(SpecificSSLOpts) -> AuthConfig = raw_https_auth_config(SpecificSSLOpts), + ct:pal("111:~p~n", [AuthConfig]), emqx:update_config(?PATH, {create_authenticator, ?GLOBAL, AuthConfig}). raw_https_auth_config(SpecificSSLOpts) -> diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 453aca7a8..79307c803 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -5,27 +5,27 @@ -module(emqx_conf_schema_tests). -include_lib("eunit/include/eunit.hrl"). +-define(BASE_CONF, + "" + "\n" + " node {\n" + " name = \"emqx1@127.0.0.1\"\n" + " cookie = \"emqxsecretcookie\"\n" + " data_dir = \"data\"\n" + " }\n" + " cluster {\n" + " name = emqxcl\n" + " discovery_strategy = static\n" + " static.seeds = ~p\n" + " core_nodes = ~p\n" + " }\n" + "" +). array_nodes_test() -> ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], - BaseConf = - "" - "\n" - " node {\n" - " name = \"emqx1@127.0.0.1\"\n" - " cookie = \"emqxsecretcookie\"\n" - " data_dir = \"data\"\n" - " }\n" - " cluster {\n" - " name = emqxcl\n" - " discovery_strategy = static\n" - " static.seeds = ~p\n" - " core_nodes = ~p\n" - " }\n" - " " - "", lists:foreach( fun(Nodes) -> - ConfFile = iolist_to_binary(io_lib:format(BaseConf, [Nodes, Nodes])), + ConfFile = iolist_to_binary(io_lib:format(?BASE_CONF, [Nodes, Nodes])), {ok, Conf} = hocon:binary(ConfFile, #{format => richmap}), ConfList = hocon_tconf:generate(emqx_conf_schema, Conf), ClusterDiscovery = proplists:get_value( @@ -46,6 +46,72 @@ array_nodes_test() -> ), ok. +authn_validations_test() -> + BaseConf = iolist_to_binary(io_lib:format(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"])), + DisableSSLWithHttps = + "" + "\n" + "authentication = [\n" + "{backend = \"http\"\n" + "body {password = \"${password}\", username = \"${username}\"}\n" + "connect_timeout = \"15s\"\n" + "enable_pipelining = 100\n" + "headers {\"content-type\" = \"application/json\"}\n" + "mechanism = \"password_based\"\n" + "method = \"post\"\n" + "pool_size = 8\n" + "request_timeout = \"5s\"\n" + "ssl {enable = false, verify = \"verify_peer\"}\n" + "url = \"https://127.0.0.1:8080\"\n" + "}\n" + "]\n" + "", + Conf = <>, + {ok, ConfMap} = hocon:binary(Conf, #{format => richmap}), + ?assertThrow( + {emqx_conf_schema, [ + #{ + kind := validation_error, + reason := integrity_validation_failure, + result := _, + validation_name := check_http_ssl_opts + } + ]}, + hocon_tconf:generate(emqx_conf_schema, ConfMap) + ), + BadHeader = + "" + "\n" + "authentication = [\n" + "{backend = \"http\"\n" + "body {password = \"${password}\", username = \"${username}\"}\n" + "connect_timeout = \"15s\"\n" + "enable_pipelining = 100\n" + "headers {\"content-type\" = \"application/json\"}\n" + "mechanism = \"password_based\"\n" + "method = \"get\"\n" + "pool_size = 8\n" + "request_timeout = \"5s\"\n" + "ssl {enable = false, verify = \"verify_peer\"}\n" + "url = \"http://127.0.0.1:8080\"\n" + "}\n" + "]\n" + "", + Conf1 = <>, + {ok, ConfMap1} = hocon:binary(Conf1, #{format => richmap}), + ?assertThrow( + {emqx_conf_schema, [ + #{ + kind := validation_error, + reason := integrity_validation_failure, + result := _, + validation_name := check_http_headers + } + ]}, + hocon_tconf:generate(emqx_conf_schema, ConfMap1) + ), + ok. + doc_gen_test() -> %% the json file too large to encode. { From 2aef9ca21544e70f7a4917e3769bebd44d12ebf7 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Wed, 19 Apr 2023 17:10:07 +0800 Subject: [PATCH 272/279] chore: add changlog for authn_http validation --- apps/emqx_authn/test/emqx_authn_https_SUITE.erl | 17 ----------------- changes/ce/fix-10449.en.md | 2 ++ 2 files changed, 2 insertions(+), 17 deletions(-) create mode 100644 changes/ce/fix-10449.en.md diff --git a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl index f23a160d1..c4315b69f 100644 --- a/apps/emqx_authn/test/emqx_authn_https_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_https_SUITE.erl @@ -114,22 +114,6 @@ t_create_invalid_version(_Config) -> emqx_access_control:authenticate(?CREDENTIALS) ). -t_create_disable_ssl_opts_when_https(_Config) -> - {ok, _} = create_https_auth_with_ssl_opts( - #{ - <<"server_name_indication">> => <<"authn-server">>, - <<"verify">> => <<"verify_peer">>, - <<"versions">> => [<<"tlsv1.2">>], - <<"ciphers">> => [<<"ECDHE-RSA-AES256-GCM-SHA384">>], - <<"enable">> => <<"false">> - } - ), - - ?assertEqual( - {error, not_authorized}, - emqx_access_control:authenticate(?CREDENTIALS) - ). - t_create_invalid_ciphers(_Config) -> {ok, _} = create_https_auth_with_ssl_opts( #{ @@ -151,7 +135,6 @@ t_create_invalid_ciphers(_Config) -> create_https_auth_with_ssl_opts(SpecificSSLOpts) -> AuthConfig = raw_https_auth_config(SpecificSSLOpts), - ct:pal("111:~p~n", [AuthConfig]), emqx:update_config(?PATH, {create_authenticator, ?GLOBAL, AuthConfig}). raw_https_auth_config(SpecificSSLOpts) -> diff --git a/changes/ce/fix-10449.en.md b/changes/ce/fix-10449.en.md new file mode 100644 index 000000000..e10b52fb4 --- /dev/null +++ b/changes/ce/fix-10449.en.md @@ -0,0 +1,2 @@ +Validate the ssl_options and header configurations when creating authentication http (`authn_http`). +Prior to this, incorrect ssl_options configuration could result in successful creation but the entire authn being unusable. From 1e54d23d316bcf137ee63399018a15e2ed31982b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Thu, 20 Apr 2023 18:20:51 +0800 Subject: [PATCH 273/279] test: add a test for authn {} --- .../emqx_conf/test/emqx_conf_schema_tests.erl | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 79307c803..e3033e4a7 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -110,6 +110,37 @@ authn_validations_test() -> ]}, hocon_tconf:generate(emqx_conf_schema, ConfMap1) ), + BadHeader2 = + "" + "\n" + "authentication = \n" + "{backend = \"http\"\n" + "body {password = \"${password}\", username = \"${username}\"}\n" + "connect_timeout = \"15s\"\n" + "enable_pipelining = 100\n" + "headers {\"content-type\" = \"application/json\"}\n" + "mechanism = \"password_based\"\n" + "method = \"get\"\n" + "pool_size = 8\n" + "request_timeout = \"5s\"\n" + "ssl {enable = false, verify = \"verify_peer\"}\n" + "url = \"http://127.0.0.1:8080\"\n" + "}\n" + "\n" + "", + Conf2 = <>, + {ok, ConfMap2} = hocon:binary(Conf2, #{format => richmap}), + ?assertThrow( + {emqx_conf_schema, [ + #{ + kind := validation_error, + reason := integrity_validation_failure, + result := _, + validation_name := check_http_headers + } + ]}, + hocon_tconf:generate(emqx_conf_schema, ConfMap2) + ), ok. doc_gen_test() -> From ad6090a77881ae85b0f0b2a84c61f4f6019a6710 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 21 Apr 2023 11:21:05 +0800 Subject: [PATCH 274/279] chore: update changes/ce/fix-10449.en.md Co-authored-by: JianBo He --- changes/ce/fix-10449.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ce/fix-10449.en.md b/changes/ce/fix-10449.en.md index e10b52fb4..005dea73c 100644 --- a/changes/ce/fix-10449.en.md +++ b/changes/ce/fix-10449.en.md @@ -1,2 +1,2 @@ Validate the ssl_options and header configurations when creating authentication http (`authn_http`). -Prior to this, incorrect ssl_options configuration could result in successful creation but the entire authn being unusable. +Prior to this, incorrect `ssl` configuration could result in successful creation but the entire authn being unusable. From fdf9b2a3837fad852252df64f7fce1334c760147 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Fri, 21 Apr 2023 11:54:11 +0800 Subject: [PATCH 275/279] chore: apply review suggestions --- .../src/simple_authn/emqx_authn_http.erl | 28 ++- apps/emqx_authz/src/emqx_authz.app.src | 2 +- .../emqx_conf/test/emqx_conf_schema_tests.erl | 186 ++++++++---------- .../emqx_connector/src/emqx_connector.app.src | 2 +- apps/emqx_prometheus/TODO | 2 - 5 files changed, 101 insertions(+), 119 deletions(-) delete mode 100644 apps/emqx_prometheus/TODO diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index 33bd07efc..eddad92a3 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -264,10 +264,9 @@ transform_header_name(Headers) -> ). check_ssl_opts(Conf) -> - case get_conf_val("url", Conf) of - undefined -> - ok; - Url -> + case is_backend_http(Conf) of + true -> + Url = get_conf_val("url", Conf), {BaseUrl, _Path, _Query} = parse_url(Url), case BaseUrl of <<"https://", _/binary>> -> @@ -279,14 +278,15 @@ check_ssl_opts(Conf) -> end; <<"http://", _/binary>> -> ok - end + end; + false -> + ok end. check_headers(Conf) -> - case get_conf_val("headers", Conf) of - undefined -> - ok; - Headers -> + case is_backend_http(Conf) of + true -> + Headers = get_conf_val("headers", Conf), case to_bin(get_conf_val("method", Conf)) of <<"post">> -> ok; @@ -295,7 +295,15 @@ check_headers(Conf) -> false -> ok; true -> <<"HTTP GET requests cannot include content-type header.">> end - end + end; + false -> + ok + end. + +is_backend_http(Conf) -> + case get_conf_val("backend", Conf) of + http -> true; + _ -> false end. parse_url(Url) -> diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index a17ce5dea..dd658a6aa 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.17"}, + {vsn, "0.1.18"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index e3033e4a7..667d1766f 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -5,27 +5,28 @@ -module(emqx_conf_schema_tests). -include_lib("eunit/include/eunit.hrl"). + +%% erlfmt-ignore -define(BASE_CONF, - "" - "\n" - " node {\n" - " name = \"emqx1@127.0.0.1\"\n" - " cookie = \"emqxsecretcookie\"\n" - " data_dir = \"data\"\n" - " }\n" - " cluster {\n" - " name = emqxcl\n" - " discovery_strategy = static\n" - " static.seeds = ~p\n" - " core_nodes = ~p\n" - " }\n" - "" -). + """ + node { + name = \"emqx1@127.0.0.1\" + cookie = \"emqxsecretcookie\" + data_dir = \"data\" + } + cluster { + name = emqxcl + discovery_strategy = static + static.seeds = ~p + core_nodes = ~p + } + """). + array_nodes_test() -> ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], lists:foreach( fun(Nodes) -> - ConfFile = iolist_to_binary(io_lib:format(?BASE_CONF, [Nodes, Nodes])), + ConfFile = to_bin(?BASE_CONF, [Nodes, Nodes]), {ok, Conf} = hocon:binary(ConfFile, #{format => richmap}), ConfList = hocon_tconf:generate(emqx_conf_schema, Conf), ClusterDiscovery = proplists:get_value( @@ -46,101 +47,73 @@ array_nodes_test() -> ), ok. +%% erlfmt-ignore +-define(BASE_AUTHN_ARRAY, + """ + authentication = [ + {backend = \"http\" + body {password = \"${password}\", username = \"${username}\"} + connect_timeout = \"15s\" + enable_pipelining = 100 + headers {\"content-type\" = \"application/json\"} + mechanism = \"password_based\" + method = \"~p\" + pool_size = 8 + request_timeout = \"5s\" + ssl {enable = ~p, verify = \"verify_peer\"} + url = \"~ts\" + } + ] + """ +). + +-define(ERROR(Reason), + {emqx_conf_schema, [ + #{ + kind := validation_error, + reason := integrity_validation_failure, + result := _, + validation_name := Reason + } + ]} +). + authn_validations_test() -> - BaseConf = iolist_to_binary(io_lib:format(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"])), - DisableSSLWithHttps = - "" - "\n" - "authentication = [\n" - "{backend = \"http\"\n" - "body {password = \"${password}\", username = \"${username}\"}\n" - "connect_timeout = \"15s\"\n" - "enable_pipelining = 100\n" - "headers {\"content-type\" = \"application/json\"}\n" - "mechanism = \"password_based\"\n" - "method = \"post\"\n" - "pool_size = 8\n" - "request_timeout = \"5s\"\n" - "ssl {enable = false, verify = \"verify_peer\"}\n" - "url = \"https://127.0.0.1:8080\"\n" - "}\n" - "]\n" - "", - Conf = <>, - {ok, ConfMap} = hocon:binary(Conf, #{format => richmap}), - ?assertThrow( - {emqx_conf_schema, [ - #{ - kind := validation_error, - reason := integrity_validation_failure, - result := _, - validation_name := check_http_ssl_opts - } - ]}, - hocon_tconf:generate(emqx_conf_schema, ConfMap) - ), - BadHeader = - "" - "\n" - "authentication = [\n" - "{backend = \"http\"\n" - "body {password = \"${password}\", username = \"${username}\"}\n" - "connect_timeout = \"15s\"\n" - "enable_pipelining = 100\n" - "headers {\"content-type\" = \"application/json\"}\n" - "mechanism = \"password_based\"\n" - "method = \"get\"\n" - "pool_size = 8\n" - "request_timeout = \"5s\"\n" - "ssl {enable = false, verify = \"verify_peer\"}\n" - "url = \"http://127.0.0.1:8080\"\n" - "}\n" - "]\n" - "", - Conf1 = <>, + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + + OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), + Conf0 = <>, + {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), + ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap0))), + + OKHttp = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"http://127.0.0.1:8080">>]), + Conf1 = <>, {ok, ConfMap1} = hocon:binary(Conf1, #{format => richmap}), - ?assertThrow( - {emqx_conf_schema, [ - #{ - kind := validation_error, - reason := integrity_validation_failure, - result := _, - validation_name := check_http_headers - } - ]}, - hocon_tconf:generate(emqx_conf_schema, ConfMap1) - ), - BadHeader2 = - "" - "\n" - "authentication = \n" - "{backend = \"http\"\n" - "body {password = \"${password}\", username = \"${username}\"}\n" - "connect_timeout = \"15s\"\n" - "enable_pipelining = 100\n" - "headers {\"content-type\" = \"application/json\"}\n" - "mechanism = \"password_based\"\n" - "method = \"get\"\n" - "pool_size = 8\n" - "request_timeout = \"5s\"\n" - "ssl {enable = false, verify = \"verify_peer\"}\n" - "url = \"http://127.0.0.1:8080\"\n" - "}\n" - "\n" - "", - Conf2 = <>, + ?assert(is_list(hocon_tconf:generate(emqx_conf_schema, ConfMap1))), + + DisableSSLWithHttps = to_bin(?BASE_AUTHN_ARRAY, [post, false, <<"https://127.0.0.1:8080">>]), + Conf2 = <>, {ok, ConfMap2} = hocon:binary(Conf2, #{format => richmap}), ?assertThrow( - {emqx_conf_schema, [ - #{ - kind := validation_error, - reason := integrity_validation_failure, - result := _, - validation_name := check_http_headers - } - ]}, + ?ERROR(check_http_ssl_opts), hocon_tconf:generate(emqx_conf_schema, ConfMap2) ), + + BadHeader = to_bin(?BASE_AUTHN_ARRAY, [get, true, <<"https://127.0.0.1:8080">>]), + Conf3 = <>, + {ok, ConfMap3} = hocon:binary(Conf3, #{format => richmap}), + ?assertThrow( + ?ERROR(check_http_headers), + hocon_tconf:generate(emqx_conf_schema, ConfMap3) + ), + + BadHeaderWithTuple = binary:replace(BadHeader, [<<"[">>, <<"]">>], <<"">>, [global]), + Conf4 = <>, + {ok, ConfMap4} = hocon:binary(Conf4, #{format => richmap}), + ?assertThrow( + ?ERROR(check_http_headers), + hocon_tconf:generate(emqx_conf_schema, ConfMap4) + ), ok. doc_gen_test() -> @@ -163,3 +136,6 @@ doc_gen_test() -> ok end }. + +to_bin(Format, Args) -> + iolist_to_binary(io_lib:format(Format, Args)). diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 38ca230b2..c0a19824c 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.20"}, + {vsn, "0.1.21"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_prometheus/TODO b/apps/emqx_prometheus/TODO deleted file mode 100644 index a868fba7e..000000000 --- a/apps/emqx_prometheus/TODO +++ /dev/null @@ -1,2 +0,0 @@ -1. Add more VM Metrics -2. Add more emqx Metrics From bdce32e713f23234ecf650b80eb3ecaae46c585b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 19 Apr 2023 13:05:25 +0800 Subject: [PATCH 276/279] refactor(cassandra): move cassandra bridge into its own app --- apps/emqx_bridge_cassandra/docker-ct | 2 + apps/emqx_bridge_cassandra/rebar.config | 11 +++++ .../src/emqx_bridge_cassandra.app.src | 2 +- .../src/emqx_bridge_cassandra.erl | 4 +- .../src/emqx_bridge_cassandra_impl.erl | 2 +- .../test/emqx_bridge_cassandra_SUITE.erl | 6 +-- .../test/emqx_bridge_cassandra_impl_SUITE.erl | 4 +- lib-ee/emqx_ee_bridge/docker-ct | 1 - lib-ee/emqx_ee_bridge/rebar.config | 3 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 8 ++-- lib-ee/emqx_ee_connector/docker-ct | 1 - .../src/emqx_ee_connector.app.src | 1 - mix.exs | 13 ++++++ rebar.config.erl | 26 ++++++++++++ ...assa.hocon => emqx_bridge_cassandra.hocon} | 2 +- ...hocon => emqx_bridge_cassandra_impl.hocon} | 2 +- scripts/ct/run.sh | 42 +++++++++++++++++++ 17 files changed, 110 insertions(+), 20 deletions(-) create mode 100644 apps/emqx_bridge_cassandra/docker-ct create mode 100644 apps/emqx_bridge_cassandra/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl => apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl (97%) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl => apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl (99%) rename lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl => apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl (99%) rename lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl => apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl (98%) rename rel/i18n/{emqx_ee_bridge_cassa.hocon => emqx_bridge_cassandra.hocon} (97%) rename rel/i18n/{emqx_ee_connector_cassa.hocon => emqx_bridge_cassandra_impl.hocon} (92%) diff --git a/apps/emqx_bridge_cassandra/docker-ct b/apps/emqx_bridge_cassandra/docker-ct new file mode 100644 index 000000000..2626b4068 --- /dev/null +++ b/apps/emqx_bridge_cassandra/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +cassandra diff --git a/apps/emqx_bridge_cassandra/rebar.config b/apps/emqx_bridge_cassandra/rebar.config new file mode 100644 index 000000000..b8bfc7dd6 --- /dev/null +++ b/apps/emqx_bridge_cassandra/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.1"}}} + , {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. + +{shell, [ + {apps, [emqx_bridge_cassandra]} +]}. diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index 1ed65ea9f..58e4a1984 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -2,7 +2,7 @@ {description, "EMQX Enterprise Cassandra Bridge"}, {vsn, "0.1.0"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, ecql]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl similarity index 97% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl rename to apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl index 26c6de04d..32dc11839 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_cassa.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_cassa). +-module(emqx_bridge_cassandra). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -88,7 +88,7 @@ fields("config") -> #{desc => ?DESC("local_topic"), default => undefined} )} ] ++ emqx_resource_schema:fields("resource_opts") ++ - (emqx_ee_connector_cassa:fields(config) -- + (emqx_bridge_cassandra_impl:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); fields("post") -> fields("post", cassandra); diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl rename to apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl index 397532f47..8fe329a98 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_cassa.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_cassa). +-module(emqx_bridge_cassandra_impl). -behaviour(emqx_resource). diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl rename to apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index 4711d1981..482f2c1f2 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_cassa_SUITE). +-module(emqx_bridge_cassandra_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -57,7 +57,7 @@ %% CASSA_TCP_HOST=127.0.0.1 CASSA_TCP_PORT=19042 \ %% CASSA_TLS_HOST=127.0.0.1 CASSA_TLS_PORT=19142 \ %% PROXY_HOST=127.0.0.1 ./rebar3 as test ct -c -v --name ct@127.0.0.1 \ -%% --suite lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_cassa_SUITE.erl +%% --suite apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl %% %%------------------------------------------------------------------------------ @@ -590,7 +590,7 @@ t_missing_data(Config) -> {ok, _}, create_bridge(Config) ), - %% emqx_ee_connector_cassa will send missed data as a `null` atom + %% emqx_bridge_cassandra_impl will send missed data as a `null` atom %% to ecql driver ?check_trace( begin diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl similarity index 98% rename from lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl rename to apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl index f2647d756..faaee3c63 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_cassa_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_cassa_SUITE). +-module(emqx_bridge_cassandra_impl_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -16,7 +16,7 @@ %% Cassandra server defined at `.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml` %% You can change it to `127.0.0.1`, if you run this SUITE locally -define(CASSANDRA_HOST, "cassandra"). --define(CASSANDRA_RESOURCE_MOD, emqx_ee_connector_cassa). +-define(CASSANDRA_RESOURCE_MOD, emqx_bridge_cassandra_impl). %% This test SUITE requires a running cassandra instance. If you don't want to %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index 35d6b9d5b..aa19a495f 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -10,5 +10,4 @@ tdengine clickhouse dynamo rocketmq -cassandra sqlserver diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config index b26df658a..358ff3bc8 100644 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ b/lib-ee/emqx_ee_bridge/rebar.config @@ -1,6 +1,5 @@ {erl_opts, [debug_info]}. -{deps, [ {ecql, {git, "https://github.com/emqx/ecql.git", {tag, "v0.5.1"}}} - , {emqx_connector, {path, "../../apps/emqx_connector"}} +{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} , {emqx_utils, {path, "../emqx_utils"}} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 9465464d9..e8be79cdc 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -17,6 +17,7 @@ api_schemas(Method) -> ref(emqx_bridge_gcp_pubsub, Method), ref(emqx_bridge_kafka, Method ++ "_consumer"), ref(emqx_bridge_kafka, Method ++ "_producer"), + ref(emqx_bridge_cassandra, Method), ref(emqx_ee_bridge_mysql, Method), ref(emqx_ee_bridge_pgsql, Method), ref(emqx_ee_bridge_mongodb, Method ++ "_rs"), @@ -34,13 +35,13 @@ api_schemas(Method) -> ref(emqx_ee_bridge_clickhouse, Method), ref(emqx_ee_bridge_dynamo, Method), ref(emqx_ee_bridge_rocketmq, Method), - ref(emqx_ee_bridge_cassa, Method), ref(emqx_ee_bridge_sqlserver, Method) ]. schema_modules() -> [ emqx_bridge_kafka, + emqx_bridge_cassandra, emqx_ee_bridge_hstreamdb, emqx_bridge_gcp_pubsub, emqx_ee_bridge_influxdb, @@ -54,7 +55,6 @@ schema_modules() -> emqx_ee_bridge_clickhouse, emqx_ee_bridge_dynamo, emqx_ee_bridge_rocketmq, - emqx_ee_bridge_cassa, emqx_ee_bridge_sqlserver ]. @@ -75,6 +75,7 @@ resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer; %% TODO: rename this to `kafka_producer' after alias support is added %% to hocon; keeping this as just `kafka' for backwards compatibility. resource_type(kafka) -> emqx_bridge_kafka_impl_producer; +resource_type(cassandra) -> emqx_bridge_cassandra_impl; resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; resource_type(gcp_pubsub) -> emqx_bridge_gcp_pubsub_connector; resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; @@ -93,7 +94,6 @@ resource_type(tdengine) -> emqx_ee_connector_tdengine; resource_type(clickhouse) -> emqx_ee_connector_clickhouse; resource_type(dynamo) -> emqx_ee_connector_dynamo; resource_type(rocketmq) -> emqx_ee_connector_rocketmq; -resource_type(cassandra) -> emqx_ee_connector_cassa; resource_type(sqlserver) -> emqx_ee_connector_sqlserver. fields(bridges) -> @@ -148,7 +148,7 @@ fields(bridges) -> )}, {cassandra, mk( - hoconsc:map(name, ref(emqx_ee_bridge_cassa, "config")), + hoconsc:map(name, ref(emqx_bridge_cassandra, "config")), #{ desc => <<"Cassandra Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct index fc8e75e68..cb2f6f028 100644 --- a/lib-ee/emqx_ee_connector/docker-ct +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -1,5 +1,4 @@ toxiproxy influxdb clickhouse -cassandra sqlserver diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index ced7ae86a..82f556bdb 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -12,7 +12,6 @@ clickhouse, erlcloud, rocketmq, - ecql, odbc ]}, {env, []}, diff --git a/mix.exs b/mix.exs index f3149a584..c32fa798d 100644 --- a/mix.exs +++ b/mix.exs @@ -156,6 +156,19 @@ defmodule EMQXUmbrella.MixProject do MapSet.new([ :emqx_bridge_kafka, :emqx_bridge_gcp_pubsub + :emqx_bridge_cassandra, + :emqx_bridge_clickhouse, + :emqx_bridge_dynamo, + :emqx_bridge_hstreamdb, + :emqx_bridge_influxdb, + :emqx_bridge_matrix, + :emqx_bridge_mongodb, + :emqx_bridge_mysql, + :emqx_bridge_pgsql, + :emqx_bridge_redis, + :emqx_bridge_rocketmq, + :emqx_bridge_tdengine, + :emqx_bridge_timescale ]) end diff --git a/rebar.config.erl b/rebar.config.erl index 9bdbfb848..88471c39d 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -80,6 +80,19 @@ is_enterprise(ee) -> true. is_community_umbrella_app("apps/emqx_bridge_kafka") -> false; is_community_umbrella_app("apps/emqx_bridge_gcp_pubsub") -> false; +is_community_umbrella_app("apps/emqx_bridge_cassandra") -> false; +is_community_umbrella_app("apps/emqx_bridge_clickhouse") -> false; +is_community_umbrella_app("apps/emqx_bridge_dynamo") -> false; +is_community_umbrella_app("apps/emqx_bridge_hstreamdb") -> false; +is_community_umbrella_app("apps/emqx_bridge_influxdb") -> false; +is_community_umbrella_app("apps/emqx_bridge_matrix") -> false; +is_community_umbrella_app("apps/emqx_bridge_mongodb") -> false; +is_community_umbrella_app("apps/emqx_bridge_mysql") -> false; +is_community_umbrella_app("apps/emqx_bridge_pgsql") -> false; +is_community_umbrella_app("apps/emqx_bridge_redis") -> false; +is_community_umbrella_app("apps/emqx_bridge_rocketmq") -> false; +is_community_umbrella_app("apps/emqx_bridge_tdengine") -> false; +is_community_umbrella_app("apps/emqx_bridge_timescale") -> false; is_community_umbrella_app(_) -> true. is_jq_supported() -> @@ -441,6 +454,19 @@ relx_apps_per_edition(ee) -> emqx_ee_bridge, emqx_bridge_kafka, emqx_bridge_gcp_pubsub, + emqx_bridge_cassandra, + emqx_bridge_clickhouse, + emqx_bridge_dynamo, + emqx_bridge_hstreamdb, + emqx_bridge_influxdb, + emqx_bridge_matrix, + emqx_bridge_mongodb, + emqx_bridge_mysql, + emqx_bridge_pgsql, + emqx_bridge_redis, + emqx_bridge_rocketmq, + emqx_bridge_tdengine, + emqx_bridge_timescale, emqx_ee_schema_registry ]; relx_apps_per_edition(ce) -> diff --git a/rel/i18n/emqx_ee_bridge_cassa.hocon b/rel/i18n/emqx_bridge_cassandra.hocon similarity index 97% rename from rel/i18n/emqx_ee_bridge_cassa.hocon rename to rel/i18n/emqx_bridge_cassandra.hocon index d86c95a5f..d598d3921 100644 --- a/rel/i18n/emqx_ee_bridge_cassa.hocon +++ b/rel/i18n/emqx_bridge_cassandra.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_cassa { +emqx_bridge_cassandra { config_enable.desc: """Enable or disable this bridge""" diff --git a/rel/i18n/emqx_ee_connector_cassa.hocon b/rel/i18n/emqx_bridge_cassandra_impl.hocon similarity index 92% rename from rel/i18n/emqx_ee_connector_cassa.hocon rename to rel/i18n/emqx_bridge_cassandra_impl.hocon index bd5fb544c..91157fc09 100644 --- a/rel/i18n/emqx_ee_connector_cassa.hocon +++ b/rel/i18n/emqx_bridge_cassandra_impl.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_cassa { +emqx_bridge_cassa_impl { keyspace.desc: """Keyspace name to connect to.""" diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 4e79476e0..aca83ae86 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -112,6 +112,48 @@ case "${WHICH_APP}" in ## ensure enterprise profile when testing ee applications export PROFILE='emqx-enterprise' ;; + apps/emqx_bridge_cassandra) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_clickhouse) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_dynamo) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_gcp_pubsub) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_hstreamdb) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_influxdb) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_matrix) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_mongodb) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_mysql) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_pgsql) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_redis) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_rocketmq) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_tdengine) + export PROFILE='emqx-enterprise' + ;; + apps/emqx_bridge_timescale) + export PROFILE='emqx-enterprise' + ;; *) export PROFILE="${PROFILE:-emqx}" ;; From 5cc28a7b4539affd0de71420911e7437cc54978c Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 19 Apr 2023 13:21:31 +0800 Subject: [PATCH 277/279] chore: fix mix.exs checking --- .../include/emqx_bridge_cassandra.hrl | 5 +++++ .../src/emqx_bridge_cassandra_impl.erl | 2 +- .../test/emqx_bridge_cassandra_impl_SUITE.erl | 4 ++-- .../emqx_ee_connector/include/emqx_ee_connector.hrl | 1 - mix.exs | 13 +++++++++++++ rel/i18n/emqx_bridge_cassandra_impl.hocon | 2 +- 6 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 apps/emqx_bridge_cassandra/include/emqx_bridge_cassandra.hrl diff --git a/apps/emqx_bridge_cassandra/include/emqx_bridge_cassandra.hrl b/apps/emqx_bridge_cassandra/include/emqx_bridge_cassandra.hrl new file mode 100644 index 000000000..eef7c5d2b --- /dev/null +++ b/apps/emqx_bridge_cassandra/include/emqx_bridge_cassandra.hrl @@ -0,0 +1,5 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(CASSANDRA_DEFAULT_PORT, 9042). diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl index 8fe329a98..a4247a8c7 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl @@ -7,7 +7,7 @@ -behaviour(emqx_resource). -include_lib("emqx_connector/include/emqx_connector.hrl"). --include_lib("emqx_ee_connector/include/emqx_ee_connector.hrl"). +-include("emqx_bridge_cassandra.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl index faaee3c63..db91ca956 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl @@ -7,8 +7,8 @@ -compile(nowarn_export_all). -compile(export_all). --include("emqx_connector.hrl"). --include("emqx_ee_connector.hrl"). +-include("emqx_bridge_cassandra.hrl"). +-include("emqx_connector/include/emqx_connector.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/assert.hrl"). diff --git a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl index 2a91d2524..4b6fbbd92 100644 --- a/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl +++ b/lib-ee/emqx_ee_connector/include/emqx_ee_connector.hrl @@ -3,4 +3,3 @@ %%------------------------------------------------------------------- -define(INFLUXDB_DEFAULT_PORT, 8086). --define(CASSANDRA_DEFAULT_PORT, 9042). diff --git a/mix.exs b/mix.exs index c32fa798d..02c239a99 100644 --- a/mix.exs +++ b/mix.exs @@ -359,6 +359,19 @@ defmodule EMQXUmbrella.MixProject do emqx_ee_bridge: :permanent, emqx_bridge_kafka: :permanent, emqx_bridge_gcp_pubsub: :permanent, + emqx_bridge_cassandra: :permanent, + emqx_bridge_clickhouse: :permanent, + emqx_bridge_dynamo: :permanent, + emqx_bridge_hstreamdb: :permanent, + emqx_bridge_influxdb: :permanent, + emqx_bridge_matrix: :permanent, + emqx_bridge_mongodb: :permanent, + emqx_bridge_mysql: :permanent, + emqx_bridge_pgsql: :permanent, + emqx_bridge_redis: :permanent, + emqx_bridge_rocketmq: :permanent, + emqx_bridge_tdengine: :permanent, + emqx_bridge_timescale: :permanent, emqx_ee_schema_registry: :permanent ], else: [] diff --git a/rel/i18n/emqx_bridge_cassandra_impl.hocon b/rel/i18n/emqx_bridge_cassandra_impl.hocon index 91157fc09..03f389edd 100644 --- a/rel/i18n/emqx_bridge_cassandra_impl.hocon +++ b/rel/i18n/emqx_bridge_cassandra_impl.hocon @@ -1,4 +1,4 @@ -emqx_bridge_cassa_impl { +emqx_bridge_cassandra_impl { keyspace.desc: """Keyspace name to connect to.""" From b270623c46fc342f71b201b3c64ce25eb20db74a Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Apr 2023 14:48:31 +0800 Subject: [PATCH 278/279] chore: rename cassandra_impl to cassandra_connector --- apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl | 2 +- ...cassandra_impl.erl => emqx_bridge_cassandra_connector.erl} | 2 +- .../test/emqx_bridge_cassandra_SUITE.erl | 2 +- ...pl_SUITE.erl => emqx_bridge_cassandra_connector_SUITE.erl} | 4 ++-- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 2 +- mix.exs | 2 +- ...andra_impl.hocon => emqx_bridge_cassandra_connector.hocon} | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) rename apps/emqx_bridge_cassandra/src/{emqx_bridge_cassandra_impl.erl => emqx_bridge_cassandra_connector.erl} (99%) rename apps/emqx_bridge_cassandra/test/{emqx_bridge_cassandra_impl_SUITE.erl => emqx_bridge_cassandra_connector_SUITE.erl} (98%) rename rel/i18n/{emqx_bridge_cassandra_impl.hocon => emqx_bridge_cassandra_connector.hocon} (90%) diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl index 32dc11839..e8f7d50ce 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.erl @@ -88,7 +88,7 @@ fields("config") -> #{desc => ?DESC("local_topic"), default => undefined} )} ] ++ emqx_resource_schema:fields("resource_opts") ++ - (emqx_bridge_cassandra_impl:fields(config) -- + (emqx_bridge_cassandra_connector:fields(config) -- emqx_connector_schema_lib:prepare_statement_fields()); fields("post") -> fields("post", cassandra); diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl similarity index 99% rename from apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl rename to apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index a4247a8c7..cf6ddff9f 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_impl.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_cassandra_impl). +-module(emqx_bridge_cassandra_connector). -behaviour(emqx_resource). diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index 482f2c1f2..7865f0415 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -590,7 +590,7 @@ t_missing_data(Config) -> {ok, _}, create_bridge(Config) ), - %% emqx_bridge_cassandra_impl will send missed data as a `null` atom + %% emqx_bridge_cassandra_connector will send missed data as a `null` atom %% to ecql driver ?check_trace( begin diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl similarity index 98% rename from apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl rename to apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index db91ca956..f419283a8 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_impl_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_bridge_cassandra_impl_SUITE). +-module(emqx_bridge_cassandra_connector_SUITE). -compile(nowarn_export_all). -compile(export_all). @@ -16,7 +16,7 @@ %% Cassandra server defined at `.ci/docker-compose-file/docker-compose-cassandra-tcp.yaml` %% You can change it to `127.0.0.1`, if you run this SUITE locally -define(CASSANDRA_HOST, "cassandra"). --define(CASSANDRA_RESOURCE_MOD, emqx_bridge_cassandra_impl). +-define(CASSANDRA_RESOURCE_MOD, emqx_bridge_cassandra_connector). %% This test SUITE requires a running cassandra instance. If you don't want to %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index e8be79cdc..7fdfbba99 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -75,7 +75,7 @@ resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer; %% TODO: rename this to `kafka_producer' after alias support is added %% to hocon; keeping this as just `kafka' for backwards compatibility. resource_type(kafka) -> emqx_bridge_kafka_impl_producer; -resource_type(cassandra) -> emqx_bridge_cassandra_impl; +resource_type(cassandra) -> emqx_bridge_cassandra_connector; resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; resource_type(gcp_pubsub) -> emqx_bridge_gcp_pubsub_connector; resource_type(mongodb_rs) -> emqx_ee_connector_mongodb; diff --git a/mix.exs b/mix.exs index 02c239a99..c5d6df804 100644 --- a/mix.exs +++ b/mix.exs @@ -155,7 +155,7 @@ defmodule EMQXUmbrella.MixProject do defp enterprise_umbrella_apps() do MapSet.new([ :emqx_bridge_kafka, - :emqx_bridge_gcp_pubsub + :emqx_bridge_gcp_pubsub, :emqx_bridge_cassandra, :emqx_bridge_clickhouse, :emqx_bridge_dynamo, diff --git a/rel/i18n/emqx_bridge_cassandra_impl.hocon b/rel/i18n/emqx_bridge_cassandra_connector.hocon similarity index 90% rename from rel/i18n/emqx_bridge_cassandra_impl.hocon rename to rel/i18n/emqx_bridge_cassandra_connector.hocon index 03f389edd..b149cce8a 100644 --- a/rel/i18n/emqx_bridge_cassandra_impl.hocon +++ b/rel/i18n/emqx_bridge_cassandra_connector.hocon @@ -1,4 +1,4 @@ -emqx_bridge_cassandra_impl { +emqx_bridge_cassandra_connector { keyspace.desc: """Keyspace name to connect to.""" From 895963c0f6e66c9399182d3ffbee2418fd2fd6aa Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 21 Apr 2023 20:00:26 +0800 Subject: [PATCH 279/279] chore: shorten ct/run.sh script --- scripts/ct/run.sh | 51 ++++++----------------------------------------- 1 file changed, 6 insertions(+), 45 deletions(-) diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index aca83ae86..ab7fff444 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -108,51 +108,12 @@ case "${WHICH_APP}" in ## ensure enterprise profile when testing lib-ee applications export PROFILE='emqx-enterprise' ;; - apps/emqx_bridge_kafka) - ## ensure enterprise profile when testing ee applications - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_cassandra) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_clickhouse) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_dynamo) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_gcp_pubsub) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_hstreamdb) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_influxdb) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_matrix) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_mongodb) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_mysql) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_pgsql) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_redis) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_rocketmq) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_tdengine) - export PROFILE='emqx-enterprise' - ;; - apps/emqx_bridge_timescale) - export PROFILE='emqx-enterprise' + apps/*) + if [[ -f "${WHICH_APP}/BSL.txt" ]]; then + export PROFILE='emqx-enterprise' + else + export PROFILE='emqx' + fi ;; *) export PROFILE="${PROFILE:-emqx}"