From fdee23f4920bc709fe0d895afde27317b9afcd3c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 27 May 2022 13:20:24 +0300 Subject: [PATCH 01/22] chore(mongodb authn): add defaults for field names --- apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 040803a0d..a46d62c8c 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -100,16 +100,19 @@ filter(_) -> password_hash_field(type) -> binary(); password_hash_field(desc) -> ?DESC(?FUNCTION_NAME); password_hash_field(required) -> false; +password_hash_field(default) -> <<"password_hash">>; password_hash_field(_) -> undefined. salt_field(type) -> binary(); salt_field(desc) -> ?DESC(?FUNCTION_NAME); salt_field(required) -> false; +salt_field(default) -> <<"salt">>; salt_field(_) -> undefined. is_superuser_field(type) -> binary(); is_superuser_field(desc) -> ?DESC(?FUNCTION_NAME); is_superuser_field(required) -> false; +is_superuser_field(default) -> <<"is_superuser">>; is_superuser_field(_) -> undefined. %%------------------------------------------------------------------------------ From d5649d1f89732482788148572b1f9838c68ddb95 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Mon, 30 May 2022 12:38:05 +0200 Subject: [PATCH 02/22] chore(mria): Bump version to 0.2.7 --- apps/emqx/rebar.config | 2 +- mix.exs | 4 ++-- rebar.config | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index c4e18a758..cb7f94b7a 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.2"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.6"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.8"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.5"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/mix.exs b/mix.exs index 86b1323a2..1fd710fbb 100644 --- a/mix.exs +++ b/mix.exs @@ -52,8 +52,8 @@ defmodule EMQXUmbrella.MixProject do {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.2", override: true}, - {:mria, github: "emqx/mria", tag: "0.2.5", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.12.6", override: true}, + {:mria, github: "emqx/mria", tag: "0.2.7", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.12.8", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.3", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, diff --git a/rebar.config b/rebar.config index 53a5edd82..d809a01d7 100644 --- a/rebar.config +++ b/rebar.config @@ -54,8 +54,8 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.2"}}} - , {mria, {git, "https://github.com/emqx/mria", {tag, "0.2.5"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.6"}}} + , {mria, {git, "https://github.com/emqx/mria", {tag, "0.2.7"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.3"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} From 932384b7bd731e8b3f4a59bbb5a711a46fa1fcc1 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Tue, 31 May 2022 10:22:06 +0200 Subject: [PATCH 03/22] chore(ci): Bump schema validate to 0.3.2 --- scripts/spellcheck | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spellcheck b/scripts/spellcheck index 8b5ba53bb..6f52e2fdd 100755 --- a/scripts/spellcheck +++ b/scripts/spellcheck @@ -7,7 +7,7 @@ else SCHEMA="$1" fi -docker run -d --name langtool "ghcr.io/iequ1/emqx-schema-validate:0.3.1" +docker run -d --name langtool "ghcr.io/iequ1/emqx-schema-validate:0.3.2" docker exec -i langtool ./emqx_schema_validate - < "${SCHEMA}" success="$?" From de6218dfa6f4a8135d00b232b0f9d9c1daec9550 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 31 May 2022 12:11:23 +0200 Subject: [PATCH 04/22] ci: do not configure quic listener for arm quic listener is now not enabled by default --- scripts/pkg-tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh index b12c8478a..b57a2bf71 100755 --- a/scripts/pkg-tests.sh +++ b/scripts/pkg-tests.sh @@ -91,9 +91,9 @@ emqx_test(){ export EMQX_MQTT__MAX_TOPIC_ALIAS=10 export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug export EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL=debug - if [[ $(arch) == *arm* || $(arch) == aarch64 ]]; then - export EMQX_LISTENERS__QUIC__DEFAULT__ENABLED=false - fi + # if [[ $(arch) == *arm* || $(arch) == aarch64 ]]; then + # export EMQX_LISTENERS__QUIC__DEFAULT__ENABLED=false + # fi # sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins echo "running ${packagename} start" From 616f1ae302ab6b84f8a324ebee34af36b5fc25e6 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 31 May 2022 12:16:33 +0200 Subject: [PATCH 05/22] ci: do not configure quic listener at all --- scripts/pkg-tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh index b57a2bf71..dac7248f4 100755 --- a/scripts/pkg-tests.sh +++ b/scripts/pkg-tests.sh @@ -184,9 +184,9 @@ export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug export EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL=debug EOF ## for ARM, due to CI env issue, skip start of quic listener for the moment - [[ $(arch) == *arm* || $(arch) == aarch64 ]] && tee -a "$emqx_env_vars" < Date: Tue, 31 May 2022 12:50:05 +0200 Subject: [PATCH 06/22] fix: ignore other os when updating sysmem alarm --- apps/emqx/src/emqx_os_mon.erl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index 1295c02e2..63c8e0298 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -168,17 +168,12 @@ start_cpu_check_timer() -> _ -> start_timer(Interval, cpu_check) end. +is_sysmem_check_supported() -> + {unix, linux} =:= os:type(). + start_mem_check_timer() -> Interval = emqx:get_config([sysmon, os, mem_check_interval]), - IsSupported = - case os:type() of - {unix, linux} -> - true; - _ -> - %% sorry Mac and windows, for now - false - end, - case is_integer(Interval) andalso IsSupported of + case is_integer(Interval) andalso is_sysmem_check_supported() of true -> start_timer(Interval, mem_check); false -> @@ -196,7 +191,11 @@ update_mem_alarm_status(HWM) when HWM > 1.0 orelse HWM < 0.0 -> #{}, <<"Deactivated mem usage alarm due to out of range threshold">> ); -update_mem_alarm_status(HWM0) -> +update_mem_alarm_status(HWM) -> + is_sysmem_check_supported() andalso + do_update_mem_alarm_status(HWM). + +do_update_mem_alarm_status(HWM0) -> HWM = HWM0 * 100, Usage = current_sysmem_percent(), case Usage > HWM of From 8aa60cc0a5a6db0e8edaa7509f71579c51fa2a9d Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 23 May 2022 09:27:52 +0800 Subject: [PATCH 07/22] feat: generate a minimized emqx.conf --- apps/emqx/etc/emqx.conf | 1580 +---------------- apps/emqx/src/emqx_config.erl | 35 +- apps/emqx_authn/etc/emqx_authn.conf | 2 +- apps/emqx_authz/etc/emqx_authz.conf | 78 +- .../etc/emqx_auto_subscribe.conf | 45 - .../src/emqx_auto_subscribe_app.erl | 2 +- .../src/emqx_auto_subscribe_schema.erl | 2 +- apps/emqx_bridge/etc/emqx_bridge.conf | 59 - apps/emqx_conf/etc/emqx_conf.conf | 832 +-------- apps/emqx_conf/src/emqx_conf.erl | 6 +- apps/emqx_conf/src/emqx_conf_schema.erl | 8 +- apps/emqx_connector/etc/emqx_connector.conf | 23 - apps/emqx_dashboard/etc/emqx_dashboard.conf | 38 +- .../src/emqx_dashboard_schema.erl | 2 +- apps/emqx_exhook/etc/emqx_exhook.conf | 46 - apps/emqx_gateway/etc/emqx_gateway.conf | 6 - apps/emqx_modules/etc/emqx_modules.conf | 39 - apps/emqx_modules/src/emqx_delayed.erl | 2 +- apps/emqx_modules/src/emqx_modules_app.erl | 2 +- apps/emqx_modules/src/emqx_modules_conf.erl | 2 +- apps/emqx_modules/src/emqx_modules_schema.erl | 6 +- apps/emqx_plugins/etc/emqx_plugins.conf | 7 - apps/emqx_prometheus/etc/emqx_prometheus.conf | 8 - apps/emqx_psk/etc/emqx_psk.conf | 21 - apps/emqx_psk/src/emqx_psk_schema.erl | 1 + apps/emqx_retainer/etc/emqx_retainer.conf | 82 - .../src/emqx_retainer_dispatcher.erl | 6 +- .../src/emqx_retainer_schema.erl | 38 +- .../etc/emqx_rule_engine.conf | 22 - apps/emqx_slow_subs/etc/emqx_slow_subs.conf | 35 - apps/emqx_statsd/etc/emqx_statsd.conf | 10 - scripts/merge-config.escript | 46 +- 32 files changed, 131 insertions(+), 2960 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 554ab05fd..768e9a856 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -1,1583 +1,39 @@ -##================================================================== -## Listeners -##================================================================== -## MQTT/TCP - TCP Listeners for MQTT Protocol -## syntax: listeners.tcp. -## example: listeners.tcp.my_tcp_listener + listeners.tcp.default { - ## The IP address and port that the listener will bind. - ## - ## @doc listeners.tcp..bind - ## ValueType: IPAddress | Port | IPAddrPort - ## Required: true - ## Examples: 1883, 127.0.0.1:1883, ::1:1883 bind = "0.0.0.0:1883" - - ## The configuration zone this listener is using. - ## If not set, the global configs are used for this listener. - ## - ## See `zones.` for more details. - ## - ## @doc listeners.tcp..zone - ## ValueType: String - ## Required: false - #zone = default - - ## The size of the acceptor pool for this listener. - ## - ## @doc listeners.tcp..acceptors - ## ValueType: Number - ## Default: 16 - acceptors = 16 - - ## Maximum number of concurrent connections. - ## - ## @doc listeners.tcp..max_connections - ## ValueType: Number | infinity - ## Default: infinity max_connections = 1024000 - - ## The access control rules for this listener. - ## - ## See: https://github.com/emqtt/esockd#allowdeny - ## - ## @doc listeners.tcp..access_rules - ## ValueType: Array - ## Default: ["allow all"] - ## Examples: - ## access_rules: [ - ## "deny 192.168.0.0/24", - ## "all all" - ## ] - access_rules = [ - "allow all" - ] - - ## Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed - ## behind HAProxy or Nginx. - ## - ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ - ## - ## @doc listeners.tcp..proxy_protocol - ## ValueType: Boolean - ## Default: false - proxy_protocol = false - - ## Sets the timeout for proxy protocol. EMQX will close the TCP connection - ## if no proxy protocol packet received within the timeout. - ## - ## @doc listeners.tcp..proxy_protocol_timeout - ## ValueType: Duration - ## Default: 3s - proxy_protocol_timeout = 3s - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - ## The prefixed string will be removed from the topic name when the message - ## is delivered to the subscriber. The mountpoint is a way that users can use - ## to implement isolation of message routing between different listeners. - ## - ## For example if a clientA subscribes to "t" with `listeners.tcp..mountpoint` - ## set to "some_tenant", then the client accually subscribes to the topic - ## "some_tenant/t". Similarly if another clientB (connected to the same listener - ## with the clientA) send a message to topic "t", the message is accually route - ## to all the clients subscribed "some_tenant/t", so clientA will receive the - ## message, with topic name "t". - ## - ## Set to "" to disable the feature. - ## - ## Variables in mountpoint string: - ## - ${clientid}: clientid - ## - ${username}: username - ## - ## @doc listeners.tcp..mountpoint - ## ValueType: String - ## Default: "" - mountpoint = "" - - ## TCP options - ## See ${example_common_tcp_options} for more information - tcp.backlog = 1024 - tcp.buffer = 4KB } -## MQTT/SSL - SSL Listeners for MQTT Protocol -## syntax: listeners.ssl. -## example: listeners.ssl.my_ssl_listener listeners.ssl.default { - ## The IP address and port that the listener will bind. - ## - ## @doc listeners.ssl..bind - ## ValueType: IPAddress | Port | IPAddrPort - ## Required: true - ## Examples: 8883, 127.0.0.1:8883, ::1:8883 bind = "0.0.0.0:8883" - - ## The configuration zone this listener is using. - ## If not set, the global configs are used for this listener. - ## - ## See `zones.` for more details. - ## - ## @doc listeners.ssl..zone - ## ValueType: String - ## Required: false - #zone = default - - ## The size of the acceptor pool for this listener. - ## - ## @doc listeners.ssl..acceptors - ## ValueType: Number - ## Default: 16 - acceptors = 16 - - ## Maximum number of concurrent connections. - ## - ## @doc listeners.ssl..max_connections - ## ValueType: Number | infinity - ## Default: infinity max_connections = 512000 - - ## The access control rules for this listener. - ## - ## See: https://github.com/emqtt/esockd#allowdeny - ## - ## @doc listeners.ssl..access_rules - ## ValueType: Array - ## Default: [] - ## Examples: - ## access_rules: [ - ## "deny 192.168.0.0/24", - ## "all all" - ## ] - access_rules = [ - "allow all" - ] - - ## Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed - ## behind HAProxy or Nginx. - ## - ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ - ## - ## @doc listeners.ssl..proxy_protocol - ## ValueType: Boolean - ## Default: true - proxy_protocol = false - - ## Sets the timeout for proxy protocol. EMQX will close the TCP connection - ## if no proxy protocol packet received within the timeout. - ## - ## @doc listeners.ssl..proxy_protocol_timeout - ## ValueType: Duration - ## Default: 3s - proxy_protocol_timeout = 3s - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - ## The prefixed string will be removed from the topic name when the message - ## is delivered to the subscriber. The mountpoint is a way that users can use - ## to implement isolation of message routing between different listeners. - ## - ## For example if a clientA subscribes to "t" with `listeners.ssl..mountpoint` - ## set to "some_tenant", then the client accually subscribes to the topic - ## "some_tenant/t". Similarly if another clientB (connected to the same listener - ## with the clientA) send a message to topic "t", the message is accually route - ## to all the clients subscribed "some_tenant/t", so clientA will receive the - ## message, with topic name "t". - ## - ## Set to "" to disable the feature. - ## - ## Variables in mountpoint string: - ## - ${clientid}: clientid - ## - ${username}: username - ## - ## @doc listeners.ssl..mountpoint - ## ValueType: String - ## Default: "" - mountpoint = "" - - ## SSL options - ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" - ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - - # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] - # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256" - # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config - # ssl.ciphers = "" - - ## TCP options - ## See ${example_common_tcp_options} for more information - tcp.backlog = 1024 - tcp.buffer = 4KB + ssl { + keyfile = "{{ platform_etc_dir }}/certs/key.pem" + certfile = "{{ platform_etc_dir }}/certs/cert.pem" + cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + } } -## MQTT/WS - Websocket Listeners for MQTT Protocol -## syntax: listeners.ws. -## example: listeners.ws.my_ws_listener listeners.ws.default { - ## The IP address and port that the listener will bind. - ## - ## @doc listeners.ws..bind - ## ValueType: IPAddress | Port | IPAddrPort - ## Required: true - ## Examples: 8083, 127.0.0.1:8083, ::1:8083 bind = "0.0.0.0:8083" - - ## The configuration zone this listener is using. - ## If not set, the global configs are used for this listener. - ## - ## See `zones.` for more details. - ## - ## @doc listeners.ws..zone - ## ValueType: String - ## Required: false - #zone = default - - ## The size of the acceptor pool for this listener. - ## - ## @doc listeners.ws..acceptors - ## ValueType: Number - ## Default: 16 - acceptors = 16 - - ## Maximum number of concurrent connections. - ## - ## @doc listeners.ws..max_connections - ## ValueType: Number | infinity - ## Default: infinity max_connections = 1024000 - - ## The access control rules for this listener. - ## - ## See: https://github.com/emqtt/esockd#allowdeny - ## - ## @doc listeners.ws..access_rules - ## ValueType: Array - ## Default: [] - ## Examples: - ## access_rules: [ - ## "deny 192.168.0.0/24", - ## "all all" - ## ] - access_rules = [ - "allow all" - ] - - ## Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed - ## behind HAProxy or Nginx. - ## - ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ - ## - ## @doc listeners.ws..proxy_protocol - ## ValueType: Boolean - ## Default: true - proxy_protocol = false - - ## Sets the timeout for proxy protocol. EMQX will close the TCP connection - ## if no proxy protocol packet received within the timeout. - ## - ## @doc listeners.ws..proxy_protocol_timeout - ## ValueType: Duration - ## Default: 3s - proxy_protocol_timeout = 3s - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - ## The prefixed string will be removed from the topic name when the message - ## is delivered to the subscriber. The mountpoint is a way that users can use - ## to implement isolation of message routing between different listeners. - ## - ## For example if a clientA subscribes to "t" with `listeners.ws..mountpoint` - ## set to "some_tenant", then the client accually subscribes to the topic - ## "some_tenant/t". Similarly if another clientB (connected to the same listener - ## with the clientA) send a message to topic "t", the message is accually route - ## to all the clients subscribed "some_tenant/t", so clientA will receive the - ## message, with topic name "t". - ## - ## Set to "" to disable the feature. - ## - ## Variables in mountpoint string: - ## - ${clientid}: clientid - ## - ${username}: username - ## - ## @doc listeners.ws..mountpoint - ## ValueType: String - ## Default: "" - mountpoint = "" - - ## TCP options - ## See ${example_common_tcp_options} for more information - tcp.backlog = 1024 - tcp.buffer = 4KB - - ## Websocket options - ## See ${example_common_websocket_options} for more information - websocket.idle_timeout = 86400s + websocket.mqtt_path: "/mqtt" } -## MQTT/WSS - WebSocket Secure Listeners for MQTT Protocol -## syntax: listeners.wss. -## example: listeners.wss.my_wss_listener listeners.wss.default { - ## The IP address and port that the listener will bind. - ## - ## @doc listeners.wss..bind - ## ValueType: IPAddress | Port | IPAddrPort - ## Required: true - ## Examples: 8084, 127.0.0.1:8084, ::1:8084 bind = "0.0.0.0:8084" - - ## The configuration zone this listener is using. - ## If not set, the global configs are used for this listener. - ## - ## See `zones.` for more details. - ## - ## @doc listeners.wss..zone - ## ValueType: String - ## Required: false - #zone = default - - ## The size of the acceptor pool for this listener. - ## - ## @doc listeners.wss..acceptors - ## ValueType: Number - ## Default: 16 - acceptors = 16 - - ## Maximum number of concurrent connections. - ## - ## @doc listeners.wss..max_connections - ## ValueType: Number | infinity - ## Default: infinity max_connections = 512000 - - ## The access control rules for this listener. - ## - ## See: https://github.com/emqtt/esockd#allowdeny - ## - ## @doc listeners.wss..access_rules - ## ValueType: Array - ## Default: [] - ## Examples: - ## access_rules: [ - ## "deny 192.168.0.0/24", - ## "all all" - ## ] - access_rules = [ - "allow all" - ] - - ## Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed - ## behind HAProxy or Nginx. - ## - ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ - ## - ## @doc listeners.wss..proxy_protocol - ## ValueType: Boolean - ## Default: true - proxy_protocol = false - - ## Sets the timeout for proxy protocol. EMQX will close the TCP connection - ## if no proxy protocol packet received within the timeout. - ## - ## @doc listeners.wss..proxy_protocol_timeout - ## ValueType: Duration - ## Default: 3s - proxy_protocol_timeout = 3s - - ## When publishing or subscribing, prefix all topics with a mountpoint string. - ## The prefixed string will be removed from the topic name when the message - ## is delivered to the subscriber. The mountpoint is a way that users can use - ## to implement isolation of message routing between different listeners. - ## - ## For example if a clientA subscribes to "t" with `listeners.wss..mountpoint` - ## set to "some_tenant", then the client accually subscribes to the topic - ## "some_tenant/t". Similarly if another clientB (connected to the same listener - ## with the clientA) send a message to topic "t", the message is accually route - ## to all the clients subscribed "some_tenant/t", so clientA will receive the - ## message, with topic name "t". - ## - ## Set to "" to disable the feature. - ## - ## Variables in mountpoint string: - ## - ${clientid}: clientid - ## - ${username}: username - ## - ## @doc listeners.wss..mountpoint - ## ValueType: String - ## Default: "" - mountpoint = "" - - ## SSL options - ## See ${example_common_ssl_options} for more information - ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" - ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - - ## TCP options - ## See ${example_common_tcp_options} for more information - tcp.backlog = 1024 - tcp.buffer = 4KB - - ## Websocket options - ## See ${example_common_websocket_options} for more information - websocket.idle_timeout = 86400s - -} - -## Enable per connection statistics. -## -## @doc stats.enable -## ValueType: Boolean -## Default: true -stats.enable = true - -authorization { - ## Behaviour after not matching a rule. - ## - ## @doc authorization.no_match - ## ValueType: allow | deny - ## Default: allow - no_match: allow - - ## The action when the authorization check rejects an operation - ## - ## @doc authorization.deny_action - ## ValueType: ignore | disconnect - ## Default: ignore - deny_action: ignore - - ## Whether to enable Authorization cache. - ## - ## If enabled, Authorization roles for each client will be cached in the memory - ## - ## @doc authorization.cache.enable - ## ValueType: Boolean - ## Default: true - cache.enable: true - - ## The maximum count of Authorization entries can be cached for a client. - ## - ## @doc authorization.cache.max_size - ## ValueType: Integer - ## Range: [0, 1048576] - ## Default: 32 - cache.max_size: 32 - - ## The time after which an Authorization cache entry will be deleted - ## - ## @doc authorization.cache.ttl - ## ValueType: Duration - ## Default: 1m - cache.ttl: 1m -} - -mqtt { - ## How long time the MQTT connection will be disconnected if the - ## TCP connection is established but MQTT CONNECT has not been - ## received. - ## - ## @doc mqtt.idle_timeout - ## ValueType: Duration - ## Default: 15s - idle_timeout = 15s - - ## Maximum MQTT packet size allowed. - ## - ## @doc mqtt.max_packet_size - ## ValueType: Bytes - ## Default: 1MB - max_packet_size = 1MB - - ## Maximum length of MQTT clientId allowed. - ## - ## @doc mqtt.max_clientid_len - ## ValueType: Integer - ## Range: [23, 65535] - ## Default: 65535 - max_clientid_len = 65535 - - ## Maximum topic levels allowed. - ## - ## @doc mqtt.max_topic_levels - ## ValueType: Integer - ## Range: [1, 65535] - ## Default: 128 - ## Depth so big may lead to subscribing performance issues - max_topic_levels = 128 - - ## Maximum QoS allowed. - ## - ## @doc mqtt.max_qos_allowed - ## ValueType: 0 | 1 | 2 - ## Default: 2 - max_qos_allowed = 2 - - ## Maximum Topic Alias, 0 means no topic alias supported. - ## - ## @doc mqtt.max_topic_alias - ## ValueType: Integer - ## Range: [0, 65535] - ## Default: 65535 - max_topic_alias = 65535 - - ## Whether the Server supports MQTT retained messages. - ## - ## @doc mqtt.retain_available - ## ValueType: Boolean - ## Default: true - retain_available = true - - ## Whether the Server supports MQTT Wildcard Subscriptions - ## - ## @doc mqtt.wildcard_subscription - ## ValueType: Boolean - ## Default: true - wildcard_subscription = true - - ## Whether the Server supports MQTT Shared Subscriptions. - ## - ## @doc mqtt.shared_subscription - ## ValueType: Boolean - ## Default: true - shared_subscription = true - - ## Whether to ignore loop delivery of messages.(for mqtt v3.1.1) - ## - ## @doc mqtt.ignore_loop_deliver - ## ValueType: Boolean - ## Default: false - ignore_loop_deliver = false - - ## Whether to parse the MQTT frame in strict mode - ## - ## @doc mqtt.strict_mode - ## ValueType: Boolean - ## Default: false - strict_mode = false - - ## Specify the response information returned to the client - ## - ## This feature is disabled if is set to "" - ## - ## @doc mqtt.response_information - ## ValueType: String - ## Default: "" - response_information = "" - - ## Server Keep Alive of MQTT 5.0 - ## - ## @doc mqtt.server_keepalive - ## ValueType: Number | disabled - ## Default: disabled - server_keepalive = disabled - - ## The backoff for MQTT keepalive timeout. The broker will close the connection - ## after idling for 'Keepalive * backoff * 2'. - ## - ## @doc mqtt.keepalive_backoff - ## ValueType: Float - ## Default: 0.75 - keepalive_backoff = 0.75 - - ## Maximum number of subscriptions allowed. - ## - ## @doc mqtt.max_subscriptions - ## ValueType: Integer | infinity - ## Range: [1, infinity) - ## Default: infinity - max_subscriptions = infinity - - ## Force to upgrade QoS according to subscription. - ## - ## @doc mqtt.upgrade_qos - ## ValueType: Boolean - ## Default: false - upgrade_qos = false - - ## Maximum size of the Inflight Window storing QoS1/2 messages delivered but unacked. - ## - ## @doc mqtt.max_inflight - ## ValueType: Integer - ## Range: [1, 65535] - ## Default: 32 - max_inflight = 32 - - ## Retry interval for QoS1/2 message delivering. - ## - ## @doc mqtt.retry_interval - ## ValueType: Duration - ## Default: 30s - retry_interval = 30s - - ## Maximum QoS2 packets (Client -> Broker) awaiting PUBREL. - ## - ## @doc mqtt.max_awaiting_rel - ## ValueType: Integer | infinity - ## Range: [1, infinity) - ## Default: 100 - max_awaiting_rel = 100 - - ## The QoS2 messages (Client -> Broker) will be dropped if awaiting PUBREL timeout. - ## - ## @doc mqtt.await_rel_timeout - ## ValueType: Duration - ## Default: 300s - await_rel_timeout = 300s - - ## Default session expiry interval for MQTT V3.1.1 connections. - ## - ## @doc mqtt.session_expiry_interval - ## ValueType: Duration - ## Default: 2h - session_expiry_interval = 2h - - ## Maximum queue length. Enqueued messages when persistent client disconnected, - ## or inflight window is full. - ## - ## @doc mqtt.max_mqueue_len - ## ValueType: Integer | infinity - ## Range: [0, infinity) - ## Default: 1000 - max_mqueue_len = 1000 - - ## Topic priorities. - ## - ## There's no priority table by default, hence all messages - ## are treated equal. - ## - ## Priority number [1-255] - ## - ## NOTE: comma and equal signs are not allowed for priority topic names - ## NOTE: Messages for topics not in the priority table are treated as - ## either highest or lowest priority depending on the configured - ## value for mqtt.mqueue_default_priority - ## - ## @doc mqtt.mqueue_priorities - ## ValueType: Map | disabled - ## Examples: - ## To configure "topic/1" > "topic/2": - ## mqueue_priorities: {"topic/1": 10, "topic/2": 8} - ## Default: disabled - mqueue_priorities = disabled - - ## Default to highest priority for topics not matching priority table - ## - ## @doc mqtt.mqueue_default_priority - ## ValueType: highest | lowest - ## Default: lowest - mqueue_default_priority = lowest - - ## Whether to enqueue QoS0 messages. - ## - ## @doc mqtt.mqueue_store_qos0 - ## ValueType: Boolean - ## Default: true - mqueue_store_qos0 = true - - ## Whether use username replace client id - ## - ## @doc mqtt.use_username_as_clientid - ## ValueType: Boolean - ## Default: false - use_username_as_clientid = false - - ## Use the CN, DN or CRT field from the client certificate as a username. - ## Only works for SSL connection. - ## - ## @doc mqtt.peer_cert_as_username - ## ValueType: cn | dn | crt | disabled - ## Default: disabled - peer_cert_as_username = disabled - - ## Use the CN, DN or CRT field from the client certificate as a clientid. - ## Only works for SSL connection. - ## - ## @doc mqtt.peer_cert_as_clientid - ## ValueType: cn | dn | crt | disabled - ## Default: disabled - peer_cert_as_clientid = disabled -} - -flapping_detect { - ## Enable Flapping Detection. - ## - ## This config controls the allowed maximum number of CONNECT received - ## from the same clientid in a time frame defined by `window_time`. - ## After the limit is reached, successive CONNECT requests are forbidden - ## (banned) until the end of the time period defined by `ban_time`. - ## - ## @doc flapping_detect.enable - ## ValueType: Boolean - ## Default: true - enable = false - - ## The max disconnect allowed of a MQTT Client in `window_time` - ## - ## @doc flapping_detect.max_count - ## ValueType: Integer - ## Default: 15 - max_count = 15 - - ## The time window for flapping detect - ## - ## @doc flapping_detect.window_time - ## ValueType: Duration - ## Default: 1m - window_time = 1m - - ## How long the clientid will be banned - ## - ## @doc flapping_detect.ban_time - ## ValueType: Duration - ## Default: 5m - ban_time = 5m - -} - -force_shutdown { - ## Enable force_shutdown - ## - ## @doc force_shutdown.enable - ## ValueType: Boolean - ## Default: true - enable = true - - ## Max message queue length - ## @doc force_shutdown.max_message_queue_len - ## ValueType: Integer - ## Range: (0, infinity) - ## Default: 1000 - max_message_queue_len = 1000 - - ## Total heap size - ## - ## @doc force_shutdown.max_heap_size - ## ValueType: Size - ## Default: 32MB - max_heap_size = 32MB -} - -overload_protection { - ## React on system overload or not - ## @doc overload_protection.enable - ## ValueType: Boolean - ## Default: false - enable = false - - ## Backoff delay in ms - ## @doc overload_protection.backoff_delay - ## ValueType: Integer - ## Range: (0, infinity) - ## Default: 1 - backoff_delay = 1 - - ## Backoff GC enabled - ## @doc overload_protection.backoff_gc - ## ValueType: Boolean - ## Default: false - backoff_gc = false - - ## Backoff hibernation enabled - ## @doc overload_protection.backoff_hibernation - ## ValueType: Boolean - ## Default: true - backoff_hibernation = true - - ## Backoff hibernation enabled - ## @doc overload_protection.backoff_hibernation - ## ValueType: Boolean - ## Default: true - backoff_new_conn = true -} - -force_gc { - ## Force the MQTT connection process GC after this number of - ## messages or bytes passed through. - ## - ## @doc force_gc.enable - ## ValueType: Boolean - ## Default: true - enable = true - - ## GC the process after how many messages received - ## @doc force_gc.max_message_queue_len - ## ValueType: Integer - ## Range: (0, infinity) - ## Default: 16000 - count = 16000 - - ## GC the process after how much bytes passed through - ## - ## @doc force_gc.bytes - ## ValueType: Size - ## Default: 16MB - bytes = 16MB -} - -conn_congestion { - ## Whether to alarm the congested connections. - ## - ## Sometimes the mqtt connection (usually an MQTT subscriber) may - ## get "congested", because there's too many packets to sent. - ## The socket tries to buffer the packets until the buffer is - ## full. If more packets come after that, the packets will be - ## "pending" in a queue, and we consider the connection is - ## "congested". - ## - ## Enable this to send an alarm when there's any bytes pending in - ## the queue. You could set the `sndbuf` to a larger value if the - ## alarm is triggered too often. - ## - ## The name of the alarm is of format "conn_congestion//". - ## Where the is the client-id of the congested MQTT connection. - ## And the is the username or "unknown_user" of not provided by the client. - ## - ## @doc conn_congestion.enable_alarm - ## ValueType: Boolean - ## Default: true - enable_alarm = true - - ## Won't clear the congested alarm in how long time. - ## The alarm is cleared only when there's no pending bytes in - ## the queue, and also it has been `min_alarm_sustain_duration` - ## time since the last time we considered the connection is "congested". - ## - ## This is to avoid clearing and sending the alarm again too often. - ## - ## @doc conn_congestion.min_alarm_sustain_duration - ## ValueType: Duration - ## Default: 1m - min_alarm_sustain_duration = 1m -} - -##================================================================== -## Zones -##================================================================== -## A zone contains a set of configurations for listeners. -## -## A zone can be used by a listener via `listener...zone`. -## -## The configs defined in zones will override the global configs with the same key. -## -## For example given the following config: -## -## ``` -## a { -## b: 1, c: 1 -## } -## -## zone.my_zone { -## a { -## b:2 -## } -## } -## ``` -## -## The global config "a" is overridden by the configs "a" inside the zone "my_zone". -## If there is a listener uses the zone "my_zone", the value of config "a" will be: -## `{b:2, c: 1}`. -## Note that although the default value of `a.c` is `0`, the global value is used. -## i.e. configs in the zone have no default values. To overridde `a.c` we must configure -## it explicitly in the zone. -## -## All the global configs that can be overridden in zones are: -## - `stats.*` -## - `mqtt.*` -## - `authorization.*` -## - `flapping_detect.*` -## - `force_shutdown.*` -## - `conn_congestion.*` -## - `force_gc.*` -## -## syntax: zones. -## example: zones.my_zone -zones.default { - -} - -##================================================================== -## Broker -##================================================================== -broker { - ## Session locking strategy in a cluster. - ## - ## @doc broker.session_locking_strategy - ## ValueType: local | one | quorum | all - ## - local: only lock the session locally on the current node - ## - one: select only one remote node to lock the session - ## - quorum: select some nodes to lock the session - ## - all: lock the session on all the nodes in the cluster - ## Default: quorum - session_locking_strategy = quorum - - ## Dispatch strategy for shared subscription - ## - ## @doc broker.shared_subscription_strategy - ## ValueType: random | round_robin | sticky | hash | local - ## - random: dispatch the message to a random selected subscriber - ## - round_robin: select the subscribers in a round-robin manner - ## - local: select random local subscriber otherwise select random cluster-wide - ## - sticky: always use the last selected subscriber to dispatch, - ## until the subscriber disconnects. - ## - hash: select the subscribers by the hash of clientIds - ## Default: round_robin - shared_subscription_strategy = round_robin - - ## Per-group dispatch strategy for shared subscription - ## - ## @doc broker.shared_subscription_group.$group_name.strategy - ## ValueType: random | round_robin | sticky | hash | local - ## - random: dispatch the message to a random selected subscriber - ## - round_robin: select the subscribers in a round-robin manner - ## - local: select the local subscriber otherwise random cluster-wide - ## - sticky: always use the last selected subscriber to dispatch, - ## until the subscriber disconnects. - ## - hash: select the subscribers by the hash of clientIds - ## Default: round_robin - shared_subscription_group { - - ## example_group { - ## strategy = random - ## } - - } - - ## Enable/disable shared dispatch acknowledgement for QoS1 and QoS2 messages - ## This should allow messages to be dispatched to a different subscriber in - ## the group in case the picked (based on shared_subscription_strategy) one # is offline - ## - ## @doc broker.shared_dispatch_ack_enabled - ## ValueType: Boolean - ## Default: false - shared_dispatch_ack_enabled = false - - ## Enable batch clean for deleted routes. - ## - ## @doc broker.route_batch_clean - ## ValueType: Boolean - ## Default: true - route_batch_clean = true - - ## Performance tuning for subscribe/unsubscribe wildcard topic. - ## Change this parameter only when there are many wildcard topics. - ## - ## NOTE: when changing from/to 'global' lock, it requires all - ## nodes in the cluster to be stopped before the change. - ## - ## @doc broker.perf.route_lock_type - ## ValueType: key | tab | global - ## - key: mnesia transactional updates with per-key locks. recommended for single node setup. - ## - tab: mnesia transactional updates with table lock. recommended for multi-nodes setup. - ## - global: global lock protected updates. recommended for larger cluster. - ## Default: key - perf.route_lock_type = key - - ## Enable trie path compaction. - ## Enabling it significantly improves wildcard topic subscribe - ## rate, if wildcard topics have unique prefixes like: - ## 'sensor/{{id}}/+/', where ID is unique per subscriber. - ## - ## Topic match performance (when publishing) may degrade if messages - ## are mostly published to topics with large number of levels. - ## - ## NOTE: This is a cluster-wide configuration. - ## It requires all nodes to be stopped before changing it. - ## - ## @doc broker.perf.trie_compaction - ## ValueType: Boolean - ## Default: true - perf.trie_compaction = true -} - -##================================================================== -## System Topic -##================================================================== - -sys_topics { - ## System interval of publishing $SYS messages. - ## - ## @doc broker.sys_msg_interval - ## ValueType: Duration | disabled - ## Default: 1m - sys_msg_interval = 1m - - ## System heartbeat interval of publishing following heart beat message: - ## - "$SYS/brokers//uptime" - ## - "$SYS/brokers//datetime" - ## - ## @doc broker.sys_heartbeat_interval - ## ValueType: Duration - ## Default: 30s | disabled - sys_heartbeat_interval = 30s - - ## Whether to enable Client lifecycle event messages publishing. - ## The following options are not only for enabling MQTT client event messages - ## publish but also for Gateway clients. However, these kinds of clients type - ## are distinguished by the Topic prefix: - ## - For the MQTT client, its event topic format is: - ## $SYS/broker//clients// - ## - For the Gateway client, it is - ## $SYS/broker//gateway//clients// - sys_event_messages { - ## Enable to publish client connected event messages. - ## - Topic: "$SYS/broker//clients//connected" - client_connected = true - ## Enable to publish client disconnected event messages. - ## - Topic: "$SYS/broker//clients//disconnected" - client_disconnected = true - ## Enable to publish event message that client subscribed a topic successfully. - ## - Topic: "$SYS/broker//clients//subscribed" - client_subscribed = false - ## Enable to publish event message that client unsubscribed a topic successfully. - ## - Topic: "$SYS/broker//clients//unsubscribed" - client_unsubscribed = false + websocket.mqtt_path: "/mqtt" + ssl { + keyfile = "{{ platform_etc_dir }}/certs/key.pem" + certfile = "{{ platform_etc_dir }}/certs/cert.pem" + cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" } } -##================================================================== -## System Monitor -##================================================================== -sysmon { - ## The time interval for the periodic process limit check - ## - ## @doc sysmon.vm.process_check_interval - ## ValueType: Duration - ## Default: 30s - vm.process_check_interval = 30s - - ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is set. - ## - ## @doc sysmon.vm.process_high_watermark - ## ValueType: Percentage - ## Default: 80% - vm.process_high_watermark = 80% - - ## The threshold, as percentage of processes, for how many processes can simultaneously exist at the local node before the corresponding alarm is clear. - ## - ## @doc sysmon.vm.process_low_watermark - ## ValueType: Percentage - ## Default: 60% - vm.process_low_watermark = 60% - - ## Enable Long GC monitoring. - ## Notice: don't enable this monitor in production, because it adds overhead to garbage collection. - ## - ## @doc sysmon.vm.long_gc - ## ValueType: Duration | disabled - ## Default: disabled - vm.long_gc = disabled - - ## Enable Long Schedule monitoring. - ## - ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 - ## - ## @doc sysmon.vm.long_schedule - ## ValueType: Duration | disabled - ## Default: disabled - vm.long_schedule = 240ms - - ## Enable Large Heap monitoring. - ## - ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 - ## - ## @doc sysmon.vm.large_heap - ## ValueType: Size | disabled - ## Default: 32MB - vm.large_heap = 32MB - - ## Enable Busy Port monitoring. - ## - ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 - ## - ## @doc sysmon.vm.busy_port - ## ValueType: Boolean - ## Default: true - vm.busy_port = true - - ## Enable Busy Dist Port monitoring. - ## - ## See: http://erlang.org/doc/man/erlang.html#system_monitor-2 - ## - ## @doc sysmon.vm.busy_dist_port - ## ValueType: Boolean - ## Default: true - vm.busy_dist_port = true - - ## The time interval for the periodic cpu check - ## - ## @doc sysmon.os.cpu_check_interval - ## ValueType: Duration - ## Default: 60s - os.cpu_check_interval = 60s - - ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is set. - ## - ## @doc sysmon.os.cpu_high_watermark - ## ValueType: Percentage - ## Default: 80% - os.cpu_high_watermark = 80% - - ## The threshold, as percentage of system cpu, for how much system cpu can be used before the corresponding alarm is clear. - ## - ## @doc sysmon.os.cpu_low_watermark - ## ValueType: Percentage - ## Default: 60% - os.cpu_low_watermark = 60% - - ## The time interval for the periodic memory check - ## - ## @doc sysmon.os.mem_check_interval - ## ValueType: Duration | disabled - ## Default: 60s - os.mem_check_interval = 60s - - ## The threshold, as percentage of system memory, for how much system memory can be allocated before the corresponding alarm is set. - ## - ## @doc sysmon.os.sysmem_high_watermark - ## ValueType: Percentage - ## Default: 70% - os.sysmem_high_watermark = 70% - - ## The threshold, as percentage of system memory, for how much system memory can be allocated by one Erlang process before the corresponding alarm is set. - ## - ## @doc sysmon.os.procmem_high_watermark - ## ValueType: Percentage - ## Default: 5% - os.procmem_high_watermark = 5% -} - -##================================================================== -## Alarm -##================================================================== -alarm { - ## Specifies the actions to take when an alarm is activated - ## - ## @doc alarm.actions - ## ValueType: Array - ## Default: [log, publish] - actions = [log, publish] - - ## The maximum number of deactivated alarms - ## - ## @doc alarm.size_limit - ## ValueType: Integer - ## Default: 1000 - size_limit = 1000 - - ## Validity Period of deactivated alarms - ## - ## @doc alarm.validity_period - ## ValueType: Duration - ## Default: 24h - validity_period = 24h -} - -## Config references for listeners - -## Socket options for TCP connections -## See: http://erlang.org/doc/man/inet.html -example_common_tcp_options { - ## Specify the {active, N} option for this Socket. - ## - ## See: https://erlang.org/doc/man/inet.html#setopts-2 - ## - ## @doc listeners..tcp.active_n - ## ValueType: Number - ## Default: 100 - tcp.active_n = 100 - - ## TCP backlog defines the maximum length that the queue of - ## pending connections can grow to. - ## - ## @doc listeners..tcp.backlog - ## ValueType: Number - ## Range: [0, 1048576] - ## Default: 1024 - tcp.backlog = 1024 - - ## The TCP send timeout for the connections. - ## - ## @doc listeners..tcp.send_timeout - ## ValueType: Duration - ## Default: 15s - tcp.send_timeout = 15s - - ## Close the connection if send timeout. - ## - ## @doc listeners..tcp.send_timeout_close - ## ValueType: Boolean - ## Default: true - tcp.send_timeout_close = true - - ## The TCP receive buffer(os kernel) for the connections. - ## - ## @doc listeners..tcp.recbuf - ## ValueType: Size - ## Default: notset - #tcp.recbuf: 2KB - - ## The TCP send buffer(os kernel) for the connections. - ## - ## @doc listeners..tcp.sndbuf - ## ValueType: Size - ## Default: notset - #tcp.sndbuf: 4KB - - ## The size of the user-level software buffer used by the driver. - ## - ## @doc listeners..tcp.buffer - ## ValueType: Size - ## Default: notset - #tcp.buffer: 4KB - - ## The socket is set to a busy state when the amount of data queued internally - ## by the ERTS socket implementation reaches this limit. - ## - ## @doc listeners..tcp.high_watermark - ## ValueType: Size - ## Default: 1MB - tcp.high_watermark = 1MB - - ## The TCP_NODELAY flag for the connections. - ## - ## @doc listeners..tcp.nodelay - ## ValueType: Boolean - ## Default: false - tcp.nodelay = false - - ## The SO_REUSEADDR flag for the connections. - ## - ## @doc listeners..tcp.reuseaddr - ## ValueType: Boolean - ## Default: true - tcp.reuseaddr = true -} - -## Socket options for SSL connections -## See: http://erlang.org/doc/man/ssl.html -example_common_ssl_options { - - ## A performance optimization setting, it allows clients to reuse - ## pre-existing sessions, instead of initializing new ones. - ## Read more about it here. - ## - ## @doc listeners..ssl.reuse_sessions - ## ValueType: Boolean - ## Default: true - ssl.reuse_sessions = true - - ## SSL parameter renegotiation is a feature that allows a client and a server - ## to renegotiate the parameters of the SSL connection on the fly. - ## RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation, - ## you drop support for the insecure renegotiation, prone to MitM attacks. - ## - ## @doc listeners..ssl.secure_renegotiate - ## ValueType: Boolean - ## Default: true - ssl.secure_renegotiate = true - - ## In protocols that support client-initiated renegotiation, - ## the cost of resources of such an operation is higher for the server than the client. - ## This can act as a vector for denial of service attacks. - ## The SSL application already takes measures to counter-act such attempts, - ## but client-initiated renegotiation can be strictly disabled by setting this option to false. - ## The default value is true. Note that disabling renegotiation can result in - ## long-lived connections becoming unusable due to limits on - ## the number of messages the underlying cipher suite can encipher. - ssl.client_renegotiation = true - - ## An important security setting, it forces the cipher to be set based - ## on the server-specified order instead of the client-specified order, - ## hence enforcing the (usually more properly configured) security - ## ordering of the server administrator. - ## - ## @doc listeners..ssl.honor_cipher_order - ## ValueType: Boolean - ## Default: true - ssl.honor_cipher_order = true - - # ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] - # TLS 1.3: "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256" - # TLS 1-1.2 "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - # PSK: "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - # NOTE: If PSK cipher-suites are intended, tlsv1.3 should not be enabled in 'versions' config - # NOTE: by default, ALL ciphers are enabled - # ssl.ciphers = "" - - ## TLS Handshake timeout. - ## - ## @doc listeners..ssl.handshake_timeout - ## ValueType: Duration - ## Default: 15s - ssl.handshake_timeout = 15s - - ## Maximum number of non-self-issued intermediate certificates that - ## can follow the peer certificate in a valid certification path. - ## - ## @doc listeners..ssl.depth - ## ValueType: Integer - ## Default: 10 - ssl.depth = 10 - - ## Path to the file containing the user's private PEM-encoded key. - ## - ## @doc listeners..ssl.keyfile - ## ValueType: File - ## Default: "{{ platform_etc_dir }}/certs/key.pem" - ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - - ## Path to a file containing the user certificate. - ## - ## @doc listeners..ssl.certfile - ## ValueType: File - ## Default: "{{ platform_etc_dir }}/certs/cert.pem" - ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" - - ## Path to the file containing PEM-encoded CA certificates. The CA certificates - ## are used during server authentication and when building the client certificate chain. - ## - ## @doc listeners..ssl.cacertfile - ## ValueType: File - ## Default: "{{ platform_etc_dir }}/certs/cacert.pem" - ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - - ## Maximum number of non-self-issued intermediate certificates that - ## can follow the peer certificate in a valid certification path. - ## - ## @doc listeners..ssl.depth - ## ValueType: Number - ## Default: 10 - ssl.depth = 10 - - ## String containing the user's password. Only used if the private keyfile - ## is password-protected. - ## - ## See: listener.ssl.$name.key_password - ## - ## @doc listeners..ssl.depth - ## ValueType: String - ## Default: "" - #ssl.key_password: "" - - ## The Ephemeral Diffie-Helman key exchange is a very effective way of - ## ensuring Forward Secrecy by exchanging a set of keys that never hit - ## the wire. Since the DH key is effectively signed by the private key, - ## it needs to be at least as strong as the private key. In addition, - ## the default DH groups that most of the OpenSSL installations have - ## are only a handful (since they are distributed with the OpenSSL - ## package that has been built for the operating system it’s running on) - ## and hence predictable (not to mention, 1024 bits only). - ## In order to escape this situation, first we need to generate a fresh, - ## strong DH group, store it in a file and then use the option above, - ## to force our SSL application to use the new DH group. Fortunately, - ## OpenSSL provides us with a tool to do that. Simply run: - ## openssl dhparam -out dh-params.pem 2048 - ## - ## @doc listeners..ssl.dhfile - ## ValueType: File - ## Default: "{{ platform_etc_dir }}/certs/dh-params.pem" - #ssl.dhfile: "{{ platform_etc_dir }}/certs/dh-params.pem" - - ## A server only does x509-path validation in mode verify_peer, - ## as it then sends a certificate request to the client (this - ## message is not sent if the verify option is verify_none). - ## You can then also want to specify option fail_if_no_peer_cert. - ## More information at: http://erlang.org/doc/man/ssl.html - ## - ## @doc listeners..ssl.verify - ## ValueType: verify_peer | verify_none - ## Default: verify_none - ssl.verify = verify_none - - ## Used together with {verify, verify_peer} by an SSL server. If set to true, - ## the server fails if the client does not have a certificate to send, that is, - ## sends an empty certificate. - ## - ## @doc listeners..ssl.fail_if_no_peer_cert - ## ValueType: Boolean - ## Default: true - ssl.fail_if_no_peer_cert = false - -} - -## Socket options for websocket connections -example_common_websocket_options { - ## The path of WebSocket MQTT endpoint - ## - ## @doc listeners..websocket.mqtt_path - ## ValueType: Path - ## Default: "/mqtt" - websocket.mqtt_path = "/mqtt" - - ## Whether a WebSocket message is allowed to contain multiple MQTT packets - ## - ## @doc listeners..websocket.mqtt_piggyback - ## ValueType: single | multiple - ## Default: multiple - websocket.mqtt_piggyback = multiple - - ## The compress flag for external WebSocket connections. - ## - ## If this Value is set true,the websocket message would be compressed - ## - ## @doc listeners..websocket.compress - ## ValueType: Boolean - ## Default: false - websocket.compress = false - - ## The idle timeout for external WebSocket connections. - ## - ## @doc listeners..websocket.idle_timeout - ## ValueType: Duration | infinity - ## Default: infinity - websocket.idle_timeout = infinity - - ## The max frame size for external WebSocket connections. - ## - ## @doc listeners..websocket.max_frame_size - ## ValueType: Size - ## Default: infinity - websocket.max_frame_size = infinity - - ## If set to true, the server fails if the client does not - ## have a Sec-WebSocket-Protocol to send. - ## Set to false for WeChat MiniApp. - ## - ## @doc listeners..websocket.fail_if_no_subprotocol - ## ValueType: Boolean - ## Default: true - websocket.fail_if_no_subprotocol = true - - ## Supported subprotocols - ## - ## @doc listeners..websocket.supported_subprotocols - ## ValueType: String - ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 - websocket.supported_subprotocols = "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" - - ## Enable origin check in header for websocket connection - ## - ## @doc listeners..websocket.check_origin_enable - ## ValueType: Boolean - ## Default: false - websocket.check_origin_enable = false - - ## Allow origin to be absent in header in websocket connection - ## when check_origin_enable is true - ## - ## @doc listeners..websocket.allow_origin_absence - ## ValueType: Boolean - ## Default: true - websocket.allow_origin_absence = true - - ## Comma separated list of allowed origin in header for websocket connection - ## - ## @doc listeners..websocket.check_origins - ## ValueType: String - ## Examples: - ## local http dashboard url - ## check_origins: "http://localhost:18083, http://127.0.0.1:18083" - ## Default: "" - websocket.check_origins = "http://localhost:18083, http://127.0.0.1:18083" - - ## Specify which HTTP header for real source IP if the EMQX cluster is - ## deployed behind NGINX or HAProxy. - ## - ## @doc listeners..websocket.proxy_address_header - ## ValueType: String - ## Default: X-Forwarded-For - websocket.proxy_address_header = X-Forwarded-For - - ## Specify which HTTP header for real source port if the EMQX cluster is - ## deployed behind NGINX or HAProxy. - ## - ## @doc listeners..websocket.proxy_port_header - ## ValueType: String - ## Default: X-Forwarded-Port - websocket.proxy_port_header = X-Forwarded-Port - - websocket.deflate_opts { - ## The level of deflate options for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.level - ## ValueType: none | default | best_compression | best_speed - ## Default: default - level = default - - ## The mem_level of deflate options for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.mem_level - ## ValueType: Integer - ## Range: [1,9] - ## Default: 8 - mem_level = 8 - - ## The strategy of deflate options for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.strategy - ## ValueType: default | filtered | huffman_only | rle - ## Default: default - strategy = default - - ## The deflate option for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.server_context_takeover - ## ValueType: takeover | no_takeover - ## Default: takeover - server_context_takeover = takeover - - ## The deflate option for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.client_context_takeover - ## ValueType: takeover | no_takeover - ## Default: takeover - client_context_takeover = takeover - - ## The deflate options for external WebSocket connections. - ## - ## - ## @doc listeners..websocket.deflate_opts.server_max_window_bits - ## ValueType: Integer - ## Range: [8,15] - ## Default: 15 - server_max_window_bits = 15 - - ## The deflate options for external WebSocket connections. - ## - ## @doc listeners..websocket.deflate_opts.client_max_window_bits - ## ValueType: Integer - ## Range: [8,15] - ## Default: 15 - client_max_window_bits = 15 - } -} - -persistent_session_store { - ## Enable/disable internal persistent session store. - ## - ## @doc persistent_session_store.enabled - ## ValueType: Boolean - ## Default: false - enabled = false - - ## How long are undelivered messages retained in the store - ## - ## @doc persistent_session_store.max_retain_undelivered - ## ValueType: Duration - ## Default: 1h - max_retain_undelivered = 1h - - ## The time interval in which to try to run garbage collection of persistent session messages - ## - ## @doc persistent_session_store.message_gc_interval - ## ValueType: Duration - ## Default: 1h - message_gc_interval = 1h - - ## The time interval in which to try to run garbage collection of persistent session transient data - ## - ## @doc persistent_session_store.session_message_gc_interval - ## ValueType: Duration - ## Default: 1m - session_message_gc_interval = 1m -} +//listeners.quic.default { +// bind = "0.0.0.0:14567" +// max_connections = 1024000 +// keyfile = "{{ platform_etc_dir }}/certs/key.pem" +// certfile = "{{ platform_etc_dir }}/certs/cert.pem" +// } diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 8cab6a99f..f4a5387d4 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -85,6 +85,7 @@ ]). -include("logger.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -define(CONF, conf). -define(RAW_CONF, raw_conf). @@ -320,14 +321,31 @@ init_load(SchemaMod, RawConf) when is_map(RawConf) -> LocalOverrides = read_override_conf(#{override_to => local}), Overrides = hocon:deep_merge(ClusterOverrides, LocalOverrides), RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), - %% check configs against the schema - {_AppEnvs, CheckedConf} = - check_config(SchemaMod, RawConfWithOverrides, #{}), RootNames = get_root_names(), - ok = save_to_config_map( - maps:with(get_atom_root_names(), CheckedConf), - maps:with(RootNames, RawConfWithOverrides) - ). + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides), + %% check configs against the schema + {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), + ok = save_to_config_map(CheckedConf, RawConfAll). + +%% keep the raw and non-raw conf has the same keys to make update raw conf easier. +raw_conf_with_default(SchemaMod, RootNames, RawConf) -> + Fun = fun(Name, Acc) -> + case maps:is_key(Name, RawConf) of + true -> + Acc; + false -> + {_, {_, Schema}} = lists:keyfind(Name, 1, hocon_schema:roots(SchemaMod)), + Default = + case hocon_schema:field_schema(Schema, type) of + ?ARRAY(_) -> []; + ?LAZY(?ARRAY(_)) -> []; + _ -> #{} + end, + Acc#{Name => Default} + end + end, + RawDefault = lists:foldl(Fun, #{}, RootNames), + maps:merge(RawConf, fill_defaults(SchemaMod, RawDefault, #{})). parse_hocon(Conf) -> IncDirs = include_dirs(), @@ -466,9 +484,6 @@ get_schema_mod(RootName) -> get_root_names() -> maps:get(names, persistent_term:get(?PERSIS_SCHEMA_MODS, #{names => []})). -get_atom_root_names() -> - [atom(N) || N <- get_root_names()]. - -spec save_configs(app_envs(), config(), raw_config(), raw_config(), update_opts()) -> ok | {error, term()}. save_configs(_AppEnvs, Conf, RawConf, OverrideConf, Opts) -> diff --git a/apps/emqx_authn/etc/emqx_authn.conf b/apps/emqx_authn/etc/emqx_authn.conf index 7503f3b89..8b1378917 100644 --- a/apps/emqx_authn/etc/emqx_authn.conf +++ b/apps/emqx_authn/etc/emqx_authn.conf @@ -1 +1 @@ -authentication: [] + diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index f35cf7abe..844d8a2e1 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -1,67 +1,15 @@ authorization { - sources = [ - # { - # type: http - # url: "https://emqx.com" - # headers: { - # Accept: "application/json" - # Content-Type: "application/json" - # } - # }, - # { - # type: mysql - # server: "127.0.0.1:3306" - # database: mqtt - # pool_size: 1 - # username: root - # password: public - # auto_reconnect: true - # ssl: { - # enable: true - # cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" - # certfile: "{{ platform_etc_dir }}/certs/client-cert.pem" - # keyfile: "{{ platform_etc_dir }}/certs/client-key.pem" - # } - # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or clientid = ${clientid}" - # }, - # { - # type: postgresql - # server: "127.0.0.1:5432" - # database: mqtt - # pool_size: 1 - # username: root - # password: public - # auto_reconnect: true - # ssl: {enable: false} - # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = ${peerhost} or username = ${username} or username = '$all' or clientid = ${clientid}" - # }, - # { - # type: redis - # server: "127.0.0.1:6379" - # database: 0 - # pool_size: 1 - # password: public - # auto_reconnect: true - # ssl: {enable: false} - # cmd: "HGETALL mqtt_authz:${username}" - # }, - # { - # type: mongodb - # mongo_type: single - # server: "127.0.0.1:27017" - # pool_size: 1 - # database: mqtt - # ssl: {enable: false} - # collection: mqtt_authz - # filter: { "$or": [ { "username": "${username}" }, { "clientid": "${clientid}" } ] } - # }, - { - type: built_in_database - }, - { - type: file - # file is loaded into cache - path: "{{ platform_etc_dir }}/acl.conf" - } - ] + cache: { + enable: true + max_size: 32 + ttl: "60s" + } + deny_action: ignore + no_match: allow + sources: [ + { + type: file + path: "{{ platform_etc_dir }}/acl.conf" + } + ] } diff --git a/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf b/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf index 724a52faf..e69de29bb 100644 --- a/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf +++ b/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf @@ -1,45 +0,0 @@ - -## When the device logs in successfully, -## the auto-subscribe will complete the subscription for the device -## according to the preset topics list. -## Placeholders are supported. -## -## - ${clientid} -## - ${username} -## - ${host} -## - ${port} -## -## Subscription options can be set at the same time. -## Please refer to the official MQTT definition for these configuration items. -## - qos -## - rh -## - rap -## - nl -## -auto_subscribe { - topics = [ - ## { - ## topic = "/c/${clientid}" - ## qos = 0 - ## rh = 0 - ## rap = 0 - ## nl = 0 - ## }, - ## { - ## topic = "/u/${username}" - ## }, - ## { - ## topic = "/h/${host}" - ## qos = 2 - ## }, - ## { - ## topic = "/p/${port}" - ## }, - ## { - ## topic = "/topic/abc" - ## }, - ## { - ## topic = "/client/${clientid}/username/${username}/host/${host}/port/${port}" - ## } - ] -} diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_app.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_app.erl index 9c708b3eb..7bac3d9fa 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_app.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_app.erl @@ -21,8 +21,8 @@ -export([start/2, stop/1]). start(_StartType, _StartArgs) -> - {ok, Sup} = emqx_auto_subscribe_sup:start_link(), ok = emqx_auto_subscribe:load(), + {ok, Sup} = emqx_auto_subscribe_sup:start_link(), {ok, Sup}. stop(_State) -> diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl index 68897b802..9531ac83f 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl @@ -38,7 +38,7 @@ fields("auto_subscribe") -> {topics, hoconsc:mk( hoconsc:array(hoconsc:ref(?MODULE, "topic")), - #{desc => ?DESC(auto_subscribe)} + #{desc => ?DESC(auto_subscribe), default => []} )} ]; fields("topic") -> diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf index 19eea5d93..e69de29bb 100644 --- a/apps/emqx_bridge/etc/emqx_bridge.conf +++ b/apps/emqx_bridge/etc/emqx_bridge.conf @@ -1,59 +0,0 @@ -##-------------------------------------------------------------------- -## EMQX Bridge -##-------------------------------------------------------------------- - -## MQTT bridges to/from another MQTT broker -#bridges.mqtt.my_ingress_mqtt_bridge { -# enable = true -# connector = "mqtt:my_mqtt_connector" -# direction = ingress -# ## topic mappings for this bridge -# remote_topic = "aws/#" -# remote_qos = 1 -# local_topic = "from_aws/${topic}" -# local_qos = "${qos}" -# payload = "${payload}" -# retain = "${retain}" -#} -# -#bridges.mqtt.my_egress_mqtt_bridge { -# enable = true -# connector = "mqtt:my_mqtt_connector" -# direction = egress -# ## topic mappings for this bridge -# local_topic = "emqx/#" -# remote_topic = "from_emqx/${topic}" -# remote_qos = "${qos}" -# payload = "${payload}" -# retain = false -#} -# -## WebHook to an HTTP server -#bridges.webhook.my_webhook { -# enable = true -# direction = egress -# ## NOTE: we cannot use placehodler variables in the `scheme://host:port` part of the url -# url = "http://localhost:9901/messages/${topic}" -# request_timeout = "15s" -# connect_timeout = "15s" -# max_retries = 3 -# retry_interval = "10s" -# pool_type = "random" -# pool_size = 4 -# enable_pipelining = true -# ssl { -# enable = false -# keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" -# certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" -# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" -# } -# -# local_topic = "emqx_http/#" -# ## the following config entries can use placehodler variables: -# ## url, method, body, headers -# method = post -# body = "${payload}" -# headers { -# "content-type": "application/json" -# } -#} diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index 5e5883dd4..a24d5e3d4 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -6,845 +6,23 @@ ## ## The *-override.conf files are overwritten at runtime when changes ## are made from EMQX dashboard UI, management HTTP API, or CLI. +## All configuration details can be found in emqx.conf.example -##================================================================== -## Node -##================================================================== node { - ## Node name. - ## See: http://erlang.org/doc/reference_manual/distributed.html - ## - ## @doc node.name - ## ValueType: NodeName - ## Default: emqx@127.0.0.1 - name = "emqx@127.0.0.1" - - ## Cookie for distributed node communication. - ## - ## @doc node.cookie - ## ValueType: String - ## Default: emqxsecretcookie + name: "emqx@127.0.0.1" cookie = emqxsecretcookie - - ## Data dir for the node - ## - ## @doc node.data_dir - ## ValueType: Folder - ## Default: "{{ platform_data_dir }}" data_dir = "{{ platform_data_dir }}" - - ## Location of crash dump file. - ## - ## @doc node.crash_dump_file - ## ValueType: File - ## Default: "{{ platform_log_dir }}/erl_crash.dump" - crash_dump_file = "{{ platform_log_dir }}/erl_crash.dump" - - ## The number of seconds that the broker is allowed to spend writing - ## a crash dump - ## - ## @doc node.crash_dump_seconds - ## ValueType: seconds - ## Default: 30s - crash_dump_seconds = 30s - - ## The maximum size of a crash dump file in bytes. - ## - ## @doc node.crash_dump_bytes - ## ValueType: bytes - ## Default: 100MB - crash_dump_bytes = 100MB - - ## Global GC Interval. - ## - ## @doc node.global_gc_interval - ## ValueType: Duration - ## Default: 15m - global_gc_interval = 15m - - ## Sets the etc directory etc_dir = "{{ platform_etc_dir }}" - - ## Sets the net_kernel tick time in seconds. - ## Notice that all communicating nodes are to have the same - ## TickTime value specified. - ## - ## See: http://www.erlang.org/doc/man/kernel_app.html#net_ticktime - ## - ## @doc node.dist_net_ticktime - ## ValueType: Number - ## Default: 2m - dist_net_ticktime = 2m - - ## Sets the maximum depth of call stack back-traces in the exit - ## reason element of 'EXIT' tuples. - ## The flag also limits the stacktrace depth returned by - ## process_info item current_stacktrace. - ## - ## @doc node.backtrace_depth - ## ValueType: Integer - ## Range: [0,1024] - ## Default: 23 - backtrace_depth = 23 - - ## Comma-separated list of applications to start with emqx_machine. - ## These applications may restart on cluster leave/join. - ## - ## @doc node.applications - ## ValueType: String - ## Default: "gproc, esockd, ranch, cowboy, emqx" - applications = "{{ emqx_machine_boot_apps }}" - - cluster_call { - retry_interval = 1s - max_history = 100 - cleanup_interval = 5m - } - - ## Database backend - ## - ## @doc node.db_backend - ## ValueType: mnesia | rlog - ## Default: rlog - db_backend = rlog - - ## RLOG role - ## - ## @doc node.db_role - ## ValueType: core | replicant - ## Default: core - db_role = core } -##================================================================== -## Cluster -##================================================================== -cluster { - ## Cluster name. - ## - ## @doc cluster.name - ## ValueType: String - ## Default: emqxcl - name = emqxcl - - ## Enable cluster autoheal from network partition. - ## - ## @doc cluster.autoheal - ## ValueType: Boolean - ## Default: true - autoheal = true - - ## Autoclean down node. A down node will be removed from the cluster - ## if this value > 0. - ## - ## @doc cluster.autoclean - ## ValueType: Duration - ## Default: 5m - autoclean = 5m - - ## Node discovery strategy to join the cluster. - ## - ## @doc cluster.discovery_strategy - ## ValueType: manual | static | mcast | dns | etcd | k8s - ## - manual: Manual join command - ## - static: Static node list - ## - mcast: IP Multicast - ## - dns: DNS A Record - ## - etcd: etcd - ## - k8s: Kubernetes - ## - ## Default: manual - discovery_strategy = manual - - ## Replicant core nodes - ## - ## @doc cluster.core_nodes - ## ValueType: comma-separated node list - ## Default: "" - core_nodes = "" - - ##---------------------------------------------------------------- - ## Cluster using static node list - ##---------------------------------------------------------------- - static { - ## Node list of the cluster - ## - ## @doc cluster.static.seeds - ## ValueType: Array - ## Default: [] - seeds = ["emqx1@127.0.0.1", "emqx2@127.0.0.1"] - } - - ##---------------------------------------------------------------- - ## Cluster using IP Multicast - ##---------------------------------------------------------------- - mcast { - ## IP Multicast Address. - ## - ## @doc cluster.mcast.addr - ## ValueType: IPAddress - ## Default: "239.192.0.1" - addr = "239.192.0.1" - - ## Multicast Ports. - ## - ## @doc cluster.mcast.ports - ## ValueType: Array - ## Default: [4369, 4370] - ports = [4369, 4370] - - ## Multicast Iface. - ## - ## @doc cluster.mcast.iface - ## ValueType: IPAddress - ## Default: "0.0.0.0" - iface = "0.0.0.0" - - ## Multicast Ttl. - ## - ## @doc cluster.mcast.ttl - ## ValueType: Integer - ## Range: [0,255] - ## Default: 255 - ttl = 255 - - ## Multicast loop. - ## - ## @doc cluster.mcast.loop - ## ValueType: Boolean - ## Default: true - loop = true - } - - ##---------------------------------------------------------------- - ## Cluster using DNS A records - ##---------------------------------------------------------------- - dns { - ## DNS name. - ## - ## @doc cluster.dns.name - ## ValueType: String - ## Default: localhost - name = localhost - - ## The App name is used to build 'node.name' with IP address. - ## - ## @doc cluster.dns.app - ## ValueType: String - ## Default: emqx - app = emqx - } - - ##---------------------------------------------------------------- - ## Cluster using etcd - ##---------------------------------------------------------------- - etcd { - ## Etcd server list, separated by ','. - ## - ## @doc cluster.etcd.server - ## ValueType: URL - ## Required: true - server = "http://127.0.0.1:2379" - - ## The prefix helps build nodes path in etcd. Each node in the cluster - ## will create a path in etcd: v2/keys/// - ## - ## @doc cluster.etcd.prefix - ## ValueType: String - ## Default: emqxcl - prefix = emqxcl - - ## The TTL for node's path in etcd. - ## - ## @doc cluster.etcd.node_ttl - ## ValueType: Duration - ## Default: 1m - node_ttl = 1m - - ## Path to the file containing the user's private PEM-encoded key. - ## - ## @doc cluster.etcd.ssl.keyfile - ## ValueType: File - ## Default: "{{ platform_etc_dir }}/certs/key.pem" - ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" - - ## Path to a file containing the user certificate. - ## - ## @doc cluster.etcd.ssl.certfile - ## ValueType: File - ## Default: "{{ platform_etc_dir }}/certs/cert.pem" - ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" - - ## Path to the file containing PEM-encoded CA certificates. The CA certificates - ## are used during server authentication and when building the client certificate chain. - ## - ## @doc cluster.etcd.ssl.cacertfile - ## ValueType: File - ## Default: "{{ platform_etc_dir }}/certs/cacert.pem" - ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" - } - - ##---------------------------------------------------------------- - ## Cluster using Kubernetes - ##---------------------------------------------------------------- - k8s { - ## Kubernetes API server list, separated by ','. - ## - ## @doc cluster.k8s.apiserver - ## ValueType: URL - ## Required: true - apiserver = "http://10.110.111.204:8080" - - ## The service name helps lookup EMQ nodes in the cluster. - ## - ## @doc cluster.k8s.service_name - ## ValueType: String - ## Default: emqx - service_name = emqx - - ## The address type is used to extract host from k8s service. - ## - ## @doc cluster.k8s.address_type - ## ValueType: ip | dns | hostname - ## Default: ip - address_type = ip - - ## The app name helps build 'node.name'. - ## - ## @doc cluster.k8s.app_name - ## ValueType: String - ## Default: emqx - app_name = emqx - - ## The suffix added to dns and hostname get from k8s service - ## - ## @doc cluster.k8s.suffix - ## ValueType: String - ## Default: "pod.local" - suffix = "pod.local" - - ## Kubernetes Namespace - ## - ## @doc cluster.k8s.namespace - ## ValueType: String - ## Default: default - namespace = default - } -} - -##================================================================== -## Log -##================================================================== log { - ##---------------------------------------------------------------- - ## The console log handler send log messages to emqx console - ##---------------------------------------------------------------- - console_handler { - ## Log to single line - ## @doc log.console_handler..enable - ## ValueType: Boolean - ## Default: false - enable = false - - ## The log level of this handler - ## All the log messages with levels lower than this level will - ## be dropped. - ## - ## @doc log.console_handler..level - ## ValueType: debug | info | notice | warning | error | critical | alert | emergency - ## Default: warning - level = warning - - ## Timezone offset to display in logs - ## - ## @doc log.console_handler..time_offset - ## ValueType: system | utc | String - ## - "system" use system zone - ## - "utc" for Universal Coordinated Time (UTC) - ## - "+hh:mm" or "-hh:mm" for a specified offset - ## Default: system - time_offset = system - - ## Limits the total number of characters printed for each log event. - ## - ## @doc log.console_handler..chars_limit - ## ValueType: unlimited | Integer - ## Range: [0, +Inf) - ## Default: unlimited - chars_limit = unlimited - - ## Maximum depth for Erlang term log formatting - ## and Erlang process message queue inspection. - ## - ## @doc log.console_handler..max_depth - ## ValueType: unlimited | Integer - ## Default: 100 - max_depth = 100 - - ## Log formatter - ## @doc log.console_handler..formatter - ## ValueType: text | json - ## Default: text - formatter = text - - ## Log to single line - ## @doc log.console_handler..single_line - ## ValueType: Boolean - ## Default: true - single_line = true - - ## The max allowed queue length before switching to sync mode. - ## - ## Log overload protection parameter. If the message queue grows - ## larger than this value the handler switches from anync to sync mode. - ## - ## @doc log.console_handler..sync_mode_qlen - ## ValueType: Integer - ## Range: [0, ${log.console_handler..drop_mode_qlen}] - ## Default: 100 - sync_mode_qlen = 100 - - ## The max allowed queue length before switching to drop mode. - ## - ## Log overload protection parameter. When the message queue grows - ## larger than this threshold, the handler switches to a mode in which - ## it drops all new events that senders want to log. - ## - ## @doc log.console_handler..drop_mode_qlen - ## ValueType: Integer - ## Range: [${log.console_handler..sync_mode_qlen}, ${log.console_handler..flush_qlen}] - ## Default: 3000 - drop_mode_qlen = 3000 - - ## The max allowed queue length before switching to flush mode. - ## - ## Log overload protection parameter. If the length of the message queue - ## grows larger than this threshold, a flush (delete) operation takes place. - ## To flush events, the handler discards the messages in the message queue - ## by receiving them in a loop without logging. - ## - ## @doc log.console_handler..flush_qlen - ## ValueType: Integer - ## Range: [${log.console_handler..drop_mode_qlen}, infinity) - ## Default: 8000 - flush_qlen = 8000 - - ## Kill the log handler when it gets overloaded. - ## - ## Log overload protection parameter. It is possible that a handler, - ## even if it can successfully manage peaks of high load without crashing, - ## can build up a large message queue, or use a large amount of memory. - ## We could kill the log handler in these cases and restart it after a - ## few seconds. - ## - ## @doc log.console_handler..overload_kill.enable - ## ValueType: Boolean - ## Default: true - overload_kill.enable = true - - ## The max allowed queue length before killing the log handler. - ## - ## Log overload protection parameter. This is the maximum allowed queue - ## length. If the message queue grows larger than this, the handler - ## process is terminated. - ## - ## @doc log.console_handler..overload_kill.qlen - ## ValueType: Integer - ## Range: [0, 1048576] - ## Default: 20000 - overload_kill.qlen = 20000 - - ## The max allowed memory size before killing the log handler. - ## - ## Log overload protection parameter. This is the maximum memory size - ## that the handler process is allowed to use. If the handler grows - ## larger than this, the process is terminated. - ## - ## @doc log.console_handler..overload_kill.mem_size - ## ValueType: Size - ## Default: 30MB - overload_kill.mem_size = 30MB - - ## Restart the log handler after some seconds. - ## - ## Log overload protection parameter. If the handler is terminated, - ## it restarts automatically after a delay specified in seconds. - ## - ## @doc log.console_handler..overload_kill.restart_after - ## ValueType: Duration - ## Default: 5s - overload_kill.restart_after = 5s - - ## Controlling Bursts of Log Requests. - ## - ## Log overload protection parameter. Large bursts of log events - many - ## events received by the handler under a short period of time - can - ## potentially cause problems. By specifying the maximum number of events - ## to be handled within a certain time frame, the handler can avoid - ## choking the log with massive amounts of printouts. - ## - ## Note that there would be no warning if any messages were - ## dropped because of burst control. - ## - ## @doc log.console_handler..burst_limit.enable - ## ValueType: Boolean - ## Default: false - burst_limit.enable = false - - ## This config controls the maximum number of events to handle within - ## a time frame. After the limit is reached, successive events are - ## dropped until the end of the time frame defined by `window_time`. - ## - ## @doc log.console_handler..burst_limit.max_count - ## ValueType: Integer - ## Default: 10000 - burst_limit.max_count = 10000 - - ## See the previous description of burst_limit_max_count. - ## - ## @doc log.console_handler..burst_limit.window_time - ## ValueType: duration - ## Default: 1s - burst_limit.window_time = 1s - } - - ##---------------------------------------------------------------- - ## The file log handlers send log messages to files - ##---------------------------------------------------------------- - ## file_handlers. file_handlers.default { - enable = true - ## The log level filter of this handler - ## All the log messages with levels lower than this level will - ## be dropped. - ## - ## @doc log.file_handlers..level - ## ValueType: debug | info | notice | warning | error | critical | alert | emergency - ## Default: warning level = warning - - ## The log file for specified level. - ## - ## If `rotation` is disabled, this is the file of the log files. - ## - ## If `rotation` is enabled, this is the base name of the files. - ## Each file in a rotated log is named .N, where N is an integer. - ## - ## Note: Log files for a specific log level will only contain all the logs - ## that higher than or equal to that level - ## - ## @doc log.file_handlers..file - ## ValueType: File - ## Required: true file = "{{ platform_log_dir }}/emqx.log" - - ## Enables the log rotation. - ## With this enabled, new log files will be created when the current - ## log file is full, max to `rotation_count` files will be created. - ## - ## @doc log.file_handlers..rotation.enable - ## ValueType: Boolean - ## Default: true - rotation.enable = true - - ## Maximum rotation count of log files. - ## - ## @doc log.file_handlers..rotation.count - ## ValueType: Integer - ## Range: [1, 2048] - ## Default: 10 - rotation.count = 10 - - ## Maximum size of each log file. - ## - ## If the max_size reached and `rotation` is disabled, the handler - ## will stop sending log messages, if the `rotation` is enabled, - ## the file rotates. - ## - ## @doc log.file_handlers..max_size - ## ValueType: Size | infinity - ## Default: 10MB - max_size = 10MB - - ## Timezone offset to display in logs - ## - ## @doc log.file_handlers..time_offset - ## ValueType: system | utc | String - ## - "system" use system zone - ## - "utc" for Universal Coordinated Time (UTC) - ## - "+hh:mm" or "-hh:mm" for a specified offset - ## Default: system - time_offset = system - - ## Limits the total number of characters printed for each log event. - ## - ## @doc log.file_handlers..chars_limit - ## ValueType: unlimited | Integer - ## Range: [0, +Inf) - ## Default: unlimited - chars_limit = unlimited - - ## Maximum depth for Erlang term log formatting - ## and Erlang process message queue inspection. - ## - ## @doc log.file_handlers..max_depth - ## ValueType: unlimited | Integer - ## Default: 100 - max_depth = 100 - - ## Log formatter - ## @doc log.file_handlers..formatter - ## ValueType: text | json - ## Default: text - formatter = text - - ## Log to single line - ## @doc log.file_handlers..single_line - ## ValueType: Boolean - ## Default: true - single_line = true - - ## The max allowed queue length before switching to sync mode. - ## - ## Log overload protection parameter. If the message queue grows - ## larger than this value the handler switches from anync to sync mode. - ## - ## @doc log.file_handlers..sync_mode_qlen - ## ValueType: Integer - ## Range: [0, ${log.file_handlers..drop_mode_qlen}] - ## Default: 100 - sync_mode_qlen = 100 - - ## The max allowed queue length before switching to drop mode. - ## - ## Log overload protection parameter. When the message queue grows - ## larger than this threshold, the handler switches to a mode in which - ## it drops all new events that senders want to log. - ## - ## @doc log.file_handlers..drop_mode_qlen - ## ValueType: Integer - ## Range: [${log.file_handlers..sync_mode_qlen}, ${log.file_handlers..flush_qlen}] - ## Default: 3000 - drop_mode_qlen = 3000 - - ## The max allowed queue length before switching to flush mode. - ## - ## Log overload protection parameter. If the length of the message queue - ## grows larger than this threshold, a flush (delete) operation takes place. - ## To flush events, the handler discards the messages in the message queue - ## by receiving them in a loop without logging. - ## - ## @doc log.file_handlers..flush_qlen - ## ValueType: Integer - ## Range: [${log.file_handlers..drop_mode_qlen}, infinity) - ## Default: 8000 - flush_qlen = 8000 - - ## Kill the log handler when it gets overloaded. - ## - ## Log overload protection parameter. It is possible that a handler, - ## even if it can successfully manage peaks of high load without crashing, - ## can build up a large message queue, or use a large amount of memory. - ## We could kill the log handler in these cases and restart it after a - ## few seconds. - ## - ## @doc log.file_handlers..overload_kill.enable - ## ValueType: Boolean - ## Default: true - overload_kill.enable = true - - ## The max allowed queue length before killing the log handler. - ## - ## Log overload protection parameter. This is the maximum allowed queue - ## length. If the message queue grows larger than this, the handler - ## process is terminated. - ## - ## @doc log.file_handlers..overload_kill.qlen - ## ValueType: Integer - ## Range: [0, 1048576] - ## Default: 20000 - overload_kill.qlen = 20000 - - ## The max allowed memory size before killing the log handler. - ## - ## Log overload protection parameter. This is the maximum memory size - ## that the handler process is allowed to use. If the handler grows - ## larger than this, the process is terminated. - ## - ## @doc log.file_handlers..overload_kill.mem_size - ## ValueType: Size - ## Default: 30MB - overload_kill.mem_size = 30MB - - ## Restart the log handler after some seconds. - ## - ## Log overload protection parameter. If the handler is terminated, - ## it restarts automatically after a delay specified in seconds. - ## - ## @doc log.file_handlers..overload_kill.restart_after - ## ValueType: Duration - ## Default: 5s - overload_kill.restart_after = 5s - - ## Controlling Bursts of Log Requests. - ## - ## Log overload protection parameter. Large bursts of log events - many - ## events received by the handler under a short period of time - can - ## potentially cause problems. By specifying the maximum number of events - ## to be handled within a certain time frame, the handler can avoid - ## choking the log with massive amounts of printouts. - ## - ## Note that there would be no warning if any messages were - ## dropped because of burst control. - ## - ## @doc log.file_handlers..burst_limit.enable - ## ValueType: Boolean - ## Default: false - burst_limit.enable = false - - ## This config controls the maximum number of events to handle within - ## a time frame. After the limit is reached, successive events are - ## dropped until the end of the time frame defined by `window_time`. - ## - ## @doc log.file_handlers..burst_limit.max_count - ## ValueType: Integer - ## Default: 10000 - burst_limit.max_count = 10000 - - ## See the previous description of burst_limit_max_count. - ## - ## @doc log.file_handlers..burst_limit.window_time - ## ValueType: duration - ## Default: 1s - burst_limit.window_time = 1s } } -##================================================================== -## RPC -##================================================================== -rpc { - ## RPC Mode. - ## - ## @doc rpc.mode - ## ValueType: sync | async - ## Default: async - mode = async - - ## Max batch size of async RPC requests. - ## - ## NOTE: RPC batch won't work when rpc.mode = sync - ## Zero value disables rpc batching. - ## - ## @doc rpc.async_batch_size - ## ValueType: Integer - ## Range: [0, 1048576] - ## Default: 0 - async_batch_size = 256 - - ## RPC port discovery - ## - ## The strategy for discovering the RPC listening port of - ## other nodes. - ## - ## @doc cluster.discovery_strategy - ## ValueType: manual | stateless - ## - manual: discover ports by `tcp_server_port`. - ## - stateless: discover ports in a stateless manner. - ## If node name is `emqx@127.0.0.1`, where the `` is - ## an integer, then the listening port will be `5370 + ` - ## - ## Default: `stateless`. - port_discovery = stateless - - ## TCP server port for RPC. - ## - ## Only takes effect when `rpc.port_discovery` = `manual`. - ## - ## @doc rpc.tcp_server_port - ## ValueType: Integer - ## Range: [1024-65535] - ## Defaults: 5369 - tcp_server_port = 5369 - - ## Number of outgoing RPC connections. - ## - ## Set this to 1 to keep the message order sent from the same - ## client. - ## - ## @doc rpc.tcp_client_num - ## ValueType: Integer - ## Range: [1, 256] - ## Defaults: 10 - tcp_client_num = 10 - - ## RCP Client connect timeout. - ## - ## @doc rpc.connect_timeout - ## ValueType: Duration - ## Default: 5s - connect_timeout = 5s - - ## TCP send timeout of RPC client and server. - ## - ## @doc rpc.send_timeout - ## ValueType: Duration - ## Default: 5s - send_timeout = 5s - - ## Authentication timeout - ## - ## @doc rpc.authentication_timeout - ## ValueType: Duration - ## Default: 5s - authentication_timeout = 5s - - ## Default receive timeout for call() functions - ## - ## @doc rpc.call_receive_timeout - ## ValueType: Duration - ## Default: 15s - call_receive_timeout = 15s - - ## Socket idle keepalive. - ## - ## @doc rpc.socket_keepalive_idle - ## ValueType: Duration - ## Default: 900s - socket_keepalive_idle = 900s - - ## TCP Keepalive probes interval. - ## - ## @doc rpc.socket_keepalive_interval - ## ValueType: Duration - ## Default: 75s - socket_keepalive_interval = 75s - - ## Probes lost to close the connection - ## - ## @doc rpc.socket_keepalive_count - ## ValueType: Integer - ## Default: 9 - socket_keepalive_count = 9 - - ## Size of TCP send buffer. - ## - ## @doc rpc.socket_sndbuf - ## ValueType: Size - ## Default: 1MB - socket_sndbuf = 1MB - - ## Size of TCP receive buffer. - ## - ## @doc rpc.socket_recbuf - ## ValueType: Size - ## Default: 1MB - socket_recbuf = 1MB - - ## Size of user-level software socket buffer. - ## - ## @doc rpc.socket_buffer - ## ValueType: Size - ## Default: 1MB - socket_buffer = 1MB +cluster { + name = emqxcl + discovery_strategy = manual } diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index f8eabbbd1..3f843063f 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -20,7 +20,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -export([add_handler/2, remove_handler/1]). --export([get/1, get/2, get_raw/2, get_all/1]). +-export([get/1, get/2, get_raw/1, get_raw/2, get_all/1]). -export([get_by_node/2, get_by_node/3]). -export([update/3, update/4]). -export([remove/2, remove/3]). @@ -54,6 +54,10 @@ get(KeyPath, Default) -> get_raw(KeyPath, Default) -> emqx_config:get_raw(KeyPath, Default). +-spec get_raw(emqx_map_lib:config_key_path()) -> term(). +get_raw(KeyPath) -> + emqx_config:get_raw(KeyPath). + %% @doc Returns all values in the cluster. -spec get_all(emqx_map_lib:config_key_path()) -> #{node() => term()}. get_all(KeyPath) -> diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 5b3f8f1f7..8e5fc99f2 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -801,7 +801,7 @@ fields("log") -> )} ]; fields("console_handler") -> - log_handler_common_confs(); + log_handler_common_confs(false); fields("log_file_handler") -> [ {"file", @@ -825,7 +825,7 @@ fields("log_file_handler") -> desc => ?DESC("log_file_handler_max_size") } )} - ] ++ log_handler_common_confs(); + ] ++ log_handler_common_confs(true); fields("log_rotation") -> [ {"enable", @@ -1063,13 +1063,13 @@ tr_logger(Conf) -> ], [{handler, default, undefined}] ++ ConsoleHandler ++ FileHandlers. -log_handler_common_confs() -> +log_handler_common_confs(Enable) -> [ {"enable", sc( boolean(), #{ - default => false, + default => Enable, desc => ?DESC("common_handler_enable") } )}, diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf index 8929598be..e69de29bb 100644 --- a/apps/emqx_connector/etc/emqx_connector.conf +++ b/apps/emqx_connector/etc/emqx_connector.conf @@ -1,23 +0,0 @@ -#connectors.mqtt.my_mqtt_connector { -# mode = cluster_shareload -# server = "127.0.0.1:1883" -# proto_ver = "v4" -# username = "username1" -# password = "" -# clean_start = true -# keepalive = 300 -# retry_interval = "30s" -# max_inflight = 32 -# reconnect_interval = "30s" -# replayq { -# dir = "{{ platform_data_dir }}/replayq/bridge_mqtt/" -# seg_bytes = "100MB" -# offload = false -# } -# ssl { -# enable = false -# keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" -# certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" -# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" -# } -#} diff --git a/apps/emqx_dashboard/etc/emqx_dashboard.conf b/apps/emqx_dashboard/etc/emqx_dashboard.conf index f0d77c589..856779500 100644 --- a/apps/emqx_dashboard/etc/emqx_dashboard.conf +++ b/apps/emqx_dashboard/etc/emqx_dashboard.conf @@ -1,39 +1,7 @@ -##-------------------------------------------------------------------- -## EMQX Dashboard -##-------------------------------------------------------------------- - dashboard { + listeners.http { + bind = 18083 + } default_username = "admin" default_password = "public" - ## Note: sample_interval should be a divisor of 60. - ## like 1s, 2s, 3s, 5s, 10s, 12s, 15s, 20s, 30s, 60s - sample_interval = 10s - ## JWT token expiration time. - token_expired_time = 60m - listeners.http { - num_acceptors = 4 - max_connections = 512 - bind = 18083 - backlog = 512 - send_timeout = 5s - inet6 = false - ipv6_v6only = false - } - #listeners.https { - # bind = "127.0.0.1:18084" - # num_acceptors = 4 - # backlog = 512 - # send_timeout = 5s - # inet6 = false - # ipv6_v6only = false - # certfile = "etc/certs/cert.pem" - # keyfile = "etc/certs/key.pem" - # cacertfile = "etc/certs/cacert.pem" - # verify = verify_peer - # versions = ["tlsv1.3","tlsv1.2","tlsv1.1","tlsv1"] - # ciphers = ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256","TLS_CHACHA20_POLY1305_SHA256","TLS_AES_128_CCM_SHA256","TLS_AES_128_CCM_8_SHA256","ECDHE-ECDSA-AES256-GCM-SHA384","ECDHE-RSA-AES256-GCM-SHA384","ECDHE-ECDSA-AES256-SHA384","ECDHE-RSA-AES256-SHA384","ECDHE-ECDSA-DES-CBC3-SHA","ECDH-ECDSA-AES256-GCM-SHA384","ECDH-RSA-AES256-GCM-SHA384","ECDH-ECDSA-AES256-SHA384","ECDH-RSA-AES256-SHA384","DHE-DSS-AES256-GCM-SHA384","DHE-DSS-AES256-SHA256","AES256-GCM-SHA384","AES256-SHA256","ECDHE-ECDSA-AES128-GCM-SHA256","ECDHE-RSA-AES128-GCM-SHA256","ECDHE-ECDSA-AES128-SHA256","ECDHE-RSA-AES128-SHA256","ECDH-ECDSA-AES128-GCM-SHA256","ECDH-RSA-AES128-GCM-SHA256","ECDH-ECDSA-AES128-SHA256","ECDH-RSA-AES128-SHA256","DHE-DSS-AES128-GCM-SHA256","DHE-DSS-AES128-SHA256","AES128-GCM-SHA256","AES128-SHA256","ECDHE-ECDSA-AES256-SHA","ECDHE-RSA-AES256-SHA","DHE-DSS-AES256-SHA","ECDH-ECDSA-AES256-SHA","ECDH-RSA-AES256-SHA","AES256-SHA","ECDHE-ECDSA-AES128-SHA","ECDHE-RSA-AES128-SHA","DHE-DSS-AES128-SHA","ECDH-ECDSA-AES128-SHA","ECDH-RSA-AES128-SHA","AES128-SHA"] - #} - - ## CORS Support. don't set cors true if you don't know what it means. - # cors = false } diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 7a8be21ed..c358aef17 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -49,7 +49,7 @@ fields("dashboard") -> sc( emqx_schema:duration(), #{ - default => "30m", + default => "60m", desc => ?DESC(token_expired_time) } )}, diff --git a/apps/emqx_exhook/etc/emqx_exhook.conf b/apps/emqx_exhook/etc/emqx_exhook.conf index 22a6401dd..e69de29bb 100644 --- a/apps/emqx_exhook/etc/emqx_exhook.conf +++ b/apps/emqx_exhook/etc/emqx_exhook.conf @@ -1,46 +0,0 @@ -##==================================================================== -## EMQX Hooks -##==================================================================== - -exhook { - - servers = [ - ##{ - ## name = default - ## - ## Whether to automatically reconnect (initialize) the gRPC server - ## When gRPC is not available, exhook tries to request the gRPC service at - ## that interval and reinitialize the list of mounted hooks. - ## - ## Default: false - ## Value: false | Duration - ## auto_reconnect = 60s - - ## The default value or action will be returned, while the request to - ## the gRPC server failed or no available grpc server running. - ## - ## Default: deny - ## Value: ignore | deny - ## failed_action = deny - - ## The timeout to request grpc server - ## - ## Default: 5s - ## Value: Duration - ## request_timeout = 5s - - ## url = "http://127.0.0.1:9000" - ## ssl { - ## cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" - ## certfile: "{{ platform_etc_dir }}/certs/cert.pem" - ## keyfile: "{{ platform_etc_dir }}/certs/key.pem" - ## } - ## - ## The process pool size for gRPC client - ## - ## Default: Equals cpu cores - ## Value: Integer - ## pool_size = 16 - ##} - ] -} diff --git a/apps/emqx_gateway/etc/emqx_gateway.conf b/apps/emqx_gateway/etc/emqx_gateway.conf index 4ca58ea9c..8b1378917 100644 --- a/apps/emqx_gateway/etc/emqx_gateway.conf +++ b/apps/emqx_gateway/etc/emqx_gateway.conf @@ -1,7 +1 @@ -##-------------------------------------------------------------------- -## EMQX Gateway configurations -##-------------------------------------------------------------------- -## No gateway by default. -## -## If you want to get how to config it, please see emqx_gateway.conf.example. diff --git a/apps/emqx_modules/etc/emqx_modules.conf b/apps/emqx_modules/etc/emqx_modules.conf index 45549edec..8b1378917 100644 --- a/apps/emqx_modules/etc/emqx_modules.conf +++ b/apps/emqx_modules/etc/emqx_modules.conf @@ -1,40 +1 @@ -delayed { - enable = true - ## 0 is no limit - max_delayed_messages = 0 -} - -observer_cli { - enable = true -} - -telemetry { - enable = true -} - -topic_metrics: [ - #{topic: "test/1"} -] - -rewrite: [ - # { - # action = publish - # source_topic = "x/#" - # re = "^x/y/(.+)$" - # dest_topic = "z/y/$1" - # }, - # { - # action = subscribe - # source_topic = "x1/#" - # re = "^x1/y/(.+)$" - # dest_topic = "z1/y/$1" - # }, - # { - # action = all - # source_topic = "x2/#" - # re = "^x2/y/(.+)$" - # dest_topic = "z2/y/$1" - # } -] - diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 3d13f91ef..72fd15d88 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -382,7 +382,7 @@ do_publish(Key = {Ts, _Id}, Now, Acc) when Ts =< Now -> delayed_count() -> mnesia:table_info(?TAB, size). enable(Enable) -> - case emqx:get_raw_config([delayed]) of + case emqx_conf:get_raw([delayed]) of #{<<"enable">> := Enable} -> ok; Cfg -> diff --git a/apps/emqx_modules/src/emqx_modules_app.erl b/apps/emqx_modules/src/emqx_modules_app.erl index 0b9cf0123..3f5d4b187 100644 --- a/apps/emqx_modules/src/emqx_modules_app.erl +++ b/apps/emqx_modules/src/emqx_modules_app.erl @@ -35,7 +35,7 @@ stop(_State) -> maybe_enable_modules() -> emqx_conf:get([delayed, enable], true) andalso emqx_delayed:enable(), emqx_modules_conf:is_telemetry_enabled() andalso emqx_telemetry:enable(), - emqx_conf:get([observer_cli, enable], true) andalso emqx_observer_cli:enable(), + emqx_observer_cli:enable(), emqx_conf_cli:load(), ok = emqx_rewrite:enable(), emqx_topic_metrics:enable(), diff --git a/apps/emqx_modules/src/emqx_modules_conf.erl b/apps/emqx_modules/src/emqx_modules_conf.erl index dbb1c8f39..448dee817 100644 --- a/apps/emqx_modules/src/emqx_modules_conf.erl +++ b/apps/emqx_modules/src/emqx_modules_conf.erl @@ -85,7 +85,7 @@ remove_topic_metrics(Topic) -> -spec is_telemetry_enabled() -> boolean(). is_telemetry_enabled() -> IsOfficial = emqx_telemetry:official_version(emqx_release:version()), - emqx:get_config([telemetry, enable], IsOfficial). + emqx_conf:get([telemetry, enable], IsOfficial). -spec set_telemetry_status(boolean()) -> ok | {error, term()}. set_telemetry_status(Status) -> diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index c8c26b20f..e1fb8c8c3 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -39,11 +39,11 @@ roots() -> ]. fields("telemetry") -> - [{enable, hoconsc:mk(boolean(), #{default => false, desc => "Enable telemetry."})}]; + [{enable, hoconsc:mk(boolean(), #{default => true, desc => "Enable telemetry."})}]; fields("delayed") -> [ - {enable, hoconsc:mk(boolean(), #{default => false, desc => ?DESC(enable)})}, - {max_delayed_messages, sc(integer(), #{desc => ?DESC(max_delayed_messages)})} + {enable, hoconsc:mk(boolean(), #{default => true, desc => ?DESC(enable)})}, + {max_delayed_messages, sc(integer(), #{desc => ?DESC(max_delayed_messages), default => 0})} ]; fields("rewrite") -> [ diff --git a/apps/emqx_plugins/etc/emqx_plugins.conf b/apps/emqx_plugins/etc/emqx_plugins.conf index 0a1dfb72d..e69de29bb 100644 --- a/apps/emqx_plugins/etc/emqx_plugins.conf +++ b/apps/emqx_plugins/etc/emqx_plugins.conf @@ -1,7 +0,0 @@ -plugins { - prebuilt { - } - external { - } - install_dir = "{{ platform_plugins_dir }}" -} diff --git a/apps/emqx_prometheus/etc/emqx_prometheus.conf b/apps/emqx_prometheus/etc/emqx_prometheus.conf index 6903959a5..e69de29bb 100644 --- a/apps/emqx_prometheus/etc/emqx_prometheus.conf +++ b/apps/emqx_prometheus/etc/emqx_prometheus.conf @@ -1,8 +0,0 @@ -##-------------------------------------------------------------------- -## emqx_prometheus for EMQX -##-------------------------------------------------------------------- -prometheus { - push_gateway_server = "http://127.0.0.1:9091" - interval = "15s" - enable = false -} diff --git a/apps/emqx_psk/etc/emqx_psk.conf b/apps/emqx_psk/etc/emqx_psk.conf index 9f63d1574..8b1378917 100644 --- a/apps/emqx_psk/etc/emqx_psk.conf +++ b/apps/emqx_psk/etc/emqx_psk.conf @@ -1,22 +1 @@ -##-------------------------------------------------------------------- -## EMQX PSK -##-------------------------------------------------------------------- -psk_authentication { - ## Whether to enable the PSK feature. - enable = false - - ## If init file is specified, emqx will import PSKs from the file - ## into the built-in database at startup for use by the runtime. - ## - ## The file has to be structured line-by-line, each line must be in - ## the format: : - ## init_file = "{{ platform_data_dir }}/init.psk" - - ## Specifies the separator for PSKIdentity and SharedSecret in the init file. - ## The default is colon (:) - ## separator = ":" - - ## The size of each chunk used to import to the built-in database from psk file - ## chunk_size = 50 -} diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index bb8deb677..201f52c57 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -44,6 +44,7 @@ fields() -> {enable, sc(boolean(), #{ default => false, + require => true, desc => ?DESC(enable) })}, {init_file, diff --git a/apps/emqx_retainer/etc/emqx_retainer.conf b/apps/emqx_retainer/etc/emqx_retainer.conf index d82dc10ad..e69de29bb 100644 --- a/apps/emqx_retainer/etc/emqx_retainer.conf +++ b/apps/emqx_retainer/etc/emqx_retainer.conf @@ -1,82 +0,0 @@ -##-------------------------------------------------------------------- -## EMQX Retainer -##-------------------------------------------------------------------- - -## Where to store the retained messages. -## -## Notice that all nodes in the same cluster have to be configured to -retainer { - ## enable/disable emqx_retainer - enable = true - - ## Periodic interval for cleaning up expired messages. Never clear if the value is 0. - ## - ## Value: Duration - ## - h: hour - ## - m: minute - ## - s: second - ## - ## Examples: - ## - 2h: 2 hours - ## - 30m: 30 minutes - ## - 20s: 20 seconds - ## - ## Default: 0s - msg_clear_interval = 0s - - ## Message retention time. 0 means message will never be expired. - ## - ## Default: 0s - msg_expiry_interval = 0s - - ## When the retained flag of the PUBLISH message is set and Payload is empty, - ## whether to continue to publish the message. - ## see: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718038 - ## - ## Default: false - #stop_publish_clear_msg = false - - ## The message read and deliver flow rate control - ## When a client subscribe to a wildcard topic, may many retained messages will be loaded. - ## If you don't want these data loaded to the memory all at once, you can use this to control. - ## The processing flow: - ## load batch_read_number retained message from storage -> - ## deliver -> - ## repeat this, until all retianed messages are delivered - ## - flow_control { - ## The messages batch number per read from storage. 0 means no limit - ## - ## Default: 0 - batch_read_number = 0 - - ## The number of retained message can be delivered per batch - ## Range: [0, 1000] - ## Note: If this value is too large, it may cause difficulty in applying for the token of deliver - ## - ## Default: 0 - batch_deliver_number = 0 - } - - ## Maximum retained message size. - ## - ## Value: Bytes - max_payload_size = 1MB - - ## Storage backend parameters - ## - ## Value: built_in_database - ## - backend { - - type = built_in_database - - ## storage_type: ram | disc - storage_type = ram - - ## Maximum number of retained messages. 0 means no limit. - ## - ## Value: Number >= 0 - max_retained_messages = 0 - } -} diff --git a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl index 36672bcc1..3cabead3e 100644 --- a/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl +++ b/apps/emqx_retainer/src/emqx_retainer_dispatcher.erl @@ -111,7 +111,7 @@ start_link(Pool, Id) -> init([Pool, Id]) -> erlang:process_flag(trap_exit, true), true = gproc_pool:connect_worker(Pool, {Pool, Id}), - BucketName = emqx:get_config([retainer, flow_control, batch_deliver_limiter], undefined), + BucketName = emqx_conf:get([retainer, flow_control, batch_deliver_limiter], undefined), {ok, Limiter} = emqx_limiter_server:connect(batch, BucketName), {ok, #{pool => Pool, id => Id, limiter => Limiter}}. @@ -151,7 +151,7 @@ handle_cast({dispatch, Context, Pid, Topic}, #{limiter := Limiter} = State) -> {ok, Limiter2} = dispatch(Context, Pid, Topic, undefined, Limiter), {noreply, State#{limiter := Limiter2}}; handle_cast(refresh_limiter, State) -> - BucketName = emqx:get_config([retainer, flow_control, batch_deliver_limiter]), + BucketName = emqx_conf:get([retainer, flow_control, batch_deliver_limiter]), {ok, Limiter} = emqx_limiter_server:connect(batch, BucketName), {noreply, State#{limiter := Limiter}}; handle_cast(Msg, State) -> @@ -249,7 +249,7 @@ deliver(Result, Context, Pid, Topic, Cursor, Limiter) -> false -> {ok, Limiter}; _ -> - DeliverNum = emqx:get_config([retainer, flow_control, batch_deliver_number]), + DeliverNum = emqx_conf:get([retainer, flow_control, batch_deliver_number], undefined), case DeliverNum of 0 -> do_deliver(Result, Pid, Topic), diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 8aa913da3..be2a64868 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -18,7 +18,7 @@ roots() -> ["retainer"]. fields("retainer") -> [ - {enable, sc(boolean(), enable, false)}, + {enable, sc(boolean(), enable, true)}, {msg_expiry_interval, sc( emqx_schema:duration_ms(), @@ -31,7 +31,11 @@ fields("retainer") -> msg_clear_interval, "0s" )}, - {flow_control, sc(hoconsc:ref(?MODULE, flow_control), flow_control)}, + {flow_control, + sc( + ?R_REF(flow_control), + flow_control + )}, {max_payload_size, sc( emqx_schema:bytesize(), @@ -48,7 +52,7 @@ fields("retainer") -> ]; fields(mnesia_config) -> [ - {type, sc(hoconsc:union([built_in_database]), mnesia_config_type, built_in_database)}, + {type, sc(hoconsc:enum([built_in_database]), mnesia_config_type, built_in_database)}, {storage_type, sc( hoconsc:union([ram, disc]), @@ -57,10 +61,9 @@ fields(mnesia_config) -> )}, {max_retained_messages, sc( - integer(), + non_neg_integer(), max_retained_messages, - 0, - fun is_pos_integer/1 + 0 )}, {index_specs, fun retainer_indices/1} ]; @@ -68,10 +71,9 @@ fields(flow_control) -> [ {batch_read_number, sc( - integer(), + non_neg_integer(), batch_read_number, - 0, - fun is_pos_integer/1 + 0 )}, {batch_deliver_number, sc( @@ -99,27 +101,17 @@ desc(_) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- + sc(Type, DescId) -> - hoconsc:mk(Type, #{required => true, desc => ?DESC(DescId)}). + hoconsc:mk(Type, #{desc => ?DESC(DescId)}). sc(Type, DescId, Default) -> hoconsc:mk(Type, #{default => Default, desc => ?DESC(DescId)}). -sc(Type, DescId, Default, Validator) -> - hoconsc:mk(Type, #{ - default => Default, - desc => ?DESC(DescId), - validator => Validator - }). - -is_pos_integer(V) -> - V >= 0. - backend_config() -> - sc( + hoconsc:mk( hoconsc:union([hoconsc:ref(?MODULE, mnesia_config)]), - backend, - mnesia_config + #{desc => ?DESC(backend)} ). retainer_indices(type) -> diff --git a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf index cbecb526c..e69de29bb 100644 --- a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf +++ b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf @@ -1,22 +0,0 @@ -##==================================================================== -## Rule Engine for EMQX R5.0 -##==================================================================== -rule_engine { - ignore_sys_message = true - jq_function_default_timeout = 10s - #rules.my_republish_rule { - # description = "A simple rule that republishs MQTT messages from topic 't/1' to 't/2'" - # enable = true - # sql = "SELECT * FROM \"t/1\"" - # actions = [ - # { - # function = republish - # args = { - # topic = "t/2" - # qos = "${qos}" - # payload = "${payload}" - # } - # } - # ] - #} -} diff --git a/apps/emqx_slow_subs/etc/emqx_slow_subs.conf b/apps/emqx_slow_subs/etc/emqx_slow_subs.conf index c09cc1c56..e69de29bb 100644 --- a/apps/emqx_slow_subs/etc/emqx_slow_subs.conf +++ b/apps/emqx_slow_subs/etc/emqx_slow_subs.conf @@ -1,35 +0,0 @@ -##-------------------------------------------------------------------- -## EMQX Slow Subscribers Statistics -##-------------------------------------------------------------------- - -slow_subs { - enable = false - - threshold = 500ms - ## The latency threshold for statistics, the minimum value is 100ms - ## - ## Default: 500ms - - ## The eviction time of the record, which in the statistics record table - ## - ## Default: 300s - expire_interval = 300s - - ## The maximum number of records in the slow subscription statistics record table - ## - ## Value: 10 - top_k_num = 10 - - ## The ways to calculate the latency are as follows: - ## - ## 1. whole - ## From the time the message arrives at EMQX until the message completes transmission - ## - ## 2.internal - ## From when the message arrives at EMQX until when EMQX starts delivering the message - ## - ## 3.response - ## From the time EMQX starts delivering the message, until the message completes transmission - ## Default: whole - stats_type = whole -} diff --git a/apps/emqx_statsd/etc/emqx_statsd.conf b/apps/emqx_statsd/etc/emqx_statsd.conf index 69ab6c3b7..e69de29bb 100644 --- a/apps/emqx_statsd/etc/emqx_statsd.conf +++ b/apps/emqx_statsd/etc/emqx_statsd.conf @@ -1,10 +0,0 @@ -##-------------------------------------------------------------------- -## Statsd for EMQX -##-------------------------------------------------------------------- - -statsd { - enable = false - server = "127.0.0.1:8125" - sample_time_interval = "10s" - flush_time_interval = "10s" -} diff --git a/scripts/merge-config.escript b/scripts/merge-config.escript index 49eba8169..67551bfbe 100755 --- a/scripts/merge-config.escript +++ b/scripts/merge-config.escript @@ -9,15 +9,19 @@ %% edition due to backward-compatibility reasons. -mode(compile). +-define(APPS, ["emqx", "emqx_dashboard", "emqx_authz"]). main(_) -> {ok, BaseConf} = file:read_file("apps/emqx_conf/etc/emqx_conf.conf"), Cfgs = get_all_cfgs("apps/"), - Conf = [merge(BaseConf, Cfgs), - io_lib:nl(), - "include emqx_enterprise.conf", - io_lib:nl()], + Conf = [ + merge(BaseConf, Cfgs), + io_lib:nl(), + io_lib:nl(), + "include emqx_enterprise.conf", + io_lib:nl() + ], ok = file:write_file("apps/emqx_conf/etc/emqx.conf.all", Conf), EnterpriseCfgs = get_all_cfgs("lib-ee/"), @@ -27,25 +31,33 @@ main(_) -> merge(BaseConf, Cfgs) -> lists:foldl( - fun(CfgFile, Acc) -> - case filelib:is_regular(CfgFile) of - true -> - {ok, Bin1} = file:read_file(CfgFile), - [Acc, io_lib:nl(), Bin1]; - false -> Acc - end - end, BaseConf, Cfgs). + fun(CfgFile, Acc) -> + case filelib:is_regular(CfgFile) of + true -> + {ok, Bin1} = file:read_file(CfgFile), + case string:trim(Bin1, both) of + <<>> -> Acc; + Bin2 -> [Acc, io_lib:nl(), io_lib:nl(), Bin2] + end; + false -> + Acc + end + end, + BaseConf, + Cfgs + ). get_all_cfgs(Root) -> - Apps = filelib:wildcard("*", Root) -- ["emqx_machine", "emqx_conf"], - Dirs = [filename:join([Root, App]) || App <- Apps], + Apps0 = filelib:wildcard("*", Root) -- ["emqx_machine", "emqx_conf"], + Apps1 = (Apps0 -- ?APPS) ++ lists:reverse(?APPS), + Dirs = [filename:join([Root, App]) || App <- Apps1], lists:foldl(fun get_cfgs/2, [], Dirs). get_all_cfgs(Dir, Cfgs) -> Fun = fun(E, Acc) -> - Path = filename:join([Dir, E]), - get_cfgs(Path, Acc) - end, + Path = filename:join([Dir, E]), + get_cfgs(Path, Acc) + end, lists:foldl(Fun, Cfgs, filelib:wildcard("*", Dir)). get_cfgs(Dir, Cfgs) -> From 96baf4ccdbd12227c33148b8e0e2b1a2dbb2fd50 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 23 May 2022 14:08:14 +0800 Subject: [PATCH 08/22] fix: don't add default if root_key not found --- apps/emqx/src/emqx_config.erl | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index f4a5387d4..f0db4a275 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -334,14 +334,18 @@ raw_conf_with_default(SchemaMod, RootNames, RawConf) -> true -> Acc; false -> - {_, {_, Schema}} = lists:keyfind(Name, 1, hocon_schema:roots(SchemaMod)), - Default = - case hocon_schema:field_schema(Schema, type) of - ?ARRAY(_) -> []; - ?LAZY(?ARRAY(_)) -> []; - _ -> #{} - end, - Acc#{Name => Default} + case lists:keyfind(Name, 1, hocon_schema:roots(SchemaMod)) of + false -> + Acc; + {_, {_, Schema}} -> + Default = + case hocon_schema:field_schema(Schema, type) of + ?ARRAY(_) -> []; + ?LAZY(?ARRAY(_)) -> []; + _ -> #{} + end, + Acc#{Name => Default} + end end end, RawDefault = lists:foldl(Fun, #{}, RootNames), From 239949ff0b4b9ee09418dcc89caf2887f8a5365c Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 23 May 2022 15:37:18 +0800 Subject: [PATCH 09/22] fix: elvis warning --- apps/emqx/src/emqx_config.erl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index f0db4a275..ae108c595 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -338,19 +338,20 @@ raw_conf_with_default(SchemaMod, RootNames, RawConf) -> false -> Acc; {_, {_, Schema}} -> - Default = - case hocon_schema:field_schema(Schema, type) of - ?ARRAY(_) -> []; - ?LAZY(?ARRAY(_)) -> []; - _ -> #{} - end, - Acc#{Name => Default} + Acc#{Name => schema_default(Schema)} end end end, RawDefault = lists:foldl(Fun, #{}, RootNames), maps:merge(RawConf, fill_defaults(SchemaMod, RawDefault, #{})). +schema_default(Schema) -> + case hocon_schema:field_schema(Schema, type) of + ?ARRAY(_) -> []; + ?LAZY(?ARRAY(_)) -> []; + _ -> #{} + end. + parse_hocon(Conf) -> IncDirs = include_dirs(), case do_parse_hocon(Conf, IncDirs) of From d215f5e4e60968544ff625ad1037ca94d9e2efbb Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 23 May 2022 23:31:15 +0800 Subject: [PATCH 10/22] test: add default quic conf --- apps/emqx/test/emqx_broker_SUITE.erl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index 158a5a420..430cc49c4 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -104,6 +104,23 @@ init_per_testcase(Case, Config) -> end_per_testcase(Case, Config) -> ?MODULE:Case({'end', Config}). +set_special_configs(emqx) -> + Quic = #{ + enabled => true, + bind => {{0, 0, 0, 0}, 14567}, + acceptors => 16, + max_connections => 1024000, + keyfile => "etc/certs/key.pem", + certfile => "etc/certs/cert.pem", + mountpoint => <<"">>, + zone => default, + idle_timeout => 15000 + }, + emqx_config:put_listener_conf(quic, default, [], Quic), + ok; +set_special_configs(_) -> + ok. + %%-------------------------------------------------------------------- %% PubSub Test %%-------------------------------------------------------------------- From 2b4b3d1b56e2d1070ebce9d8d55a9dd0a546fe7d Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 24 May 2022 09:27:20 +0800 Subject: [PATCH 11/22] test: debug broker segmentation fault --- apps/emqx/etc/emqx.conf | 41 ++++++++++--------- apps/emqx/src/emqx_config.erl | 15 +++++-- .../src/emqx_limiter/etc/emqx_limiter.conf | 5 --- apps/emqx/test/emqx_broker_SUITE.erl | 4 ++ 4 files changed, 37 insertions(+), 28 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 768e9a856..a19bf1dab 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -1,39 +1,40 @@ listeners.tcp.default { - bind = "0.0.0.0:1883" - max_connections = 1024000 + bind: "0.0.0.0:1883" + max_connections: 1024000 } listeners.ssl.default { - bind = "0.0.0.0:8883" - max_connections = 512000 + bind: "0.0.0.0:8883" + max_connections: 512000 ssl { - keyfile = "{{ platform_etc_dir }}/certs/key.pem" - certfile = "{{ platform_etc_dir }}/certs/cert.pem" - cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + keyfile: "{{ platform_etc_dir }}/certs/key.pem" + certfile: "{{ platform_etc_dir }}/certs/cert.pem" + cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" } } listeners.ws.default { - bind = "0.0.0.0:8083" - max_connections = 1024000 + bind: "0.0.0.0:8083" + max_connections: 1024000 websocket.mqtt_path: "/mqtt" } listeners.wss.default { - bind = "0.0.0.0:8084" - max_connections = 512000 + bind: "0.0.0.0:8084" + max_connections: 512000 websocket.mqtt_path: "/mqtt" ssl { - keyfile = "{{ platform_etc_dir }}/certs/key.pem" - certfile = "{{ platform_etc_dir }}/certs/cert.pem" - cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + keyfile: "{{ platform_etc_dir }}/certs/key.pem" + certfile: "{{ platform_etc_dir }}/certs/cert.pem" + cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" } } -//listeners.quic.default { -// bind = "0.0.0.0:14567" -// max_connections = 1024000 -// keyfile = "{{ platform_etc_dir }}/certs/key.pem" -// certfile = "{{ platform_etc_dir }}/certs/cert.pem" -// } +listeners.quic.default { + enabled: true + bind: "0.0.0.0:14567" + max_connections: 1024000 + keyfile: "{{ platform_etc_dir }}/certs/key.pem" + certfile: "{{ platform_etc_dir }}/certs/cert.pem" +} diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index ae108c595..bfac1f7d2 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -17,6 +17,7 @@ -compile({no_auto_import, [get/0, get/1, put/2, erase/1]}). -elvis([{elvis_style, god_modules, disable}]). +-include("logger.hrl"). -export([ init_load/1, @@ -347,9 +348,17 @@ raw_conf_with_default(SchemaMod, RootNames, RawConf) -> schema_default(Schema) -> case hocon_schema:field_schema(Schema, type) of - ?ARRAY(_) -> []; - ?LAZY(?ARRAY(_)) -> []; - _ -> #{} + ?ARRAY(_) -> + []; + ?LAZY(?ARRAY(_)) -> + []; + ?LAZY(?UNION(Unions)) -> + case [A || ?ARRAY(A) <- Unions] of + [_ | _] -> []; + _ -> #{} + end; + _ -> + #{} end. parse_hocon(Conf) -> diff --git a/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf b/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf index 889fd8b63..e69de29bb 100644 --- a/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf +++ b/apps/emqx/src/emqx_limiter/etc/emqx_limiter.conf @@ -1,5 +0,0 @@ -##-------------------------------------------------------------------- -## EMQX Rate Limiter -##-------------------------------------------------------------------- - -limiter {} diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index 430cc49c4..c7a5e990b 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -435,6 +435,10 @@ t_connected_client_count_persistent(Config) when is_list(Config) -> {clientid, ClientID} | Config ]), + {{ok, _}, {ok, [_]}} = wait_for_events( + fun() -> emqtt:ConnFun(ConnPid2) end, + [emqx_cm_connected_client_count_inc] + ), {{ok, _}, {ok, [_, _]}} = wait_for_events( fun() -> emqtt:ConnFun(ConnPid2) end, [ From 8519b161dc25adcb8ed0cfae424773e014434b1b Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 24 May 2022 21:54:35 +0800 Subject: [PATCH 12/22] chore: replace = by : in conf --- apps/emqx_conf/etc/emqx_conf.conf | 14 +++++++------- apps/emqx_dashboard/etc/emqx_dashboard.conf | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index a24d5e3d4..7d608031d 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -10,19 +10,19 @@ node { name: "emqx@127.0.0.1" - cookie = emqxsecretcookie - data_dir = "{{ platform_data_dir }}" - etc_dir = "{{ platform_etc_dir }}" + cookie: emqxsecretcookie + data_dir: "{{ platform_data_dir }}" + etc_dir: "{{ platform_etc_dir }}" } log { file_handlers.default { - level = warning - file = "{{ platform_log_dir }}/emqx.log" + level: warning + file: "{{ platform_log_dir }}/emqx.log" } } cluster { - name = emqxcl - discovery_strategy = manual + name: emqxcl + discovery_strategy: manual } diff --git a/apps/emqx_dashboard/etc/emqx_dashboard.conf b/apps/emqx_dashboard/etc/emqx_dashboard.conf index 856779500..2d54431fa 100644 --- a/apps/emqx_dashboard/etc/emqx_dashboard.conf +++ b/apps/emqx_dashboard/etc/emqx_dashboard.conf @@ -1,7 +1,7 @@ dashboard { listeners.http { - bind = 18083 + bind: 18083 } - default_username = "admin" - default_password = "public" + default_username: "admin" + default_password: "public" } From 9ec804ae03bb20db0fab4c057861173148eb0d3e Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 30 May 2022 14:43:35 +0800 Subject: [PATCH 13/22] feat: generate example.conf from schemas --- .../emqx_limiter/src/emqx_limiter_schema.erl | 13 +- apps/emqx/src/emqx_schema.erl | 16 +- apps/emqx_authz/etc/emqx_authz.conf | 5 - apps/emqx_conf/src/emqx_conf.erl | 14 +- apps/emqx_conf/src/emqx_conf_schema.erl | 36 +- apps/emqx_conf/src/hocon_schema_example.erl | 539 ++++++++++++++++++ .../src/emqx_connector_schema.erl | 2 +- .../src/emqx_dashboard_schema.erl | 1 + apps/emqx_gateway/src/emqx_gateway_schema.erl | 5 +- .../src/emqx_retainer_schema.erl | 5 +- .../src/emqx_rule_engine_schema.erl | 2 +- .../src/emqx_slow_subs_schema.erl | 2 +- 12 files changed, 597 insertions(+), 43 deletions(-) create mode 100644 apps/emqx_conf/src/hocon_schema_example.erl diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index dd3b81f6e..5d1d720e6 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -105,7 +105,18 @@ fields(limiter_opts) -> {bucket, sc( map("bucket_name", ref(bucket_opts)), - #{desc => ?DESC(bucket_cfg), default => #{<<"default">> => #{}}} + #{ + desc => ?DESC(bucket_cfg), + default => #{<<"default">> => #{}}, + examples => #{ + <<"mybucket-name">> => #{ + <<"rate">> => <<"infinity">>, + <<"capcity">> => <<"infinity">>, + <<"initial">> => <<"100">>, + <<"per_client">> => #{<<"rate">> => <<"infinity">>} + } + } + } )} ]; fields(bucket_opts) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ee59b039d..423ac96f6 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -118,7 +118,7 @@ roots(high) -> )}, {"zones", sc( - map("name", ref("zone")), + map("my_zone_name", ref("zone")), #{desc => ?DESC(zones)} )}, {"mqtt", @@ -744,7 +744,7 @@ fields("listeners") -> [ {"tcp", sc( - map(name, ref("mqtt_tcp_listener")), + map(default, ref("mqtt_tcp_listener")), #{ desc => ?DESC(fields_listeners_tcp), required => {false, recursively} @@ -752,7 +752,7 @@ fields("listeners") -> )}, {"ssl", sc( - map(name, ref("mqtt_ssl_listener")), + map(default, ref("mqtt_ssl_listener")), #{ desc => ?DESC(fields_listeners_ssl), required => {false, recursively} @@ -760,7 +760,7 @@ fields("listeners") -> )}, {"ws", sc( - map(name, ref("mqtt_ws_listener")), + map(default, ref("mqtt_ws_listener")), #{ desc => ?DESC(fields_listeners_ws), required => {false, recursively} @@ -768,7 +768,7 @@ fields("listeners") -> )}, {"wss", sc( - map(name, ref("mqtt_wss_listener")), + map(default, ref("mqtt_wss_listener")), #{ desc => ?DESC(fields_listeners_wss), required => {false, recursively} @@ -776,7 +776,7 @@ fields("listeners") -> )}, {"quic", sc( - map(name, ref("mqtt_quic_listener")), + map(default, ref("mqtt_quic_listener")), #{ desc => ?DESC(fields_listeners_quic), required => {false, recursively} @@ -1582,7 +1582,7 @@ base_listener() -> )}, {"limiter", sc( - map("ratelimit's type", emqx_limiter_schema:bucket_name()), + map("ratelimit_name", emqx_limiter_schema:bucket_name()), #{ desc => ?DESC(base_listener_limiter), default => #{} @@ -2183,7 +2183,7 @@ authentication(Type) -> %% authentication schema is lazy to make it more 'plugable' %% the type checks are done in emqx_auth application when it boots. %% and in emqx_authentication_config module for runtime changes. - Default = hoconsc:lazy(hoconsc:union([typerefl:map(), hoconsc:array(typerefl:map())])), + Default = hoconsc:lazy(hoconsc:union([hoconsc:array(typerefl:map())])), %% as the type is lazy, the runtime module injection %% from EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY %% is for now only affecting document generation. diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index 844d8a2e1..09dc0febc 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -1,9 +1,4 @@ authorization { - cache: { - enable: true - max_size: 32 - ttl: "60s" - } deny_action: ignore no_match: allow sources: [ diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 3f843063f..e9a6abadb 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -144,7 +144,8 @@ dump_schema(Dir, SchemaModule, I18nFile) -> lists:foreach( fun(Lang) -> gen_config_md(Dir, I18nFile, SchemaModule, Lang), - gen_hot_conf_schema_json(Dir, I18nFile, Lang) + gen_hot_conf_schema_json(Dir, I18nFile, Lang), + gen_example_conf(Dir, I18nFile, SchemaModule, Lang) end, [en, zh] ), @@ -173,6 +174,12 @@ gen_config_md(Dir, I18nFile, SchemaModule, Lang0) -> io:format(user, "===< Generating: ~s~n", [SchemaMdFile]), ok = gen_doc(SchemaMdFile, SchemaModule, I18nFile, Lang). +gen_example_conf(Dir, I18nFile, SchemaModule, Lang0) -> + Lang = atom_to_list(Lang0), + SchemaMdFile = filename:join([Dir, "emqx-" ++ Lang ++ ".conf.example"]), + io:format(user, "===< Generating: ~s~n", [SchemaMdFile]), + ok = gen_example(SchemaMdFile, SchemaModule, I18nFile, Lang). + %% @doc return the root schema module. -spec schema_module() -> module(). schema_module() -> @@ -195,6 +202,11 @@ gen_doc(File, SchemaModule, I18nFile, Lang) -> Doc = hocon_schema_md:gen(SchemaModule, Opts), file:write_file(File, Doc). +gen_example(File, SchemaModule, I18nFile, Lang) -> + Opts = #{title => <<"Title">>, body => <<"Body">>, desc_file => I18nFile, lang => Lang}, + Example = hocon_schema_example:gen(SchemaModule, Opts), + file:write_file(File, Example). + check_cluster_rpc_result(Result) -> case Result of {ok, _TnxId, Res} -> diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 8e5fc99f2..b68a7fc2b 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -76,22 +76,22 @@ roots() -> [ {"node", sc( - ref("node"), + ?R_REF("node"), #{translate_to => ["emqx"]} )}, {"cluster", sc( - ref("cluster"), + ?R_REF("cluster"), #{translate_to => ["ekka"]} )}, {"log", sc( - ref("log"), + ?R_REF("log"), #{translate_to => ["kernel"]} )}, {"rpc", sc( - ref("rpc"), + ?R_REF("rpc"), #{translate_to => ["gen_rpc"]} )} ] ++ @@ -166,27 +166,27 @@ fields("cluster") -> )}, {"static", sc( - ref(cluster_static), + ?R_REF(cluster_static), #{} )}, {"mcast", sc( - ref(cluster_mcast), + ?R_REF(cluster_mcast), #{} )}, {"dns", sc( - ref(cluster_dns), + ?R_REF(cluster_dns), #{} )}, {"etcd", sc( - ref(cluster_etcd), + ?R_REF(cluster_etcd), #{} )}, {"k8s", sc( - ref(cluster_k8s), + ?R_REF(cluster_k8s), #{} )} ]; @@ -328,7 +328,7 @@ fields(cluster_etcd) -> )}, {"ssl", sc( - hoconsc:ref(emqx_schema, "ssl_client_opts"), + ?R_REF(emqx_schema, "ssl_client_opts"), #{ desc => ?DESC(cluster_etcd_ssl), 'readOnly' => true @@ -512,7 +512,7 @@ fields("node") -> )}, {"cluster_call", sc( - ref("cluster_call"), + ?R_REF("cluster_call"), #{'readOnly' => true} )}, {"db_backend", @@ -783,10 +783,10 @@ fields("rpc") -> ]; fields("log") -> [ - {"console_handler", ref("console_handler")}, + {"console_handler", ?R_REF("console_handler")}, {"file_handlers", sc( - map(name, ref("log_file_handler")), + map(name, ?R_REF("log_file_handler")), #{desc => ?DESC("log_file_handlers")} )}, {"error_logger", @@ -814,7 +814,7 @@ fields("log_file_handler") -> )}, {"rotation", sc( - ref("log_rotation"), + ?R_REF("log_rotation"), #{} )}, {"max_size", @@ -1137,8 +1137,8 @@ log_handler_common_confs(Enable) -> desc => ?DESC("common_handler_flush_qlen") } )}, - {"overload_kill", sc(ref("log_overload_kill"), #{})}, - {"burst_limit", sc(ref("log_burst_limit"), #{})}, + {"overload_kill", sc(?R_REF("log_overload_kill"), #{})}, + {"burst_limit", sc(?R_REF("log_burst_limit"), #{})}, {"supervisor_reports", sc( hoconsc:enum([error, progress]), @@ -1251,8 +1251,6 @@ sc(Type, Meta) -> hoconsc:mk(Type, Meta). map(Name, Type) -> hoconsc:map(Name, Type). -ref(Field) -> hoconsc:ref(?MODULE, Field). - options(static, Conf) -> [{seeds, conf_get("cluster.static.seeds", Conf, [])}]; options(mcast, Conf) -> @@ -1321,7 +1319,7 @@ emqx_schema_high_prio_roots() -> Authz = {"authorization", sc( - hoconsc:ref(?MODULE, "authorization"), + ?R_REF("authorization"), #{desc => ?DESC(authorization)} )}, lists:keyreplace("authorization", 1, Roots, Authz). diff --git a/apps/emqx_conf/src/hocon_schema_example.erl b/apps/emqx_conf/src/hocon_schema_example.erl new file mode 100644 index 000000000..189be6787 --- /dev/null +++ b/apps/emqx_conf/src/hocon_schema_example.erl @@ -0,0 +1,539 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(hocon_schema_example). +-include_lib("hocon/include/hoconsc.hrl"). + +-export([gen/2]). + +-define(COMMENT, "# "). +-define(COMMENT2, "## "). +-define(INDENT, " "). +-define(NL, io_lib:nl()). +-define(DOC, "@doc "). +-define(TYPE, "@type "). +-define(PATH, "@path "). +-define(LINK, "@link "). +-define(DEFAULT, "@default "). +-define(BIND, ": "). + +gen(Schema, undefined) -> + gen(Schema, "# HOCON Example"); +gen(Schema, Title) when is_list(Title) orelse is_binary(Title) -> + gen(Schema, #{title => Title, body => <<>>}); +gen(Schema, #{title := Title, body := Body} = Opts) -> + File = maps:get(desc_file, Opts, undefined), + Lang = maps:get(lang, Opts, "en"), + [Roots | Fields0] = hocon_schema_json:gen(Schema, #{desc_file => File, lang => Lang}), + Fields = lists:foldl(fun(F = #{full_name := Name}, Acc) -> Acc#{Name => F} end, #{}, Fields0), + #{fields := RootKeys} = Roots, + FmtOpts = #{tid => new_link_cache(), indent => "", comment => false}, + try + Structs = lists:map( + fun(Root) -> + [ + fmt_desc(Root, ""), + fmt_field(Root, Fields, "", FmtOpts) + ] + end, + RootKeys + ), + [ + ?COMMENT2, + Title, + ?NL, + ?NL, + ?COMMENT2, + Body, + ?NL, + ?NL, + Structs + ] + after + delete_link_cache(FmtOpts) + end. + +fmt_field(#{type := #{kind := struct, name := SubName}, name := Name} = Field, All, Path0, Opts) -> + case maps:find(SubName, All) of + {ok, #{fields := SubFields}} -> + #{indent := Indent, comment := Comment} = Opts, + Opts1 = Opts#{indent => Indent ++ ?INDENT}, + {PathName, ValName} = resolve_name(Name), + Path = [str(PathName) | Path0], + SubStructs = + case maps:get(examples, Field, #{}) of + #{} = Example -> + fmt_field_with_example(Path, SubFields, Example, All, Opts1); + {union, UnionExamples} -> + Examples1 = filter_union_example(UnionExamples, SubFields), + fmt_field_with_example(Path, SubFields, Examples1, All, Opts1); + {array, ArrayExamples} -> + lists:flatmap( + fun(SubExample) -> + fmt_field_with_example(Path, SubFields, SubExample, All, Opts1) + end, + ArrayExamples + ) + end, + [ + Indent, + comment(Comment), + ValName, + " {", + ?NL, + lists:join(?NL, SubStructs), + Indent, + comment(Comment), + " }", + ?NL + ]; + Unknown -> + throw({error, {Path0, SubName, Unknown}}) + end; +fmt_field(#{type := #{kind := primitive, name := TypeName}} = Field, _All, Path, Opts) -> + Name = str(maps:get(name, Field)), + Fix = fmt_fix_header(Field, TypeName, [Name | Path], Opts), + [Fix, fmt_examples(Name, Field, Opts)]; +fmt_field(#{type := #{kind := singleton, name := SingleTon}} = Field, _All, Path, Opts) -> + Name = str(maps:get(name, Field)), + #{indent := Indent, comment := Comment} = Opts, + Fix = fmt_fix_header(Field, "singleton", [Name | Path], Opts), + [Fix, fmt(Indent, Comment, Name, SingleTon)]; +fmt_field(#{type := #{kind := enum, symbols := Symbols}} = Field, _All, Path, Opts) -> + TypeName = ["enum: ", lists:join(" | ", Symbols)], + Name = str(maps:get(name, Field)), + Fix = fmt_fix_header(Field, TypeName, [str(Name) | Path], Opts), + [Fix, fmt_examples(Name, Field, Opts)]; +fmt_field(#{type := #{kind := union, members := Members0} = Type} = Field, All, Path0, Opts) -> + Name = str(maps:get(name, Field)), + Names = lists:map(fun(#{name := N}) -> N end, Members0), + Path = [Name | Path0], + TypeStr = ["union() ", lists:join(" | ", Names)], + Fix = fmt_fix_header(Field, TypeStr, Path, Opts), + Link = fmt_union_link(Type, Path, Opts), + Fix1 = [Fix, Link], + case Link =:= "" andalso need_comment_example(union, Opts, Type, Path) of + true -> + #{indent := Indent} = Opts, + Indent1 = Indent ++ ?INDENT, + Opts1 = Opts#{indent => Indent1}, + case fmt_sub_fields(Opts1, Field, All, Path0) of + [] -> fallback_to_example(Field, Fix1, Indent1, Name, Opts1, Indent, ""); + ValFields -> [Fix1, ValFields, ?NL] + end; + false -> + [Fix1, ?NL] + end; +fmt_field(#{type := #{kind := map, name := MapName} = Type} = Field, All, Path0, Opts) -> + Name = str(maps:get(name, Field)), + #{indent := Indent} = Opts, + Path = [Name | Path0], + Path1 = ["$" ++ str(MapName) | Path], + Fix = fmt_fix_header(Field, "map_struct()", Path, Opts), + Link = fmt_map_link(Path1, Type, All, Opts), + Fix1 = [Fix, Link], + case Link =:= "" andalso need_comment_example(map, Opts, Path1) of + true -> + Indent1 = Indent ++ ?INDENT, + Opts1 = Opts#{indent => Indent1}, + ValFields = fmt_sub_fields(Opts1, Field, All, Path), + [Fix1, Indent1, ?COMMENT, Name, ?BIND, ?NL, ValFields, ?NL]; + false -> + [Fix1, ?NL] + end; +fmt_field(#{type := #{kind := array} = Type} = Field, All, Path0, Opts) -> + #{indent := Indent, comment := Comment} = Opts, + Name = str(maps:get(name, Field)), + Path = [Name | Path0], + Fix = fmt_fix_header(Field, "array()", Path, Opts), + Link = fmt_array_link(Type, Path, Opts), + Fix1 = [Fix, Link], + case Link =:= "" andalso need_comment_example(array, Opts, Type, Path) of + true -> + Indent1 = Indent ++ ?INDENT, + Opts1 = Opts#{indent => Indent1}, + case fmt_sub_fields(Opts1, Field, All, Path) of + [] -> + fallback_to_example(Field, Fix1, Indent1, Name, Opts1, Indent, "[]"); + ValFields -> + [ + Fix1, + Indent1, + comment(Comment), + Name, + ?BIND, + "[", + ?NL, + ValFields, + ?NL, + Indent1, + comment(Comment), + "]", + ?NL + ] + end; + false -> + [Fix1, ?NL] + end. + +fmt(Indent, Comment, Name, Value) -> + [Indent, comment(Comment), Name, ?BIND, Value, ?NL]. + +fallback_to_example(Field, Fix1, Indent1, Name, Opts, Indent, Default) -> + case Field of + #{examples := Examples} -> + [ + Fix1, + Indent1, + ?COMMENT, + Name, + ?BIND, + fmt_example(Examples, Opts#{comment => true}), + ?NL + ]; + _ -> + Default2 = + case get_default(Field, Opts) of + undefined -> Default; + Default1 -> Default1 + end, + [Fix1, Indent, ?COMMENT, Name, ?BIND, Default2, ?NL] + end. + +fmt_field_with_example(Path, SubFields, Examples, All, Opts1) -> + lists:map( + fun(F) -> + #{name := N} = F, + case maps:find(N, Examples) of + {ok, SubExample} -> + fmt_field(F#{examples => SubExample}, All, Path, Opts1); + error -> + fmt_field(F, All, Path, Opts1) + end + end, + SubFields + ). + +fmt_sub_fields(Opts, Field, All, Path) -> + Opts1 = Opts#{comment => true}, + SubFields = get_sub_fields(Field), + [fmt_field(F, All, Path, Opts1) || F <- SubFields]. + +get_sub_fields(#{type := #{kind := array, elements := ElemT}, name := Name} = Field) -> + case is_simple_type(ElemT) of + true -> + []; + false -> + Examples = + case get_examples(Name, Field) of + undefined -> []; + Example0 -> Example0 + end, + [ + #{ + name => {"$INDEX", str(Name) ++ ".$INDEX"}, + type => ElemT, + examples => {array, Examples} + } + ] + end; +get_sub_fields(#{type := #{kind := union, members := Members}, name := Name} = Field) -> + case is_simple_type(Members) of + true -> + []; + false -> + Example = + case get_examples(Name, Field) of + undefined -> + []; + [Example0] when is_map(Example0) -> + [Value || #{value := Value} <- maps:values(Example0)]; + %% TODO array + _ -> + [] + end, + lists:map( + fun(M) -> #{name => Name, type => M, examples => {union, Example}} end, Members + ) + end; +get_sub_fields(#{type := #{kind := map, values := ValT, name := MapName0}} = Field) -> + MapName = "$" ++ str(MapName0), + case get_examples(MapName0, Field) of + undefined -> + [#{name => MapName, type => ValT}]; + [] -> + [#{name => MapName, type => ValT}]; + Examples -> + lists:map( + fun(Example) -> + [{SubName, SubValue}] = maps:to_list(Example), + #{name => {MapName, SubName}, type => ValT, examples => SubValue} + end, + Examples + ) + end. + +filter_union_example(Examples0, SubFields) -> + TargetKeys = lists:sort([binary_to_atom(Name) || #{name := Name} <- SubFields]), + Examples = + lists:filtermap( + fun(Example) -> + case lists:all(fun(K) -> lists:member(K, TargetKeys) end, maps:keys(Example)) of + true -> {true, ensure_bin_key(Example)}; + false -> false + end + end, + Examples0 + ), + case Examples of + [Example] -> Example; + [] -> #{}; + Other -> throw({error, {find_union_example_failed, Examples, SubFields, Other}}) + end. + +ensure_bin_key(Map) -> + maps:fold( + fun + (K0, V0 = #{}, Acc) -> Acc#{bin(K0) => ensure_bin_key(V0)}; + (K0, V, Acc) -> Acc#{bin(K0) => V} + end, + #{}, + Map + ). + +fmt_desc(#{desc := Desc0}, Indent) -> + Target = iolist_to_binary([?NL, Indent, ?COMMENT2]), + Desc = string:trim(Desc0, both), + replace_nl(Indent, true, Desc, Target); +fmt_desc(_, _) -> + <<"">>. + +fmt_type(Type, Indent) -> + [Indent, ?COMMENT2, ?TYPE, Type, ?NL]. + +fmt_path(Path, Indent) -> [Indent, ?COMMENT2, ?PATH, hocon_schema:path(Path), ?NL]. + +fmt_fix_header(Field, Type, Path, #{indent := Indent}) -> + [ + fmt_desc(Field, Indent), + fmt_path(Path, Indent), + fmt_type(Type, Indent), + fmt_default(Field, Indent) + ]. + +fmt_map_link(Path0, Type, All, Opts) -> + case Type of + #{values := #{name := ValueName}} -> + fmt_map_link2(Path0, ValueName, All, Opts); + #{values := #{members := Members}} -> + lists:map(fun(M) -> fmt_map_link(Path0, M, All, Opts) end, Members); + _ -> + [] + end. + +fmt_map_link2(Path0, ValueName, All, Opts) -> + Paths = + case maps:find(ValueName, All) of + {ok, #{paths := SubPaths}} -> SubPaths; + _ -> [] + end, + PathStr = hocon_schema:path(Path0), + Path = bin(PathStr), + #{indent := Indent} = Opts, + case find_link(Opts, {map, PathStr}) of + {ok, Link} -> + [Indent, ?COMMENT2, ?LINK, Link, ?NL]; + {error, not_found} -> + case lists:member(Path, Paths) of + true -> + insert_link(Opts, [{{map, binary_to_list(P)}, Path} || P <- Paths, P =/= Path]); + false -> + ok + end, + "" + end. + +fmt_union_link(Type = #{members := Members}, Path, Opts = #{indent := Indent}) -> + case find_link(Opts, {union, Type}) of + {ok, Link} -> + link(Link, Indent); + {error, not_found} -> + case is_simple_type(Members) of + true -> ok; + false -> insert_link(Opts, {{union, Type}, Path}) + end, + "" + end. + +fmt_array_link(Type = #{elements := ElemT}, Path, Opts = #{indent := Indent}) -> + case find_link(Opts, {array, Type}) of + {ok, Link} -> + link(Link, Indent); + {error, not_found} -> + case is_simple_type(ElemT) of + true -> ok; + false -> insert_link(Opts, {{array, Type}, Path}) + end, + "" + end. + +link(Link, Indent) -> + [Indent, ?COMMENT2, ?LINK, hocon_schema:path(Link), ?NL]. + +is_simple_type(Types) when is_list(Types) -> + lists:all( + fun(#{kind := Kind}) -> + Kind =:= primitive orelse Kind =:= singleton + end, + Types + ); +is_simple_type(Type) -> + is_simple_type([Type]). + +need_comment_example(map, Opts, Path) -> + case find_link(Opts, {map, hocon_schema:path(Path)}) of + {ok, _} -> false; + {error, not_found} -> true + end. + +need_comment_example(Type, Opts, Key, Link) when Type =:= union; Type =:= array -> + case find_link(Opts, {union, Key}) of + {ok, Link} -> true; + {error, not_found} -> true; + {ok, _} -> false + end. + +get_examples(_MapName, #{examples := Examples}) -> + ensure_list(Examples); +get_examples(MapName, #{default := #{hocon := Hocon}}) -> + case hocon:binary(Hocon) of + {ok, Default} -> [#{MapName => Default}]; + {error, _} -> [#{MapName => Hocon}] + end; +get_examples(_, _) -> + undefined. + +fmt_examples(Name, #{examples := {union, Examples}}, Opts) -> + fmt_examples(Name, #{examples => Examples}, Opts); +fmt_examples(Name, #{examples := Examples}, Opts) -> + #{indent := Indent, comment := Comment} = Opts, + lists:map( + fun(E) -> + [Indent, comment(Comment), Name, ?BIND, fmt_example(E, Opts), ?NL] + end, + ensure_list(Examples) + ); +fmt_examples(Name, Field, Opts = #{indent := Indent, comment := Comment}) -> + case get_default(Field, Opts) of + undefined -> [Indent, ?COMMENT, Name, ?BIND, ?NL]; + Default -> fmt(Indent, Comment, Name, Default) + end. + +ensure_list(L) when is_list(L) -> L; +ensure_list(T) -> [T]. + +fmt_example(Value, #{indent := Indent0, comment := Comment}) -> + case hocon_pp:do(Value, #{newline => "", embedded => true}) of + [OneLine] -> + [try_to_remove_quote(OneLine)]; + Lines -> + Indent = Indent0 ++ ?INDENT, + Target = iolist_to_binary([?NL, Indent, comment(Comment)]), + [ + ?NL, + Indent, + comment(Comment), + binary:replace(bin(Lines), [<<"\n">>], Target, [global]), + ?NL + ] + end. + +fmt_default(Field, Indent) -> + case get_default(Field, #{indent => Indent, comment => true}) of + undefined -> ""; + Default -> [Indent, ?COMMENT2, ?DEFAULT, Default, ?NL] + end. + +get_default(#{default := Default}, Opts) when is_map(Opts) -> + #{indent := Indent, comment := Comment} = Opts, + get_default(Default, Indent, Comment); +get_default(_, _Opts) -> + undefined. + +-define(RE, <<"^[A-Za-z0-9\"]+$">>). + +get_default(#{oneliner := true, hocon := Content}, _Indent, _Comment) -> + try_to_remove_quote(Content); +get_default(#{oneliner := false, hocon := Content}, Indent0, Comment) -> + Target = iolist_to_binary([?NL, Indent0, comment2(Comment), ?INDENT]), + replace_nl(Indent0, Comment, Content, Target); +get_default(Bin, _Indent, _Comment) -> + Bin. + +replace_nl(Indent0, Comment, Content, Target) -> + [ + ?NL, + Indent0, + comment2(Comment), + ?INDENT, + binary:replace(Content, [<<"\n">>], Target, [global]), + ?NL + ]. + +try_to_remove_quote(Content) -> + case re:run(Content, ?RE) of + nomatch -> + Content; + _ -> + case string:trim(Content, both, [$"]) of + <<"">> -> Content; + Other -> Other + end + end. + +bin(S) when is_list(S) -> unicode:characters_to_binary(S, utf8); +bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom); +bin(Int) when is_integer(Int) -> integer_to_binary(Int); +bin(Bin) -> Bin. + +str(A) when is_atom(A) -> atom_to_list(A); +str(S) when is_list(S) -> S; +str(B) when is_binary(B) -> binary_to_list(B); +str({KeyName, _ValName}) -> str(KeyName). + +comment(true) -> ?COMMENT; +comment(false) -> "". + +comment2(true) -> ?COMMENT2; +comment2(false) -> "". + +new_link_cache() -> + ets:new(?MODULE, [private, set, {keypos, 1}]). + +delete_link_cache(#{tid := Tid}) -> + ets:delete(Tid). + +find_link(#{tid := Tid}, Key) -> + case ets:lookup(Tid, Key) of + [{_, Value}] -> {ok, Value}; + [] -> {error, not_found} + end. + +insert_link(#{tid := Tid}, Item) -> + ets:insert(Tid, Item). + +resolve_name({N1, N2}) -> {N1, N2}; +resolve_name(N) -> {N, N}. diff --git a/apps/emqx_connector/src/emqx_connector_schema.erl b/apps/emqx_connector/src/emqx_connector_schema.erl index 3f1d4f4aa..f92c12287 100644 --- a/apps/emqx_connector/src/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/emqx_connector_schema.erl @@ -64,7 +64,7 @@ fields("connectors") -> mk( hoconsc:map( name, - hoconsc:union([ref(emqx_connector_mqtt_schema, "connector")]) + ref(emqx_connector_mqtt_schema, "connector") ), #{desc => ?DESC("mqtt")} )} diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index c358aef17..12a799418 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -171,6 +171,7 @@ bind(Port) -> #{ default => Port, required => true, + extra => #{example => [Port, "0.0.0.0:" ++ integer_to_list(Port)]}, desc => ?DESC(bind) } )}. diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 83f51d110..f3b3a0fa6 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -567,7 +567,8 @@ authentication_schema() -> emqx_authn_schema:authenticator_type(), #{ required => {false, recursively}, - desc => ?DESC(gateway_common_authentication) + desc => ?DESC(gateway_common_authentication), + examples => emqx_authn_api:authenticator_examples() } ). @@ -606,7 +607,7 @@ gateway_common_options() -> ]. mountpoint() -> - mountpoint(<<>>). + mountpoint(<<"">>). mountpoint(Default) -> sc( binary(), diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index be2a64868..db419bf1d 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -109,10 +109,7 @@ sc(Type, DescId, Default) -> hoconsc:mk(Type, #{default => Default, desc => ?DESC(DescId)}). backend_config() -> - hoconsc:mk( - hoconsc:union([hoconsc:ref(?MODULE, mnesia_config)]), - #{desc => ?DESC(backend)} - ). + hoconsc:mk(hoconsc:ref(?MODULE, mnesia_config), #{desc => ?DESC(backend)}). retainer_indices(type) -> list(list(integer())); diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index 61b8dd4b9..55d8586e4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -186,7 +186,7 @@ rule_name() -> binary(), #{ desc => ?DESC("rules_name"), - default => "", + default => <<"">>, required => false, example => "foo" } diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index d570e2004..b7f7337d2 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -32,7 +32,7 @@ fields("slow_subs") -> )}, {stats_type, sc( - hoconsc:union([whole, internal, response]), + hoconsc:enum([whole, internal, response]), whole, stats_type )} From 86113f3494fc028dbba23f0519e73a5c1a98bc16 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 30 May 2022 22:01:25 +0800 Subject: [PATCH 14/22] chore: don't gen example.conf in v5.0.0 --- apps/emqx_conf/src/emqx_conf.erl | 6 ++++-- apps/emqx_conf/src/hocon_schema_example.erl | 11 +++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index e9a6abadb..aa28e81c3 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -27,6 +27,7 @@ -export([reset/2, reset/3]). -export([dump_schema/1, dump_schema/3]). -export([schema_module/0]). +-export([gen_example_conf/4]). %% for rpc -export([get_node_and_config/1]). @@ -144,8 +145,9 @@ dump_schema(Dir, SchemaModule, I18nFile) -> lists:foreach( fun(Lang) -> gen_config_md(Dir, I18nFile, SchemaModule, Lang), - gen_hot_conf_schema_json(Dir, I18nFile, Lang), - gen_example_conf(Dir, I18nFile, SchemaModule, Lang) + gen_hot_conf_schema_json(Dir, I18nFile, Lang) + %% TODO + %%gen_example_conf(Dir, I18nFile, SchemaModule, Lang) end, [en, zh] ), diff --git a/apps/emqx_conf/src/hocon_schema_example.erl b/apps/emqx_conf/src/hocon_schema_example.erl index 189be6787..050075e43 100644 --- a/apps/emqx_conf/src/hocon_schema_example.erl +++ b/apps/emqx_conf/src/hocon_schema_example.erl @@ -324,7 +324,10 @@ fmt_desc(_, _) -> fmt_type(Type, Indent) -> [Indent, ?COMMENT2, ?TYPE, Type, ?NL]. -fmt_path(Path, Indent) -> [Indent, ?COMMENT2, ?PATH, hocon_schema:path(Path), ?NL]. +fmt_path(Path, Indent) -> [Indent, ?COMMENT2, ?PATH, path(Path), ?NL]. + +path(Stack) when is_list(Stack) -> + string:join(lists:reverse(lists:map(fun str/1, Stack)), "."). fmt_fix_header(Field, Type, Path, #{indent := Indent}) -> [ @@ -350,7 +353,7 @@ fmt_map_link2(Path0, ValueName, All, Opts) -> {ok, #{paths := SubPaths}} -> SubPaths; _ -> [] end, - PathStr = hocon_schema:path(Path0), + PathStr = path(Path0), Path = bin(PathStr), #{indent := Indent} = Opts, case find_link(Opts, {map, PathStr}) of @@ -391,7 +394,7 @@ fmt_array_link(Type = #{elements := ElemT}, Path, Opts = #{indent := Indent}) -> end. link(Link, Indent) -> - [Indent, ?COMMENT2, ?LINK, hocon_schema:path(Link), ?NL]. + [Indent, ?COMMENT2, ?LINK, path(Link), ?NL]. is_simple_type(Types) when is_list(Types) -> lists:all( @@ -404,7 +407,7 @@ is_simple_type(Type) -> is_simple_type([Type]). need_comment_example(map, Opts, Path) -> - case find_link(Opts, {map, hocon_schema:path(Path)}) of + case find_link(Opts, {map, path(Path)}) of {ok, _} -> false; {error, not_found} -> true end. From 61237cd97b75a4679ed3d2f22234935045c54c4a Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Mon, 30 May 2022 23:53:40 +0800 Subject: [PATCH 15/22] chore: debug for docker start failed --- apps/emqx/src/emqx_router_helper.erl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl index 1340848ec..a459577b6 100644 --- a/apps/emqx/src/emqx_router_helper.erl +++ b/apps/emqx/src/emqx_router_helper.erl @@ -99,7 +99,19 @@ init([]) -> process_flag(trap_exit, true), ok = ekka:monitor(membership), _ = mria:wait_for_tables([?ROUTING_NODE]), - {ok, _} = mnesia:subscribe({table, ?ROUTING_NODE, simple}), + %{ok, _} = mnesia:subscribe({table, ?ROUTING_NODE, simple}), + %% Temporary fix for debugging: + WhereToRead = ets:lookup(mnesia_gvar, {?ROUTING_NODE, where_to_read}), + case mnesia:subscribe({table, ?ROUTING_NODE, simple}) of + {ok, _} -> + ok; + Err -> + error(#{ + failed_to_subscribe => Err, + where_to_read => WhereToRead, + status => mria_rlog:status() + }) + end, Nodes = lists:foldl( fun(Node, Acc) -> case ekka:is_member(Node) of From 1345b7e9937c8118616fd8e05012535b6f22b299 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 31 May 2022 08:49:37 +0800 Subject: [PATCH 16/22] chore: bump hocon to 0.28.0 --- apps/emqx/rebar.config | 2 +- apps/emqx_conf/src/emqx_conf.erl | 5 +- apps/emqx_conf/src/hocon_schema_example.erl | 542 -------------------- apps/emqx_prometheus/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 6 files changed, 6 insertions(+), 549 deletions(-) delete mode 100644 apps/emqx_conf/src/hocon_schema_example.erl diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index cb7f94b7a..263de55f1 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.2"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.8"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.5"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.0"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index aa28e81c3..fb0082819 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -145,9 +145,8 @@ dump_schema(Dir, SchemaModule, I18nFile) -> lists:foreach( fun(Lang) -> gen_config_md(Dir, I18nFile, SchemaModule, Lang), - gen_hot_conf_schema_json(Dir, I18nFile, Lang) - %% TODO - %%gen_example_conf(Dir, I18nFile, SchemaModule, Lang) + gen_hot_conf_schema_json(Dir, I18nFile, Lang), + gen_example_conf(Dir, I18nFile, SchemaModule, Lang) end, [en, zh] ), diff --git a/apps/emqx_conf/src/hocon_schema_example.erl b/apps/emqx_conf/src/hocon_schema_example.erl deleted file mode 100644 index 050075e43..000000000 --- a/apps/emqx_conf/src/hocon_schema_example.erl +++ /dev/null @@ -1,542 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(hocon_schema_example). --include_lib("hocon/include/hoconsc.hrl"). - --export([gen/2]). - --define(COMMENT, "# "). --define(COMMENT2, "## "). --define(INDENT, " "). --define(NL, io_lib:nl()). --define(DOC, "@doc "). --define(TYPE, "@type "). --define(PATH, "@path "). --define(LINK, "@link "). --define(DEFAULT, "@default "). --define(BIND, ": "). - -gen(Schema, undefined) -> - gen(Schema, "# HOCON Example"); -gen(Schema, Title) when is_list(Title) orelse is_binary(Title) -> - gen(Schema, #{title => Title, body => <<>>}); -gen(Schema, #{title := Title, body := Body} = Opts) -> - File = maps:get(desc_file, Opts, undefined), - Lang = maps:get(lang, Opts, "en"), - [Roots | Fields0] = hocon_schema_json:gen(Schema, #{desc_file => File, lang => Lang}), - Fields = lists:foldl(fun(F = #{full_name := Name}, Acc) -> Acc#{Name => F} end, #{}, Fields0), - #{fields := RootKeys} = Roots, - FmtOpts = #{tid => new_link_cache(), indent => "", comment => false}, - try - Structs = lists:map( - fun(Root) -> - [ - fmt_desc(Root, ""), - fmt_field(Root, Fields, "", FmtOpts) - ] - end, - RootKeys - ), - [ - ?COMMENT2, - Title, - ?NL, - ?NL, - ?COMMENT2, - Body, - ?NL, - ?NL, - Structs - ] - after - delete_link_cache(FmtOpts) - end. - -fmt_field(#{type := #{kind := struct, name := SubName}, name := Name} = Field, All, Path0, Opts) -> - case maps:find(SubName, All) of - {ok, #{fields := SubFields}} -> - #{indent := Indent, comment := Comment} = Opts, - Opts1 = Opts#{indent => Indent ++ ?INDENT}, - {PathName, ValName} = resolve_name(Name), - Path = [str(PathName) | Path0], - SubStructs = - case maps:get(examples, Field, #{}) of - #{} = Example -> - fmt_field_with_example(Path, SubFields, Example, All, Opts1); - {union, UnionExamples} -> - Examples1 = filter_union_example(UnionExamples, SubFields), - fmt_field_with_example(Path, SubFields, Examples1, All, Opts1); - {array, ArrayExamples} -> - lists:flatmap( - fun(SubExample) -> - fmt_field_with_example(Path, SubFields, SubExample, All, Opts1) - end, - ArrayExamples - ) - end, - [ - Indent, - comment(Comment), - ValName, - " {", - ?NL, - lists:join(?NL, SubStructs), - Indent, - comment(Comment), - " }", - ?NL - ]; - Unknown -> - throw({error, {Path0, SubName, Unknown}}) - end; -fmt_field(#{type := #{kind := primitive, name := TypeName}} = Field, _All, Path, Opts) -> - Name = str(maps:get(name, Field)), - Fix = fmt_fix_header(Field, TypeName, [Name | Path], Opts), - [Fix, fmt_examples(Name, Field, Opts)]; -fmt_field(#{type := #{kind := singleton, name := SingleTon}} = Field, _All, Path, Opts) -> - Name = str(maps:get(name, Field)), - #{indent := Indent, comment := Comment} = Opts, - Fix = fmt_fix_header(Field, "singleton", [Name | Path], Opts), - [Fix, fmt(Indent, Comment, Name, SingleTon)]; -fmt_field(#{type := #{kind := enum, symbols := Symbols}} = Field, _All, Path, Opts) -> - TypeName = ["enum: ", lists:join(" | ", Symbols)], - Name = str(maps:get(name, Field)), - Fix = fmt_fix_header(Field, TypeName, [str(Name) | Path], Opts), - [Fix, fmt_examples(Name, Field, Opts)]; -fmt_field(#{type := #{kind := union, members := Members0} = Type} = Field, All, Path0, Opts) -> - Name = str(maps:get(name, Field)), - Names = lists:map(fun(#{name := N}) -> N end, Members0), - Path = [Name | Path0], - TypeStr = ["union() ", lists:join(" | ", Names)], - Fix = fmt_fix_header(Field, TypeStr, Path, Opts), - Link = fmt_union_link(Type, Path, Opts), - Fix1 = [Fix, Link], - case Link =:= "" andalso need_comment_example(union, Opts, Type, Path) of - true -> - #{indent := Indent} = Opts, - Indent1 = Indent ++ ?INDENT, - Opts1 = Opts#{indent => Indent1}, - case fmt_sub_fields(Opts1, Field, All, Path0) of - [] -> fallback_to_example(Field, Fix1, Indent1, Name, Opts1, Indent, ""); - ValFields -> [Fix1, ValFields, ?NL] - end; - false -> - [Fix1, ?NL] - end; -fmt_field(#{type := #{kind := map, name := MapName} = Type} = Field, All, Path0, Opts) -> - Name = str(maps:get(name, Field)), - #{indent := Indent} = Opts, - Path = [Name | Path0], - Path1 = ["$" ++ str(MapName) | Path], - Fix = fmt_fix_header(Field, "map_struct()", Path, Opts), - Link = fmt_map_link(Path1, Type, All, Opts), - Fix1 = [Fix, Link], - case Link =:= "" andalso need_comment_example(map, Opts, Path1) of - true -> - Indent1 = Indent ++ ?INDENT, - Opts1 = Opts#{indent => Indent1}, - ValFields = fmt_sub_fields(Opts1, Field, All, Path), - [Fix1, Indent1, ?COMMENT, Name, ?BIND, ?NL, ValFields, ?NL]; - false -> - [Fix1, ?NL] - end; -fmt_field(#{type := #{kind := array} = Type} = Field, All, Path0, Opts) -> - #{indent := Indent, comment := Comment} = Opts, - Name = str(maps:get(name, Field)), - Path = [Name | Path0], - Fix = fmt_fix_header(Field, "array()", Path, Opts), - Link = fmt_array_link(Type, Path, Opts), - Fix1 = [Fix, Link], - case Link =:= "" andalso need_comment_example(array, Opts, Type, Path) of - true -> - Indent1 = Indent ++ ?INDENT, - Opts1 = Opts#{indent => Indent1}, - case fmt_sub_fields(Opts1, Field, All, Path) of - [] -> - fallback_to_example(Field, Fix1, Indent1, Name, Opts1, Indent, "[]"); - ValFields -> - [ - Fix1, - Indent1, - comment(Comment), - Name, - ?BIND, - "[", - ?NL, - ValFields, - ?NL, - Indent1, - comment(Comment), - "]", - ?NL - ] - end; - false -> - [Fix1, ?NL] - end. - -fmt(Indent, Comment, Name, Value) -> - [Indent, comment(Comment), Name, ?BIND, Value, ?NL]. - -fallback_to_example(Field, Fix1, Indent1, Name, Opts, Indent, Default) -> - case Field of - #{examples := Examples} -> - [ - Fix1, - Indent1, - ?COMMENT, - Name, - ?BIND, - fmt_example(Examples, Opts#{comment => true}), - ?NL - ]; - _ -> - Default2 = - case get_default(Field, Opts) of - undefined -> Default; - Default1 -> Default1 - end, - [Fix1, Indent, ?COMMENT, Name, ?BIND, Default2, ?NL] - end. - -fmt_field_with_example(Path, SubFields, Examples, All, Opts1) -> - lists:map( - fun(F) -> - #{name := N} = F, - case maps:find(N, Examples) of - {ok, SubExample} -> - fmt_field(F#{examples => SubExample}, All, Path, Opts1); - error -> - fmt_field(F, All, Path, Opts1) - end - end, - SubFields - ). - -fmt_sub_fields(Opts, Field, All, Path) -> - Opts1 = Opts#{comment => true}, - SubFields = get_sub_fields(Field), - [fmt_field(F, All, Path, Opts1) || F <- SubFields]. - -get_sub_fields(#{type := #{kind := array, elements := ElemT}, name := Name} = Field) -> - case is_simple_type(ElemT) of - true -> - []; - false -> - Examples = - case get_examples(Name, Field) of - undefined -> []; - Example0 -> Example0 - end, - [ - #{ - name => {"$INDEX", str(Name) ++ ".$INDEX"}, - type => ElemT, - examples => {array, Examples} - } - ] - end; -get_sub_fields(#{type := #{kind := union, members := Members}, name := Name} = Field) -> - case is_simple_type(Members) of - true -> - []; - false -> - Example = - case get_examples(Name, Field) of - undefined -> - []; - [Example0] when is_map(Example0) -> - [Value || #{value := Value} <- maps:values(Example0)]; - %% TODO array - _ -> - [] - end, - lists:map( - fun(M) -> #{name => Name, type => M, examples => {union, Example}} end, Members - ) - end; -get_sub_fields(#{type := #{kind := map, values := ValT, name := MapName0}} = Field) -> - MapName = "$" ++ str(MapName0), - case get_examples(MapName0, Field) of - undefined -> - [#{name => MapName, type => ValT}]; - [] -> - [#{name => MapName, type => ValT}]; - Examples -> - lists:map( - fun(Example) -> - [{SubName, SubValue}] = maps:to_list(Example), - #{name => {MapName, SubName}, type => ValT, examples => SubValue} - end, - Examples - ) - end. - -filter_union_example(Examples0, SubFields) -> - TargetKeys = lists:sort([binary_to_atom(Name) || #{name := Name} <- SubFields]), - Examples = - lists:filtermap( - fun(Example) -> - case lists:all(fun(K) -> lists:member(K, TargetKeys) end, maps:keys(Example)) of - true -> {true, ensure_bin_key(Example)}; - false -> false - end - end, - Examples0 - ), - case Examples of - [Example] -> Example; - [] -> #{}; - Other -> throw({error, {find_union_example_failed, Examples, SubFields, Other}}) - end. - -ensure_bin_key(Map) -> - maps:fold( - fun - (K0, V0 = #{}, Acc) -> Acc#{bin(K0) => ensure_bin_key(V0)}; - (K0, V, Acc) -> Acc#{bin(K0) => V} - end, - #{}, - Map - ). - -fmt_desc(#{desc := Desc0}, Indent) -> - Target = iolist_to_binary([?NL, Indent, ?COMMENT2]), - Desc = string:trim(Desc0, both), - replace_nl(Indent, true, Desc, Target); -fmt_desc(_, _) -> - <<"">>. - -fmt_type(Type, Indent) -> - [Indent, ?COMMENT2, ?TYPE, Type, ?NL]. - -fmt_path(Path, Indent) -> [Indent, ?COMMENT2, ?PATH, path(Path), ?NL]. - -path(Stack) when is_list(Stack) -> - string:join(lists:reverse(lists:map(fun str/1, Stack)), "."). - -fmt_fix_header(Field, Type, Path, #{indent := Indent}) -> - [ - fmt_desc(Field, Indent), - fmt_path(Path, Indent), - fmt_type(Type, Indent), - fmt_default(Field, Indent) - ]. - -fmt_map_link(Path0, Type, All, Opts) -> - case Type of - #{values := #{name := ValueName}} -> - fmt_map_link2(Path0, ValueName, All, Opts); - #{values := #{members := Members}} -> - lists:map(fun(M) -> fmt_map_link(Path0, M, All, Opts) end, Members); - _ -> - [] - end. - -fmt_map_link2(Path0, ValueName, All, Opts) -> - Paths = - case maps:find(ValueName, All) of - {ok, #{paths := SubPaths}} -> SubPaths; - _ -> [] - end, - PathStr = path(Path0), - Path = bin(PathStr), - #{indent := Indent} = Opts, - case find_link(Opts, {map, PathStr}) of - {ok, Link} -> - [Indent, ?COMMENT2, ?LINK, Link, ?NL]; - {error, not_found} -> - case lists:member(Path, Paths) of - true -> - insert_link(Opts, [{{map, binary_to_list(P)}, Path} || P <- Paths, P =/= Path]); - false -> - ok - end, - "" - end. - -fmt_union_link(Type = #{members := Members}, Path, Opts = #{indent := Indent}) -> - case find_link(Opts, {union, Type}) of - {ok, Link} -> - link(Link, Indent); - {error, not_found} -> - case is_simple_type(Members) of - true -> ok; - false -> insert_link(Opts, {{union, Type}, Path}) - end, - "" - end. - -fmt_array_link(Type = #{elements := ElemT}, Path, Opts = #{indent := Indent}) -> - case find_link(Opts, {array, Type}) of - {ok, Link} -> - link(Link, Indent); - {error, not_found} -> - case is_simple_type(ElemT) of - true -> ok; - false -> insert_link(Opts, {{array, Type}, Path}) - end, - "" - end. - -link(Link, Indent) -> - [Indent, ?COMMENT2, ?LINK, path(Link), ?NL]. - -is_simple_type(Types) when is_list(Types) -> - lists:all( - fun(#{kind := Kind}) -> - Kind =:= primitive orelse Kind =:= singleton - end, - Types - ); -is_simple_type(Type) -> - is_simple_type([Type]). - -need_comment_example(map, Opts, Path) -> - case find_link(Opts, {map, path(Path)}) of - {ok, _} -> false; - {error, not_found} -> true - end. - -need_comment_example(Type, Opts, Key, Link) when Type =:= union; Type =:= array -> - case find_link(Opts, {union, Key}) of - {ok, Link} -> true; - {error, not_found} -> true; - {ok, _} -> false - end. - -get_examples(_MapName, #{examples := Examples}) -> - ensure_list(Examples); -get_examples(MapName, #{default := #{hocon := Hocon}}) -> - case hocon:binary(Hocon) of - {ok, Default} -> [#{MapName => Default}]; - {error, _} -> [#{MapName => Hocon}] - end; -get_examples(_, _) -> - undefined. - -fmt_examples(Name, #{examples := {union, Examples}}, Opts) -> - fmt_examples(Name, #{examples => Examples}, Opts); -fmt_examples(Name, #{examples := Examples}, Opts) -> - #{indent := Indent, comment := Comment} = Opts, - lists:map( - fun(E) -> - [Indent, comment(Comment), Name, ?BIND, fmt_example(E, Opts), ?NL] - end, - ensure_list(Examples) - ); -fmt_examples(Name, Field, Opts = #{indent := Indent, comment := Comment}) -> - case get_default(Field, Opts) of - undefined -> [Indent, ?COMMENT, Name, ?BIND, ?NL]; - Default -> fmt(Indent, Comment, Name, Default) - end. - -ensure_list(L) when is_list(L) -> L; -ensure_list(T) -> [T]. - -fmt_example(Value, #{indent := Indent0, comment := Comment}) -> - case hocon_pp:do(Value, #{newline => "", embedded => true}) of - [OneLine] -> - [try_to_remove_quote(OneLine)]; - Lines -> - Indent = Indent0 ++ ?INDENT, - Target = iolist_to_binary([?NL, Indent, comment(Comment)]), - [ - ?NL, - Indent, - comment(Comment), - binary:replace(bin(Lines), [<<"\n">>], Target, [global]), - ?NL - ] - end. - -fmt_default(Field, Indent) -> - case get_default(Field, #{indent => Indent, comment => true}) of - undefined -> ""; - Default -> [Indent, ?COMMENT2, ?DEFAULT, Default, ?NL] - end. - -get_default(#{default := Default}, Opts) when is_map(Opts) -> - #{indent := Indent, comment := Comment} = Opts, - get_default(Default, Indent, Comment); -get_default(_, _Opts) -> - undefined. - --define(RE, <<"^[A-Za-z0-9\"]+$">>). - -get_default(#{oneliner := true, hocon := Content}, _Indent, _Comment) -> - try_to_remove_quote(Content); -get_default(#{oneliner := false, hocon := Content}, Indent0, Comment) -> - Target = iolist_to_binary([?NL, Indent0, comment2(Comment), ?INDENT]), - replace_nl(Indent0, Comment, Content, Target); -get_default(Bin, _Indent, _Comment) -> - Bin. - -replace_nl(Indent0, Comment, Content, Target) -> - [ - ?NL, - Indent0, - comment2(Comment), - ?INDENT, - binary:replace(Content, [<<"\n">>], Target, [global]), - ?NL - ]. - -try_to_remove_quote(Content) -> - case re:run(Content, ?RE) of - nomatch -> - Content; - _ -> - case string:trim(Content, both, [$"]) of - <<"">> -> Content; - Other -> Other - end - end. - -bin(S) when is_list(S) -> unicode:characters_to_binary(S, utf8); -bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom); -bin(Int) when is_integer(Int) -> integer_to_binary(Int); -bin(Bin) -> Bin. - -str(A) when is_atom(A) -> atom_to_list(A); -str(S) when is_list(S) -> S; -str(B) when is_binary(B) -> binary_to_list(B); -str({KeyName, _ValName}) -> str(KeyName). - -comment(true) -> ?COMMENT; -comment(false) -> "". - -comment2(true) -> ?COMMENT2; -comment2(false) -> "". - -new_link_cache() -> - ets:new(?MODULE, [private, set, {keypos, 1}]). - -delete_link_cache(#{tid := Tid}) -> - ets:delete(Tid). - -find_link(#{tid := Tid}, Key) -> - case ets:lookup(Tid, Key) of - [{_, Value}] -> {ok, Value}; - [] -> {error, not_found} - end. - -insert_link(#{tid := Tid}, Item) -> - ets:insert(Tid, Item). - -resolve_name({N1, N2}) -> {N1, N2}; -resolve_name(N) -> {N, N}. diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 67e664b0a..b26dfa05a 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -4,7 +4,7 @@ {emqx, {path, "../emqx"}}, %% FIXME: tag this as v3.1.3 {prometheus, {git, "https://github.com/deadtrickster/prometheus.erl", {tag, "v4.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.5"}}} + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.0"}}} ]}. {edoc_opts, [{preprocess, true}]}. diff --git a/mix.exs b/mix.exs index 1fd710fbb..0713cfafc 100644 --- a/mix.exs +++ b/mix.exs @@ -66,7 +66,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.0", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.27.5", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.28.0", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.1", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index d809a01d7..499e58bb5 100644 --- a/rebar.config +++ b/rebar.config @@ -67,7 +67,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.27.5"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.1"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From 26a57a00bbc73b3cc3081257bbdaf91e87773725 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 31 May 2022 11:15:12 +0800 Subject: [PATCH 17/22] fix: cluster crash --- apps/emqx/src/emqx_metrics.erl | 16 +++++++++++----- apps/emqx_conf/etc/emqx_conf.conf | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 529b1b431..8aabd7ac8 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -337,11 +337,17 @@ all() -> %% @doc Get metric value -spec val(metric_name()) -> non_neg_integer(). val(Name) -> - case ets:lookup(?TAB, Name) of - [#metric{idx = Idx}] -> - CRef = persistent_term:get(?MODULE), - counters:get(CRef, Idx); - [] -> + try + case ets:lookup(?TAB, Name) of + [#metric{idx = Idx}] -> + CRef = persistent_term:get(?MODULE), + counters:get(CRef, Idx); + [] -> + 0 + end + %% application will restart when join cluster, then ets not exist. + catch + error:badarg -> 0 end. diff --git a/apps/emqx_conf/etc/emqx_conf.conf b/apps/emqx_conf/etc/emqx_conf.conf index 7d608031d..6f5478c52 100644 --- a/apps/emqx_conf/etc/emqx_conf.conf +++ b/apps/emqx_conf/etc/emqx_conf.conf @@ -13,6 +13,7 @@ node { cookie: emqxsecretcookie data_dir: "{{ platform_data_dir }}" etc_dir: "{{ platform_etc_dir }}" + applications: "{{ emqx_machine_boot_apps }}" } log { From 6c9b959651dc0e21e540a865b233767f9439c3d7 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 31 May 2022 12:32:05 +0800 Subject: [PATCH 18/22] fix: ct failed --- apps/emqx/src/emqx_config.erl | 22 +++++++++++++------ apps/emqx/src/emqx_router_helper.erl | 14 +----------- apps/emqx/src/emqx_schema.erl | 14 ++++++------ apps/emqx/test/emqx_broker_SUITE.erl | 22 +------------------ apps/emqx/test/emqx_common_test_helpers.erl | 3 ++- apps/emqx_conf/src/emqx_conf_app.erl | 12 +++++++++- .../test/emqx_mgmt_api_configs_SUITE.erl | 1 + 7 files changed, 38 insertions(+), 50 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index bfac1f7d2..d9cf42c74 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -306,15 +306,21 @@ put_raw(KeyPath, Config) -> %%============================================================================ init_load(SchemaMod) -> ConfFiles = application:get_env(emqx, config_files, []), - init_load(SchemaMod, ConfFiles). + init_load(SchemaMod, ConfFiles, #{raw_with_default => true}). + +init_load(SchemaMod, Opts) when is_map(Opts) -> + ConfFiles = application:get_env(emqx, config_files, []), + init_load(SchemaMod, ConfFiles, Opts); +init_load(SchemaMod, ConfFiles) -> + init_load(SchemaMod, ConfFiles, #{raw_with_default => false}). %% @doc Initial load of the given config files. %% NOTE: The order of the files is significant, configs from files ordered %% in the rear of the list overrides prior values. -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok. -init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> - init_load(SchemaMod, parse_hocon(Conf)); -init_load(SchemaMod, RawConf) when is_map(RawConf) -> +init_load(SchemaMod, Conf, Opts) when is_list(Conf) orelse is_binary(Conf) -> + init_load(SchemaMod, parse_hocon(Conf), Opts); +init_load(SchemaMod, RawConf, Opts) when is_map(RawConf) -> ok = save_schema_mod_and_names(SchemaMod), %% Merge environment variable overrides on top RawConfWithEnvs = merge_envs(SchemaMod, RawConf), @@ -323,13 +329,13 @@ init_load(SchemaMod, RawConf) when is_map(RawConf) -> Overrides = hocon:deep_merge(ClusterOverrides, LocalOverrides), RawConfWithOverrides = hocon:deep_merge(RawConfWithEnvs, Overrides), RootNames = get_root_names(), - RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides), + RawConfAll = raw_conf_with_default(SchemaMod, RootNames, RawConfWithOverrides, Opts), %% check configs against the schema {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConfAll, #{}), ok = save_to_config_map(CheckedConf, RawConfAll). %% keep the raw and non-raw conf has the same keys to make update raw conf easier. -raw_conf_with_default(SchemaMod, RootNames, RawConf) -> +raw_conf_with_default(SchemaMod, RootNames, RawConf, #{raw_with_default := true}) -> Fun = fun(Name, Acc) -> case maps:is_key(Name, RawConf) of true -> @@ -344,7 +350,9 @@ raw_conf_with_default(SchemaMod, RootNames, RawConf) -> end end, RawDefault = lists:foldl(Fun, #{}, RootNames), - maps:merge(RawConf, fill_defaults(SchemaMod, RawDefault, #{})). + maps:merge(RawConf, fill_defaults(SchemaMod, RawDefault, #{})); +raw_conf_with_default(_SchemaMod, _RootNames, RawConf, _Opts) -> + RawConf. schema_default(Schema) -> case hocon_schema:field_schema(Schema, type) of diff --git a/apps/emqx/src/emqx_router_helper.erl b/apps/emqx/src/emqx_router_helper.erl index a459577b6..1340848ec 100644 --- a/apps/emqx/src/emqx_router_helper.erl +++ b/apps/emqx/src/emqx_router_helper.erl @@ -99,19 +99,7 @@ init([]) -> process_flag(trap_exit, true), ok = ekka:monitor(membership), _ = mria:wait_for_tables([?ROUTING_NODE]), - %{ok, _} = mnesia:subscribe({table, ?ROUTING_NODE, simple}), - %% Temporary fix for debugging: - WhereToRead = ets:lookup(mnesia_gvar, {?ROUTING_NODE, where_to_read}), - case mnesia:subscribe({table, ?ROUTING_NODE, simple}) of - {ok, _} -> - ok; - Err -> - error(#{ - failed_to_subscribe => Err, - where_to_read => WhereToRead, - status => mria_rlog:status() - }) - end, + {ok, _} = mnesia:subscribe({table, ?ROUTING_NODE, simple}), Nodes = lists:foldl( fun(Node, Acc) -> case ekka:is_member(Node) of diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 423ac96f6..8f836a3dc 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -118,7 +118,7 @@ roots(high) -> )}, {"zones", sc( - map("my_zone_name", ref("zone")), + map("name", ref("zone")), #{desc => ?DESC(zones)} )}, {"mqtt", @@ -703,7 +703,7 @@ fields("conn_congestion") -> sc( boolean(), #{ - default => false, + default => true, desc => ?DESC(conn_congestion_enable_alarm) } )}, @@ -744,7 +744,7 @@ fields("listeners") -> [ {"tcp", sc( - map(default, ref("mqtt_tcp_listener")), + map(name, ref("mqtt_tcp_listener")), #{ desc => ?DESC(fields_listeners_tcp), required => {false, recursively} @@ -752,7 +752,7 @@ fields("listeners") -> )}, {"ssl", sc( - map(default, ref("mqtt_ssl_listener")), + map(name, ref("mqtt_ssl_listener")), #{ desc => ?DESC(fields_listeners_ssl), required => {false, recursively} @@ -760,7 +760,7 @@ fields("listeners") -> )}, {"ws", sc( - map(default, ref("mqtt_ws_listener")), + map(name, ref("mqtt_ws_listener")), #{ desc => ?DESC(fields_listeners_ws), required => {false, recursively} @@ -768,7 +768,7 @@ fields("listeners") -> )}, {"wss", sc( - map(default, ref("mqtt_wss_listener")), + map(name, ref("mqtt_wss_listener")), #{ desc => ?DESC(fields_listeners_wss), required => {false, recursively} @@ -776,7 +776,7 @@ fields("listeners") -> )}, {"quic", sc( - map(default, ref("mqtt_quic_listener")), + map(name, ref("mqtt_quic_listener")), #{ desc => ?DESC(fields_listeners_quic), required => {false, recursively} diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index c7a5e990b..398d81614 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -104,23 +104,6 @@ init_per_testcase(Case, Config) -> end_per_testcase(Case, Config) -> ?MODULE:Case({'end', Config}). -set_special_configs(emqx) -> - Quic = #{ - enabled => true, - bind => {{0, 0, 0, 0}, 14567}, - acceptors => 16, - max_connections => 1024000, - keyfile => "etc/certs/key.pem", - certfile => "etc/certs/cert.pem", - mountpoint => <<"">>, - zone => default, - idle_timeout => 15000 - }, - emqx_config:put_listener_conf(quic, default, [], Quic), - ok; -set_special_configs(_) -> - ok. - %%-------------------------------------------------------------------- %% PubSub Test %%-------------------------------------------------------------------- @@ -435,10 +418,7 @@ t_connected_client_count_persistent(Config) when is_list(Config) -> {clientid, ClientID} | Config ]), - {{ok, _}, {ok, [_]}} = wait_for_events( - fun() -> emqtt:ConnFun(ConnPid2) end, - [emqx_cm_connected_client_count_inc] - ), + {{ok, _}, {ok, [_, _]}} = wait_for_events( fun() -> emqtt:ConnFun(ConnPid2) end, [ diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 4e5a0ffbc..ee2d446c7 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -467,7 +467,8 @@ copy_certs(_, _) -> load_config(SchemaModule, Config) -> ok = emqx_config:delete_override_conf_files(), - ok = emqx_config:init_load(SchemaModule, Config). + ok = emqx_config:init_load(SchemaModule, Config), + ok. -spec is_tcp_server_available( Host :: inet:socket_address() | inet:hostname(), diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 1ad460043..61c56a398 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -60,10 +60,20 @@ get_override_config_file() -> %% Internal functions %% ------------------------------------------------------------------------------ +-ifdef(TEST). +init_load() -> + emqx_config:init_load(emqx_conf:schema_module(), #{raw_with_default => false}). + +-else. + +init_load() -> + emqx_config:init_load(emqx_conf:schema_module(), #{raw_with_default => true}). +-endif. + init_conf() -> {ok, TnxId} = copy_override_conf_from_core_node(), emqx_app:set_init_tnx_id(TnxId), - emqx_config:init_load(emqx_conf:schema_module()), + init_load(), emqx_app:set_init_config_load_done(). cluster_nodes() -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index ba87812a6..e2e73fe1f 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -178,6 +178,7 @@ t_dashboard(_Config) -> {ok, Dashboard1} = get_config("dashboard"), ?assertNotEqual(Dashboard, Dashboard1), + timer:sleep(1000), ok. get_config(Name) -> From 0fcc02caf4565d783d4819f264f73ab3345b4a06 Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 31 May 2022 18:01:09 +0800 Subject: [PATCH 19/22] chore: comment quic listener from emqx.conf --- apps/emqx/etc/emqx.conf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index a19bf1dab..372ea99b7 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -31,10 +31,10 @@ listeners.wss.default { } } -listeners.quic.default { - enabled: true - bind: "0.0.0.0:14567" - max_connections: 1024000 - keyfile: "{{ platform_etc_dir }}/certs/key.pem" - certfile: "{{ platform_etc_dir }}/certs/cert.pem" -} +# listeners.quic.default { +# enabled: false +# bind: "0.0.0.0:14567" +# max_connections: 1024000 +# keyfile: "{{ platform_etc_dir }}/certs/key.pem" +# certfile: "{{ platform_etc_dir }}/certs/cert.pem" +#} From fe5998d7739439112b7ca7caa88c604a55f3c34f Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 31 May 2022 18:05:11 +0800 Subject: [PATCH 20/22] chore: bump dashboard to v0.35.0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 57d07c522..68d593165 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-15:1.13.3-24.2.1-1-a export EMQX_DEFAULT_RUNNER = alpine:3.15.1 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v0.34.0 +export EMQX_DASHBOARD_VERSION ?= v0.35.0 export DOCKERFILE := deploy/docker/Dockerfile export EMQX_REL_FORM ?= tgz ifeq ($(OS),Windows_NT) From d61b44ef977cddb5090130c2bf7d7dbcf0687eab Mon Sep 17 00:00:00 2001 From: Zhongwen Deng Date: Tue, 31 May 2022 19:20:09 +0800 Subject: [PATCH 21/22] fix: ct failed again --- apps/emqx/src/emqx_config.erl | 1 + apps/emqx/test/emqx_common_test_helpers.erl | 13 ++++++++-- apps/emqx_bridge/test/emqx_bridge_SUITE.erl | 5 ++-- .../test/emqx_delayed_api_SUITE.erl | 4 +++- .../test/emqx_modules_conf_SUITE.erl | 4 +++- apps/emqx_modules/test/emqx_rewrite_SUITE.erl | 20 ++++++++++++---- .../test/emqx_rewrite_api_SUITE.erl | 4 +++- .../test/emqx_telemetry_SUITE.erl | 24 ++++++++++++++++++- .../test/emqx_telemetry_api_SUITE.erl | 4 +++- .../test/emqx_topic_metrics_SUITE.erl | 4 +++- .../test/emqx_topic_metrics_api_SUITE.erl | 4 +++- 11 files changed, 71 insertions(+), 16 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index d9cf42c74..5e50189be 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -22,6 +22,7 @@ -export([ init_load/1, init_load/2, + init_load/3, read_override_conf/1, delete_override_conf_files/0, check_config/2, diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index ee2d446c7..38064914c 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -51,6 +51,7 @@ render_config_file/2, read_schema_configs/2, load_config/2, + load_config/3, is_tcp_server_available/2, is_tcp_server_available/3 ]). @@ -465,11 +466,19 @@ copy_certs(emqx_conf, Dest0) -> copy_certs(_, _) -> ok. -load_config(SchemaModule, Config) -> +load_config(SchemaModule, Config, Opts) -> + ConfigBin = + case is_map(Config) of + true -> jsx:encode(Config); + false -> Config + end, ok = emqx_config:delete_override_conf_files(), - ok = emqx_config:init_load(SchemaModule, Config), + ok = emqx_config:init_load(SchemaModule, ConfigBin, Opts), ok. +load_config(SchemaModule, Config) -> + load_config(SchemaModule, Config, #{raw_with_default => false}). + -spec is_tcp_server_available( Host :: inet:socket_address() | inet:hostname(), Port :: inet:port_number() diff --git a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl index fb7d1e314..894322c61 100644 --- a/apps/emqx_bridge/test/emqx_bridge_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_SUITE.erl @@ -142,8 +142,9 @@ setup_fake_telemetry_data() -> } } }, - ok = emqx_common_test_helpers:load_config(emqx_connector_schema, ConnectorConf), - ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, Conf), + Opts = #{raw_with_default => true}, + ok = emqx_common_test_helpers:load_config(emqx_connector_schema, ConnectorConf, Opts), + ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, Conf, Opts), ok = snabbkaffe:start_trace(), Predicate = fun(#{?snk_kind := K}) -> K =:= emqx_bridge_monitor_loaded_bridge end, diff --git a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl index 590fe24e6..0a17e12d2 100644 --- a/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_delayed_api_SUITE.erl @@ -32,7 +32,9 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?BASE_CONF), #{ + raw_with_default => true + }), ok = emqx_common_test_helpers:start_apps( [emqx_conf, emqx_modules, emqx_dashboard], diff --git a/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl b/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl index 72258c413..10847e8ec 100644 --- a/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl +++ b/apps/emqx_modules/test/emqx_modules_conf_SUITE.erl @@ -29,7 +29,9 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Conf) -> - emqx_common_test_helpers:load_config(emqx_modules_schema, <<"gateway {}">>), + emqx_common_test_helpers:load_config(emqx_modules_schema, <<"gateway {}">>, #{ + raw_with_default => true + }), emqx_common_test_helpers:start_apps([emqx_conf, emqx_modules]), Conf. diff --git a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl index 7a6f096d5..4fc3816fa 100644 --- a/apps/emqx_modules/test/emqx_rewrite_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_SUITE.erl @@ -157,13 +157,17 @@ t_rewrite_re_error(_Config) -> ok. t_list(_Config) -> - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?REWRITE), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?REWRITE), #{ + raw_with_default => true + }), Expect = maps:get(<<"rewrite">>, ?REWRITE), ?assertEqual(Expect, emqx_rewrite:list()), ok. t_update(_Config) -> - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?REWRITE), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?REWRITE), #{ + raw_with_default => true + }), Init = emqx_rewrite:list(), Rules = [ #{ @@ -179,7 +183,9 @@ t_update(_Config) -> ok. t_update_disable(_Config) -> - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?REWRITE), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?REWRITE), #{ + raw_with_default => true + }), ?assertEqual(ok, emqx_rewrite:update([])), timer:sleep(150), @@ -194,7 +200,9 @@ t_update_disable(_Config) -> ok. t_update_re_failed(_Config) -> - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?REWRITE), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?REWRITE), #{ + raw_with_default => true + }), Re = <<"*^test/*">>, Rules = [ #{ @@ -249,7 +257,9 @@ receive_publish(Timeout) -> end. init() -> - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?REWRITE), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?REWRITE), #{ + raw_with_default => true + }), ok = emqx_rewrite:enable(), {ok, C} = emqtt:start_link([{clientid, <<"c1">>}, {username, <<"u1">>}]), {ok, _} = emqtt:connect(C), diff --git a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl index 4702b7d88..d837006dd 100644 --- a/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_rewrite_api_SUITE.erl @@ -33,7 +33,9 @@ init_per_testcase(_, Config) -> Config. init_per_suite(Config) -> - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?BASE_CONF), #{ + raw_with_default => true + }), ok = emqx_common_test_helpers:start_apps( [emqx_conf, emqx_modules, emqx_dashboard], diff --git a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl index 7965d187e..ac5c8bf35 100644 --- a/apps/emqx_modules/test/emqx_telemetry_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_SUITE.erl @@ -25,6 +25,11 @@ -import(proplists, [get_value/2]). +-define(BASE_CONF, #{ + <<"dealyed">> => <<"true">>, + <<"max_delayed_messages">> => <<"0">> +}). + all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> @@ -36,6 +41,9 @@ init_per_suite(Config) -> emqx_common_test_helpers:deps_path(emqx_authz, "etc/acl.conf") end ), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?BASE_CONF), #{ + raw_with_default => true + }), emqx_common_test_helpers:start_apps( [emqx_conf, emqx_authn, emqx_authz, emqx_modules], fun set_special_configs/1 @@ -144,7 +152,9 @@ init_per_testcase(t_exhook_info, Config) -> {ok, _} = emqx_exhook_demo_svr:start(), {ok, Sock} = gen_tcp:connect("localhost", 9000, [], 3000), _ = gen_tcp:close(Sock), - ok = emqx_common_test_helpers:load_config(emqx_exhook_schema, ExhookConf), + ok = emqx_common_test_helpers:load_config(emqx_exhook_schema, ExhookConf, #{ + raw_with_default => true + }), {ok, _} = application:ensure_all_started(emqx_exhook), Config; init_per_testcase(t_cluster_uuid, Config) -> @@ -166,6 +176,9 @@ init_per_testcase(t_uuid_restored_from_file, Config) -> %% clear the UUIDs in the DB {atomic, ok} = mria:clear_table(emqx_telemetry), emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authn, emqx_authz, emqx_modules]), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?BASE_CONF), #{ + raw_with_default => true + }), emqx_common_test_helpers:start_apps( [emqx_conf, emqx_authn, emqx_authz, emqx_modules], fun set_special_configs/1 @@ -319,6 +332,9 @@ t_uuid_saved_to_file(_Config) -> %% clear the UUIDs in the DB {atomic, ok} = mria:clear_table(emqx_telemetry), emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authn, emqx_authz, emqx_modules]), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?BASE_CONF), #{ + raw_with_default => true + }), emqx_common_test_helpers:start_apps( [emqx_conf, emqx_authn, emqx_authz, emqx_modules], fun set_special_configs/1 @@ -841,6 +857,12 @@ setup_slave(Node) -> (_) -> ok end, + ok = rpc:call( + Node, + emqx_common_test_helpers, + load_config, + [emqx_modules_schema, jsx:encode(?BASE_CONF), #{raw_with_default => true}] + ), ok = rpc:call( Node, emqx_common_test_helpers, diff --git a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl index 4ebd155d2..8c55583b6 100644 --- a/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_telemetry_api_SUITE.erl @@ -29,7 +29,9 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?BASE_CONF), #{ + raw_with_default => true + }), ok = emqx_common_test_helpers:start_apps( [emqx_conf, emqx_authn, emqx_authz, emqx_modules, emqx_dashboard], diff --git a/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl index 4e87374cc..05949a648 100644 --- a/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl +++ b/apps/emqx_modules/test/emqx_topic_metrics_SUITE.erl @@ -29,7 +29,9 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> emqx_common_test_helpers:boot_modules(all), emqx_common_test_helpers:start_apps([emqx_conf, emqx_modules]), - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?TOPIC), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?TOPIC), #{ + raw_with_default => true + }), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl index 6bb6cac14..8e57a8b79 100644 --- a/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl +++ b/apps/emqx_modules/test/emqx_topic_metrics_api_SUITE.erl @@ -40,7 +40,9 @@ init_per_testcase(_, Config) -> Config. init_per_suite(Config) -> - ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF), + ok = emqx_common_test_helpers:load_config(emqx_modules_schema, jsx:encode(?BASE_CONF), #{ + raw_with_default => true + }), ok = emqx_common_test_helpers:start_apps( [emqx_conf, emqx_modules, emqx_dashboard], From 11ef46bc1df6d2628dbcbd3671c9189f7d46cbba Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 31 May 2022 20:46:48 +0800 Subject: [PATCH 22/22] fix: fix badmatch for emqx_os_mon:update_mem_alarm_status/1 --- apps/emqx/src/emqx_os_mon.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index 63c8e0298..74acfff93 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -193,7 +193,8 @@ update_mem_alarm_status(HWM) when HWM > 1.0 orelse HWM < 0.0 -> ); update_mem_alarm_status(HWM) -> is_sysmem_check_supported() andalso - do_update_mem_alarm_status(HWM). + do_update_mem_alarm_status(HWM), + ok. do_update_mem_alarm_status(HWM0) -> HWM = HWM0 * 100,