From 49c4ea9ae96d4438fe9cdcc97c16e0f80297b12e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 22 Jun 2023 14:50:31 +0200 Subject: [PATCH 01/13] refactor: mvoe config examples to 'rel/config' --- mix.exs | 2 +- rebar.config.erl | 2 +- {examples => rel/config/examples}/README.md | 0 {examples => rel/config/examples}/alarm.conf.example | 0 {examples => rel/config/examples}/cluster-with-dns.conf.example | 0 .../config/examples}/cluster-with-etcd-ssl.conf.example | 0 .../config/examples}/cluster-with-etcd.conf.example | 0 {examples => rel/config/examples}/cluster-with-k8s.conf.example | 0 .../config/examples}/cluster-with-manual.conf.example | 0 .../config/examples}/cluster-with-static.conf.example | 0 {examples => rel/config/examples}/conn_congestion.conf.example | 0 .../config/examples}/dashboard-with-http.conf.example | 0 .../config/examples}/dashboard-with-https.conf.example | 0 {examples => rel/config/examples}/delayed.conf.example | 0 {examples => rel/config/examples}/exhook.conf.example | 0 .../examples}/file_transfer-with-local-exporter.conf.example | 0 .../examples}/file_transfer-with-s3-exporter.conf.example | 0 {examples => rel/config/examples}/flapping_detect.conf.example | 0 {examples => rel/config/examples}/force_gc.conf.example | 0 {examples => rel/config/examples}/force_shutdown.conf.example | 0 {examples => rel/config/examples}/gateway.coap.conf.example | 0 {examples => rel/config/examples}/gateway.exproto.conf.example | 0 {examples => rel/config/examples}/gateway.lwm2m.conf.example | 0 {examples => rel/config/examples}/gateway.mqttsn.conf.example | 0 {examples => rel/config/examples}/gateway.stomp.conf.example | 0 {examples => rel/config/examples}/license.conf.example | 0 {examples => rel/config/examples}/listeners.quic.conf.example | 0 {examples => rel/config/examples}/listeners.ssl.conf.example | 0 {examples => rel/config/examples}/listeners.tcp.conf.example | 0 {examples => rel/config/examples}/listeners.ws.conf.example | 0 {examples => rel/config/examples}/listeners.wss.conf.example | 0 {examples => rel/config/examples}/log.console.conf.example | 0 {examples => rel/config/examples}/log.file.conf.example | 0 {examples => rel/config/examples}/mqtt.conf.example | 0 {examples => rel/config/examples}/node.conf.example | 0 {examples => rel/config/examples}/plugin.conf.example | 0 {examples => rel/config/examples}/prometheus.conf.example | 0 .../config/examples}/psk_authentication.conf.example | 0 {examples => rel/config/examples}/retainer.conf.example | 0 {examples => rel/config/examples}/sys_topics.conf.example | 0 {examples => rel/config/examples}/sysmon.os.conf.example | 0 {examples => rel/config/examples}/sysmon.vm.conf.example | 0 42 files changed, 2 insertions(+), 2 deletions(-) rename {examples => rel/config/examples}/README.md (100%) rename {examples => rel/config/examples}/alarm.conf.example (100%) rename {examples => rel/config/examples}/cluster-with-dns.conf.example (100%) rename {examples => rel/config/examples}/cluster-with-etcd-ssl.conf.example (100%) rename {examples => rel/config/examples}/cluster-with-etcd.conf.example (100%) rename {examples => rel/config/examples}/cluster-with-k8s.conf.example (100%) rename {examples => rel/config/examples}/cluster-with-manual.conf.example (100%) rename {examples => rel/config/examples}/cluster-with-static.conf.example (100%) rename {examples => rel/config/examples}/conn_congestion.conf.example (100%) rename {examples => rel/config/examples}/dashboard-with-http.conf.example (100%) rename {examples => rel/config/examples}/dashboard-with-https.conf.example (100%) rename {examples => rel/config/examples}/delayed.conf.example (100%) rename {examples => rel/config/examples}/exhook.conf.example (100%) rename {examples => rel/config/examples}/file_transfer-with-local-exporter.conf.example (100%) rename {examples => rel/config/examples}/file_transfer-with-s3-exporter.conf.example (100%) rename {examples => rel/config/examples}/flapping_detect.conf.example (100%) rename {examples => rel/config/examples}/force_gc.conf.example (100%) rename {examples => rel/config/examples}/force_shutdown.conf.example (100%) rename {examples => rel/config/examples}/gateway.coap.conf.example (100%) rename {examples => rel/config/examples}/gateway.exproto.conf.example (100%) rename {examples => rel/config/examples}/gateway.lwm2m.conf.example (100%) rename {examples => rel/config/examples}/gateway.mqttsn.conf.example (100%) rename {examples => rel/config/examples}/gateway.stomp.conf.example (100%) rename {examples => rel/config/examples}/license.conf.example (100%) rename {examples => rel/config/examples}/listeners.quic.conf.example (100%) rename {examples => rel/config/examples}/listeners.ssl.conf.example (100%) rename {examples => rel/config/examples}/listeners.tcp.conf.example (100%) rename {examples => rel/config/examples}/listeners.ws.conf.example (100%) rename {examples => rel/config/examples}/listeners.wss.conf.example (100%) rename {examples => rel/config/examples}/log.console.conf.example (100%) rename {examples => rel/config/examples}/log.file.conf.example (100%) rename {examples => rel/config/examples}/mqtt.conf.example (100%) rename {examples => rel/config/examples}/node.conf.example (100%) rename {examples => rel/config/examples}/plugin.conf.example (100%) rename {examples => rel/config/examples}/prometheus.conf.example (100%) rename {examples => rel/config/examples}/psk_authentication.conf.example (100%) rename {examples => rel/config/examples}/retainer.conf.example (100%) rename {examples => rel/config/examples}/sys_topics.conf.example (100%) rename {examples => rel/config/examples}/sysmon.os.conf.example (100%) rename {examples => rel/config/examples}/sysmon.vm.conf.example (100%) diff --git a/mix.exs b/mix.exs index cf75bcb87..5ded486c9 100644 --- a/mix.exs +++ b/mix.exs @@ -538,7 +538,7 @@ defmodule EMQXUmbrella.MixProject do profile = System.get_env("MIX_ENV") File.cp_r!( - "examples", + "rel/config/examples", Path.join(etc, "examples"), force: overwrite? ) diff --git a/rebar.config.erl b/rebar.config.erl index 8aa65a5e8..f8f96e82c 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -525,7 +525,7 @@ etc_overlay(ReleaseType, _Edition) -> [ {mkdir, "etc/"}, {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"}, - {copy, "examples", "etc/"} + {copy, "rel/config/examples", "etc/"} ] ++ lists:map( fun diff --git a/examples/README.md b/rel/config/examples/README.md similarity index 100% rename from examples/README.md rename to rel/config/examples/README.md diff --git a/examples/alarm.conf.example b/rel/config/examples/alarm.conf.example similarity index 100% rename from examples/alarm.conf.example rename to rel/config/examples/alarm.conf.example diff --git a/examples/cluster-with-dns.conf.example b/rel/config/examples/cluster-with-dns.conf.example similarity index 100% rename from examples/cluster-with-dns.conf.example rename to rel/config/examples/cluster-with-dns.conf.example diff --git a/examples/cluster-with-etcd-ssl.conf.example b/rel/config/examples/cluster-with-etcd-ssl.conf.example similarity index 100% rename from examples/cluster-with-etcd-ssl.conf.example rename to rel/config/examples/cluster-with-etcd-ssl.conf.example diff --git a/examples/cluster-with-etcd.conf.example b/rel/config/examples/cluster-with-etcd.conf.example similarity index 100% rename from examples/cluster-with-etcd.conf.example rename to rel/config/examples/cluster-with-etcd.conf.example diff --git a/examples/cluster-with-k8s.conf.example b/rel/config/examples/cluster-with-k8s.conf.example similarity index 100% rename from examples/cluster-with-k8s.conf.example rename to rel/config/examples/cluster-with-k8s.conf.example diff --git a/examples/cluster-with-manual.conf.example b/rel/config/examples/cluster-with-manual.conf.example similarity index 100% rename from examples/cluster-with-manual.conf.example rename to rel/config/examples/cluster-with-manual.conf.example diff --git a/examples/cluster-with-static.conf.example b/rel/config/examples/cluster-with-static.conf.example similarity index 100% rename from examples/cluster-with-static.conf.example rename to rel/config/examples/cluster-with-static.conf.example diff --git a/examples/conn_congestion.conf.example b/rel/config/examples/conn_congestion.conf.example similarity index 100% rename from examples/conn_congestion.conf.example rename to rel/config/examples/conn_congestion.conf.example diff --git a/examples/dashboard-with-http.conf.example b/rel/config/examples/dashboard-with-http.conf.example similarity index 100% rename from examples/dashboard-with-http.conf.example rename to rel/config/examples/dashboard-with-http.conf.example diff --git a/examples/dashboard-with-https.conf.example b/rel/config/examples/dashboard-with-https.conf.example similarity index 100% rename from examples/dashboard-with-https.conf.example rename to rel/config/examples/dashboard-with-https.conf.example diff --git a/examples/delayed.conf.example b/rel/config/examples/delayed.conf.example similarity index 100% rename from examples/delayed.conf.example rename to rel/config/examples/delayed.conf.example diff --git a/examples/exhook.conf.example b/rel/config/examples/exhook.conf.example similarity index 100% rename from examples/exhook.conf.example rename to rel/config/examples/exhook.conf.example diff --git a/examples/file_transfer-with-local-exporter.conf.example b/rel/config/examples/file_transfer-with-local-exporter.conf.example similarity index 100% rename from examples/file_transfer-with-local-exporter.conf.example rename to rel/config/examples/file_transfer-with-local-exporter.conf.example diff --git a/examples/file_transfer-with-s3-exporter.conf.example b/rel/config/examples/file_transfer-with-s3-exporter.conf.example similarity index 100% rename from examples/file_transfer-with-s3-exporter.conf.example rename to rel/config/examples/file_transfer-with-s3-exporter.conf.example diff --git a/examples/flapping_detect.conf.example b/rel/config/examples/flapping_detect.conf.example similarity index 100% rename from examples/flapping_detect.conf.example rename to rel/config/examples/flapping_detect.conf.example diff --git a/examples/force_gc.conf.example b/rel/config/examples/force_gc.conf.example similarity index 100% rename from examples/force_gc.conf.example rename to rel/config/examples/force_gc.conf.example diff --git a/examples/force_shutdown.conf.example b/rel/config/examples/force_shutdown.conf.example similarity index 100% rename from examples/force_shutdown.conf.example rename to rel/config/examples/force_shutdown.conf.example diff --git a/examples/gateway.coap.conf.example b/rel/config/examples/gateway.coap.conf.example similarity index 100% rename from examples/gateway.coap.conf.example rename to rel/config/examples/gateway.coap.conf.example diff --git a/examples/gateway.exproto.conf.example b/rel/config/examples/gateway.exproto.conf.example similarity index 100% rename from examples/gateway.exproto.conf.example rename to rel/config/examples/gateway.exproto.conf.example diff --git a/examples/gateway.lwm2m.conf.example b/rel/config/examples/gateway.lwm2m.conf.example similarity index 100% rename from examples/gateway.lwm2m.conf.example rename to rel/config/examples/gateway.lwm2m.conf.example diff --git a/examples/gateway.mqttsn.conf.example b/rel/config/examples/gateway.mqttsn.conf.example similarity index 100% rename from examples/gateway.mqttsn.conf.example rename to rel/config/examples/gateway.mqttsn.conf.example diff --git a/examples/gateway.stomp.conf.example b/rel/config/examples/gateway.stomp.conf.example similarity index 100% rename from examples/gateway.stomp.conf.example rename to rel/config/examples/gateway.stomp.conf.example diff --git a/examples/license.conf.example b/rel/config/examples/license.conf.example similarity index 100% rename from examples/license.conf.example rename to rel/config/examples/license.conf.example diff --git a/examples/listeners.quic.conf.example b/rel/config/examples/listeners.quic.conf.example similarity index 100% rename from examples/listeners.quic.conf.example rename to rel/config/examples/listeners.quic.conf.example diff --git a/examples/listeners.ssl.conf.example b/rel/config/examples/listeners.ssl.conf.example similarity index 100% rename from examples/listeners.ssl.conf.example rename to rel/config/examples/listeners.ssl.conf.example diff --git a/examples/listeners.tcp.conf.example b/rel/config/examples/listeners.tcp.conf.example similarity index 100% rename from examples/listeners.tcp.conf.example rename to rel/config/examples/listeners.tcp.conf.example diff --git a/examples/listeners.ws.conf.example b/rel/config/examples/listeners.ws.conf.example similarity index 100% rename from examples/listeners.ws.conf.example rename to rel/config/examples/listeners.ws.conf.example diff --git a/examples/listeners.wss.conf.example b/rel/config/examples/listeners.wss.conf.example similarity index 100% rename from examples/listeners.wss.conf.example rename to rel/config/examples/listeners.wss.conf.example diff --git a/examples/log.console.conf.example b/rel/config/examples/log.console.conf.example similarity index 100% rename from examples/log.console.conf.example rename to rel/config/examples/log.console.conf.example diff --git a/examples/log.file.conf.example b/rel/config/examples/log.file.conf.example similarity index 100% rename from examples/log.file.conf.example rename to rel/config/examples/log.file.conf.example diff --git a/examples/mqtt.conf.example b/rel/config/examples/mqtt.conf.example similarity index 100% rename from examples/mqtt.conf.example rename to rel/config/examples/mqtt.conf.example diff --git a/examples/node.conf.example b/rel/config/examples/node.conf.example similarity index 100% rename from examples/node.conf.example rename to rel/config/examples/node.conf.example diff --git a/examples/plugin.conf.example b/rel/config/examples/plugin.conf.example similarity index 100% rename from examples/plugin.conf.example rename to rel/config/examples/plugin.conf.example diff --git a/examples/prometheus.conf.example b/rel/config/examples/prometheus.conf.example similarity index 100% rename from examples/prometheus.conf.example rename to rel/config/examples/prometheus.conf.example diff --git a/examples/psk_authentication.conf.example b/rel/config/examples/psk_authentication.conf.example similarity index 100% rename from examples/psk_authentication.conf.example rename to rel/config/examples/psk_authentication.conf.example diff --git a/examples/retainer.conf.example b/rel/config/examples/retainer.conf.example similarity index 100% rename from examples/retainer.conf.example rename to rel/config/examples/retainer.conf.example diff --git a/examples/sys_topics.conf.example b/rel/config/examples/sys_topics.conf.example similarity index 100% rename from examples/sys_topics.conf.example rename to rel/config/examples/sys_topics.conf.example diff --git a/examples/sysmon.os.conf.example b/rel/config/examples/sysmon.os.conf.example similarity index 100% rename from examples/sysmon.os.conf.example rename to rel/config/examples/sysmon.os.conf.example diff --git a/examples/sysmon.vm.conf.example b/rel/config/examples/sysmon.vm.conf.example similarity index 100% rename from examples/sysmon.vm.conf.example rename to rel/config/examples/sysmon.vm.conf.example From e0d05a02ff970a92c7a0bd6b8dc412b5854ccf5d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 22 Jun 2023 17:08:54 +0200 Subject: [PATCH 02/13] refactor: refine examples Also turns off vm_dist_collector metrics collector by default --- .../src/emqx_prometheus_schema.erl | 14 ++--- rel/config/examples/README.md | 18 +++++-- rel/config/examples/alarm.conf.example | 6 --- .../examples/cluster-with-dns.conf.example | 11 +--- .../cluster-with-etcd-ssl.conf.example | 54 ++----------------- .../examples/cluster-with-etcd.conf.example | 12 +---- .../examples/cluster-with-k8s.conf.example | 11 +--- .../examples/cluster-with-manual.conf.example | 11 +--- .../examples/cluster-with-static.conf.example | 11 +--- .../examples/conn_congestion.conf.example | 4 -- .../examples/dashboard-with-http.conf.example | 8 +-- .../dashboard-with-https.conf.example | 8 +-- rel/config/examples/exhook.conf.example | 4 +- ..._transfer-with-local-exporter.conf.example | 10 +--- ...ile_transfer-with-s3-exporter.conf.example | 9 +--- .../examples/flapping_detect.conf.example | 10 ++-- rel/config/examples/force_gc.conf.example | 10 +--- .../examples/force_shutdown.conf.example | 8 +-- .../examples/gateway.exproto.conf.example | 6 --- .../examples/gateway.lwm2m.conf.example | 6 --- .../examples/gateway.mqttsn.conf.example | 6 --- .../examples/gateway.stomp.conf.example | 6 --- rel/config/examples/license.conf.example | 10 +--- .../examples/listeners.quic.conf.example | 15 ++---- .../examples/listeners.ssl.conf.example | 22 +++----- .../examples/listeners.tcp.conf.example | 11 +--- rel/config/examples/listeners.ws.conf.example | 14 +---- .../examples/listeners.wss.conf.example | 18 ++----- rel/config/examples/log.console.conf.example | 8 +-- rel/config/examples/log.file.conf.example | 13 ++--- rel/config/examples/mqtt.conf.example | 11 ++-- rel/config/examples/node.conf.example | 34 ++++++------ rel/config/examples/plugin.conf.example | 9 +--- .../prometheus-pushgateway.conf.example | 28 ++++++++++ rel/config/examples/prometheus.conf.example | 32 +++-------- .../examples/psk_authentication.conf.example | 8 +-- rel/config/examples/retainer.conf.example | 16 ++---- rel/config/examples/sys_topics.conf.example | 5 -- rel/config/examples/sysmon.os.conf.example | 14 ++--- rel/config/examples/sysmon.vm.conf.example | 8 +-- rel/i18n/emqx_conf_schema.hocon | 6 +-- rel/i18n/emqx_prometheus_schema.hocon | 6 ++- rel/i18n/emqx_retainer_schema.hocon | 6 +-- 43 files changed, 144 insertions(+), 393 deletions(-) create mode 100644 rel/config/examples/prometheus-pushgateway.conf.example diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index d657e0772..82d951ab3 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -88,9 +88,9 @@ fields("prometheus") -> ?HOCON( hoconsc:enum([enabled, disabled]), #{ - default => enabled, + default => disabled, required => true, - importance => ?IMPORTANCE_HIDDEN, + importance => ?IMPORTANCE_LOW, desc => ?DESC(vm_dist_collector) } )}, @@ -100,7 +100,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - importance => ?IMPORTANCE_HIDDEN, + importance => ?IMPORTANCE_LOW, desc => ?DESC(mnesia_collector) } )}, @@ -110,7 +110,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - importance => ?IMPORTANCE_HIDDEN, + importance => ?IMPORTANCE_LOW, desc => ?DESC(vm_statistics_collector) } )}, @@ -120,7 +120,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - importance => ?IMPORTANCE_HIDDEN, + importance => ?IMPORTANCE_LOW, desc => ?DESC(vm_system_info_collector) } )}, @@ -130,7 +130,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - importance => ?IMPORTANCE_HIDDEN, + importance => ?IMPORTANCE_LOW, desc => ?DESC(vm_memory_collector) } )}, @@ -140,7 +140,7 @@ fields("prometheus") -> #{ default => enabled, required => true, - importance => ?IMPORTANCE_HIDDEN, + importance => ?IMPORTANCE_LOW, desc => ?DESC(vm_msacc_collector) } )} diff --git a/rel/config/examples/README.md b/rel/config/examples/README.md index 013939394..326b5c730 100644 --- a/rel/config/examples/README.md +++ b/rel/config/examples/README.md @@ -1,16 +1,24 @@ # Examples -Here are examples of how to configure features In EMQX, most of them can be used directly by copy-paste content into the `emqx.conf` file, others may need to be slightly modified to use, for example, you should change the listener port or HTTP URL to what you actually used. +Here are examples of how to configure EMQX. +The main purpose of the examples are to serve as a reference for configuration layout and syntax, but not a guide to how to configure EMQX. +For more information about EMQX configuration, please refer to EMQX documentation (links below). -Although we have tried to show every configurable field in the example, -you do not need to care about each one, since most of them already have default values and can be omitted in the configuration file. +There are two ways to extend the configuration of EMQX: -If you are confused about some fields, please refer to our documents, here are just some simple configuration examples with necessary descriptions. +* By adding the configs to `emqx.conf` file. +* By adding the configs to a new file and include it in `emqx.conf` file. For example, add `include "mylisteners.conf"` to `emqx.conf` file and add the listeners to `mylisteners.conf`. +EMQX configuration consists of two parts: static configs and dynamic configs. + +* Configs loaded from `emqx.conf` (and included files) are static configs. +* Configs added or updated from the dashboard or CLI are dynamic configs which are synced to all nodes in the cluster and stored in the data directory on each node. + +It is important to note that static configs are loaded when EMQX starts and overlays on top of the dynamic configs, +to avoid confusion, it is highly recommended NOT to use the same config name for both static and dynamic configs. ## Documentation The EMQX documentation is available at [www.emqx.io/docs/en/latest/](https://www.emqx.io/docs/en/latest/). The EMQX Enterprise documentation is available at [docs.emqx.com/en/](https://docs.emqx.com/en/). - diff --git a/rel/config/examples/alarm.conf.example b/rel/config/examples/alarm.conf.example index 537341ea2..e7f505b59 100644 --- a/rel/config/examples/alarm.conf.example +++ b/rel/config/examples/alarm.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- ## Alarm -## -## Configuring how to handle the alarms generated from sysmon.*.conf.example -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working alarm { ## The actions triggered when the alarm is activated diff --git a/rel/config/examples/cluster-with-dns.conf.example b/rel/config/examples/cluster-with-dns.conf.example index ae617a808..4cade3b54 100644 --- a/rel/config/examples/cluster-with-dns.conf.example +++ b/rel/config/examples/cluster-with-dns.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- -## Cluster in service discovery via DNS SRV records mode -## -## Configs to instruct how individual nodes can discover each other -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Cluster discovery via DNS resolution cluster { ## Human-friendly name of the EMQX cluster. @@ -13,9 +7,6 @@ cluster { ## Service discovery method for the cluster nodes discovery_strategy = dns - ## List of core nodes that the replicant will connect to - core_nodes = ["emqx1@192.168.0.1", "emqx2@192.168.0.2"] - ## If true, the node will try to heal network partitions automatically autoheal = true diff --git a/rel/config/examples/cluster-with-etcd-ssl.conf.example b/rel/config/examples/cluster-with-etcd-ssl.conf.example index d3a31c7d0..9f9486bae 100644 --- a/rel/config/examples/cluster-with-etcd-ssl.conf.example +++ b/rel/config/examples/cluster-with-etcd-ssl.conf.example @@ -1,21 +1,12 @@ -##-------------------------------------------------------------------- -## Cluster in service discovery using 'etcd' service mode -## -## Configs to instruct how individual nodes can discover each other -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Cluster discovery using 'etcd' +## connect to etcd over TLS cluster { - ## Human-friendly name of the EMQX cluster. name = emqxcl ## Service discovery method for the cluster nodes discovery_strategy = etcd - ## List of core nodes that the replicant will connect to - core_nodes = ["emqx1@192.168.0.1", "emqx2@192.168.0.2"] - ## If true, the node will try to heal network partitions automatically autoheal = true @@ -31,51 +22,16 @@ cluster { ssl_options { ## Trusted PEM format CA certificates bundle file - cacertfile = "data/certs/cacert.pem" + cacertfile = "${EMQX_ETC_DIR}/certs/etcd-cacert.pem" ## PEM format certificates chain file - certfile = "data/certs/cert.pem" + certfile = "${EMQX_ETC_DIR}/certs/etcd-client-cert.pem" ## PEM format private key file - keyfile = "data/certs/key.pem" + keyfile = "${EMQX_ETC_DIR}/certs/etcd-client-key.pem" ## Enable or disable peer verification verify = verify_none ## use verify_peer to enable - - ## if `verify' is ebabled, whit true, the connection fails if the client does not have a certificate to send - fail_if_no_peer_cert = false - - ## Enable TLS session reuse - reuse_sessions = true - - ## Maximum number of non-self-issued intermediate certificates that can follow the peer certificate in a valid certification path - depth = 10 - - ## Which versions are to be supported - versions = [tlsv1.3, tlsv1.2] - - ## TLS cipher suite names - ## Note: By default, all available suites are supported, you do not need to set this - ciphers = ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"] - - ## Allows a client and a server to renegotiate the parameters of the SSL connection on the fly - secure_renegotiate = true - - ## Log level for SSL communication - ## Type: emergency | alert | critical | error | warning | notice | info | debug | none | all - log_level = notice - - ## Hibernate the SSL process after idling for amount of time reducing its memory footprint - hibernate_after = 5s - - ## Forces the cipher to be set based on the server-specified order instead of the client-specified order - honor_cipher_order = true - - ## Setting this to false to disable client-initiated renegotiation - client_renegotiation = true - - ## Maximum time duration allowed for the handshake to complete - handshake_timeout = 15s } } } diff --git a/rel/config/examples/cluster-with-etcd.conf.example b/rel/config/examples/cluster-with-etcd.conf.example index 3eae3b980..2ec888b42 100644 --- a/rel/config/examples/cluster-with-etcd.conf.example +++ b/rel/config/examples/cluster-with-etcd.conf.example @@ -1,10 +1,5 @@ -##-------------------------------------------------------------------- -## Cluster in service discovery using 'etcd' service mode -## -## Configs to instruct how individual nodes can discover each other -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Cluster discovery using 'etcd' service +## Connect to etcd over plain TCP (not TLS) cluster { ## Human-friendly name of the EMQX cluster. @@ -13,9 +8,6 @@ cluster { ## Service discovery method for the cluster nodes discovery_strategy = etcd - ## List of core nodes that the replicant will connect to - core_nodes = ["emqx1@192.168.0.1", "emqx2@192.168.0.2"] - ## If true, the node will try to heal network partitions automatically autoheal = true diff --git a/rel/config/examples/cluster-with-k8s.conf.example b/rel/config/examples/cluster-with-k8s.conf.example index 7a0ad3b29..a24409a3c 100644 --- a/rel/config/examples/cluster-with-k8s.conf.example +++ b/rel/config/examples/cluster-with-k8s.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- -## Cluster in service discovery via Kubernetes API server mode -## -## Configs to instruct how individual nodes can discover each other -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Cluster discovery via Kubernetes API server mode cluster { ## Human-friendly name of the EMQX cluster. @@ -13,9 +7,6 @@ cluster { ## Service discovery method for the cluster nodes discovery_strategy = k8s - ## List of core nodes that the replicant will connect to - core_nodes = ["emqx1@192.168.0.1", "emqx2@192.168.0.2"] - ## If true, the node will try to heal network partitions automatically autoheal = true diff --git a/rel/config/examples/cluster-with-manual.conf.example b/rel/config/examples/cluster-with-manual.conf.example index 7c031b44a..7df0f305f 100644 --- a/rel/config/examples/cluster-with-manual.conf.example +++ b/rel/config/examples/cluster-with-manual.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- -## Cluster in service discovery via manual join mode -## -## Configs to instruct how individual nodes can discover each other -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Cluster discovery via `emqx ctl cluster` command cluster { ## Human-friendly name of the EMQX cluster. @@ -13,9 +7,6 @@ cluster { ## Service discovery method for the cluster nodes discovery_strategy = manual - ## List of core nodes that the replicant will connect to - core_nodes = ["emqx1@192.168.0.1", "emqx2@192.168.0.2"] - ## If true, the node will try to heal network partitions automatically autoheal = true } diff --git a/rel/config/examples/cluster-with-static.conf.example b/rel/config/examples/cluster-with-static.conf.example index e2263da15..7013ee7ad 100644 --- a/rel/config/examples/cluster-with-static.conf.example +++ b/rel/config/examples/cluster-with-static.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- -## Cluster in service discovery via static nodes mode -## -## Configs to instruct how individual nodes can discover each other -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Cluster discovery via static seed nodes cluster { ## Human-friendly name of the EMQX cluster. @@ -13,9 +7,6 @@ cluster { ## Service discovery method for the cluster nodes discovery_strategy = static - ## List of core nodes that the replicant will connect to - core_nodes = ["emqx1@192.168.0.1", "emqx2@192.168.0.2"] - ## If true, the node will try to heal network partitions automatically autoheal = true diff --git a/rel/config/examples/conn_congestion.conf.example b/rel/config/examples/conn_congestion.conf.example index e6f3597d1..66a47a65f 100644 --- a/rel/config/examples/conn_congestion.conf.example +++ b/rel/config/examples/conn_congestion.conf.example @@ -1,10 +1,6 @@ -##-------------------------------------------------------------------- ## Connection Congestion ## ## Generating alarm when MQTT connection congested -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working conn_congestion { ## Enable or disable connection congestion alarm diff --git a/rel/config/examples/dashboard-with-http.conf.example b/rel/config/examples/dashboard-with-http.conf.example index 8cf68ab33..e6107e116 100644 --- a/rel/config/examples/dashboard-with-http.conf.example +++ b/rel/config/examples/dashboard-with-http.conf.example @@ -1,10 +1,6 @@ -##-------------------------------------------------------------------- -## Dashboard with HTTP Listener +## Dashboard on HTTP ## -## Configuration for EMQX dashboard -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Configure HTTP for EMQX dashboard dashboard { ## JWT token expiration time diff --git a/rel/config/examples/dashboard-with-https.conf.example b/rel/config/examples/dashboard-with-https.conf.example index 5cc277e47..8e15077e0 100644 --- a/rel/config/examples/dashboard-with-https.conf.example +++ b/rel/config/examples/dashboard-with-https.conf.example @@ -1,10 +1,6 @@ -##-------------------------------------------------------------------- -## Dashboard with HTTPS Listener +## Dashboard on HTTPS ## -## Configuration for EMQX dashboard -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Configure HTTPS for EMQX dashboard dashboard { ## JWT token expiration time diff --git a/rel/config/examples/exhook.conf.example b/rel/config/examples/exhook.conf.example index 8adcfcab9..cf9fffc71 100644 --- a/rel/config/examples/exhook.conf.example +++ b/rel/config/examples/exhook.conf.example @@ -1,8 +1,6 @@ -##-------------------------------------------------------------------- ## gRPC Hook Extension ## -## Allows users to process EMQX Hooks using other programming languages -##-------------------------------------------------------------------- +## Allows users to provide hook callbacks over gRPC for EMQX message lifecycle events. exhook.servers = [ { diff --git a/rel/config/examples/file_transfer-with-local-exporter.conf.example b/rel/config/examples/file_transfer-with-local-exporter.conf.example index 8dbd04f66..860826d38 100644 --- a/rel/config/examples/file_transfer-with-local-exporter.conf.example +++ b/rel/config/examples/file_transfer-with-local-exporter.conf.example @@ -1,12 +1,6 @@ -##-------------------------------------------------------------------- -## File Transfer +## File Transfer over MQTT exporting files to local file system ## -## Enables the File Transfer over MQTT feature -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working - -## Note: This configuration only works for the EMQX Enterprise version +## NOTE: This configuration is only applicable in EMQX Enterprise edition 5.1 or later. file_transfer { ## Enable the File Transfer feature diff --git a/rel/config/examples/file_transfer-with-s3-exporter.conf.example b/rel/config/examples/file_transfer-with-s3-exporter.conf.example index a59c7918c..cba6fada5 100644 --- a/rel/config/examples/file_transfer-with-s3-exporter.conf.example +++ b/rel/config/examples/file_transfer-with-s3-exporter.conf.example @@ -1,12 +1,7 @@ -##-------------------------------------------------------------------- -## File Transfer +## File Transfer over MQTT ## ## Enables the File Transfer over MQTT feature -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working - -## Note: This configuration only works for the EMQX Enterprise version +## Note: This configuration is only applicable for EMQX Enterprise edition 5.1 or later. file_transfer { ## Enable the File Transfer feature diff --git a/rel/config/examples/flapping_detect.conf.example b/rel/config/examples/flapping_detect.conf.example index b33a805ea..fe4522b1e 100644 --- a/rel/config/examples/flapping_detect.conf.example +++ b/rel/config/examples/flapping_detect.conf.example @@ -1,14 +1,10 @@ -##-------------------------------------------------------------------- ## Flapping Detect ## -## Ban the client when the times of connections exceed the limit in the time window -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Ban the client when the times of connections exceed the limit in the configured time window flapping_detect { - ## use false to disabled - enable = true + ## use 'true' to enable this feature + enable = false ## Time window for flapping detection window_time = 1m diff --git a/rel/config/examples/force_gc.conf.example b/rel/config/examples/force_gc.conf.example index e682d723d..ee07ee4d2 100644 --- a/rel/config/examples/force_gc.conf.example +++ b/rel/config/examples/force_gc.conf.example @@ -1,13 +1,7 @@ -##-------------------------------------------------------------------- -## Force garbage collection -## -## Force garbage collection in MQTT connection process after they process certain number of messages or bytes of data -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Force Elrang VM garbage collection force_gc { - ## set to false to disable this + ## set 'false' to disable this feature enable = true ## GC the process after this many received messages diff --git a/rel/config/examples/force_shutdown.conf.example b/rel/config/examples/force_shutdown.conf.example index b049691c6..d0a466aa0 100644 --- a/rel/config/examples/force_shutdown.conf.example +++ b/rel/config/examples/force_shutdown.conf.example @@ -1,13 +1,9 @@ -##-------------------------------------------------------------------- ## Force Shutdown ## -## Forced closing of the overloaded session -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Forced shutdown MQTT clients for overload protection force_shutdown { - ## false to disable this + ## set 'false' to disable force shutdown feature enable = true ## Maximum mailbox size for each Erlang process diff --git a/rel/config/examples/gateway.exproto.conf.example b/rel/config/examples/gateway.exproto.conf.example index 04c95d98c..fcedb944b 100644 --- a/rel/config/examples/gateway.exproto.conf.example +++ b/rel/config/examples/gateway.exproto.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- ## Gateway Exproto -## -## Add an Exproto gateway -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working gateway.exproto { diff --git a/rel/config/examples/gateway.lwm2m.conf.example b/rel/config/examples/gateway.lwm2m.conf.example index 2c9b55c04..60943f47c 100644 --- a/rel/config/examples/gateway.lwm2m.conf.example +++ b/rel/config/examples/gateway.lwm2m.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- ## Gateway LwM2M -## -## Add a LwM2M gateway -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working gateway.lwm2m { diff --git a/rel/config/examples/gateway.mqttsn.conf.example b/rel/config/examples/gateway.mqttsn.conf.example index 7785454f3..aac7d6898 100644 --- a/rel/config/examples/gateway.mqttsn.conf.example +++ b/rel/config/examples/gateway.mqttsn.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- ## Gateway MQTT-SN -## -## Add a MQTT-SN gateway -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working gateway.mqttsn { diff --git a/rel/config/examples/gateway.stomp.conf.example b/rel/config/examples/gateway.stomp.conf.example index ab09a45f7..e90a04620 100644 --- a/rel/config/examples/gateway.stomp.conf.example +++ b/rel/config/examples/gateway.stomp.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- ## Gateway STOMP -## -## Add STOMP CoAP gateway -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working gateway.stomp { diff --git a/rel/config/examples/license.conf.example b/rel/config/examples/license.conf.example index 7444ae795..2cb0dc3bd 100644 --- a/rel/config/examples/license.conf.example +++ b/rel/config/examples/license.conf.example @@ -1,12 +1,6 @@ -##-------------------------------------------------------------------- -## License -## -## Defines the EMQX Enterprise license -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## EMQX Enterprise License -## Note: This configuration only works for the EMQX Enterprise version +## NOTE: This configuration is only applicable for the EMQX Enterprise eidtion license { ## License Key diff --git a/rel/config/examples/listeners.quic.conf.example b/rel/config/examples/listeners.quic.conf.example index 49d4f58a1..52161e828 100644 --- a/rel/config/examples/listeners.quic.conf.example +++ b/rel/config/examples/listeners.quic.conf.example @@ -1,13 +1,6 @@ -##-------------------------------------------------------------------- -## QUIC Listener -## -## Add a QUIC Listener -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## MQTT over QUIC Listener -## Note: Modifying the 'quicname' to what you need -listeners.quic.quicname { +listeners.quic.my_quick_listener_name { ## Port or Address to listen on, 0 means disable bind = 14567 ## or with an IP, e.g. "127.0.0.1:14567" @@ -28,8 +21,8 @@ listeners.quic.quicname { ## Type: infinity | Integer max_connections = infinity - ## TLS cipher suite names - ciphers = ["TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256", "TLS_CHACHA20_POLY1305_SHA256"] + ## TLS v1.3 exclusive cipher suite names + ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256" ssl_options { ## Trusted PEM format CA certificates bundle file diff --git a/rel/config/examples/listeners.ssl.conf.example b/rel/config/examples/listeners.ssl.conf.example index 11078db6c..a33ce7efe 100644 --- a/rel/config/examples/listeners.ssl.conf.example +++ b/rel/config/examples/listeners.ssl.conf.example @@ -1,16 +1,6 @@ -##-------------------------------------------------------------------- -## SSL Listener -## -## Add a SSL Listener -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## MQTT over TLS(SSL) Listener -## The SSL listener also supports all the fields listed in listeners.tcp.conf.example -## only the SSL-specific fields are shown here - -## Note: Modifying the 'sslname' to what you need -listeners.ssl.sslname { +listeners.ssl.my_ssl_listener_name { ## Port or Address to listen on, 0 means disable bind = 8883 ## or with an IP e.g. "127.0.0.1:8883" enabled = true @@ -33,13 +23,13 @@ listeners.ssl.sslname { } ssl_options { ## Trusted PEM format CA certificates bundle file - cacertfile = "data/certs/cacert.pem" + cacertfile = "${EMQX_ETC_DIR}/certs/cacert.pem" ## PEM format certificates chain file - certfile = "data/certs/cert.pem" + certfile = "${EMQX_ETC_DIR}/certs/cert.pem" ## PEM format private key file - keyfile = "data/certs/key.pem" + keyfile = "${EMQX_ETC_DIR}/certs/key.pem" ## Enable or disable peer verification verify = verify_none ## use verify_peer to enable @@ -58,7 +48,7 @@ listeners.ssl.sslname { ## TLS cipher suite names ## Note: By default, all available suites are supported, you do not need to set this - ciphers = ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"] + ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,ECDHE-RSA-AES256-GCM-SHA384" ## Allows a client and a server to renegotiate the parameters of the SSL connection on the fly secure_renegotiate = true diff --git a/rel/config/examples/listeners.tcp.conf.example b/rel/config/examples/listeners.tcp.conf.example index 42e98b071..f03d98cc2 100644 --- a/rel/config/examples/listeners.tcp.conf.example +++ b/rel/config/examples/listeners.tcp.conf.example @@ -1,13 +1,6 @@ -##-------------------------------------------------------------------- -## TCP Listener -## -## Add a TCP Listener -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## MQTT over TCP Listener -## Note: Modifying the 'tcpname' to what you need -listeners.tcp.tcpname { +listeners.tcp.my_tcp_listener_name { ## Port or Address to listen on, 0 means disable bind = 1883 ## or with an IP e.g. "127.0.0.1:1883" diff --git a/rel/config/examples/listeners.ws.conf.example b/rel/config/examples/listeners.ws.conf.example index 60523ac7a..f6a6adae8 100644 --- a/rel/config/examples/listeners.ws.conf.example +++ b/rel/config/examples/listeners.ws.conf.example @@ -1,16 +1,6 @@ -##-------------------------------------------------------------------- -## WebSocket Listener -## -## Add a WebSocket Listener -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## MQTT over WebSocket (HTTP) Listener -## The WebSocket listener supports all the fields listed in listeners.tcp.conf.example -## only the WebSocket-specific fields are shown here - -## Note: Modifying the 'wsname' to what you need -listeners.ws.wsname { +listeners.ws.my_ws_listener_name { ## Port or Address to listen on, 0 means disable bind = "0.0.0.0:8083" # or just a port number, e.g. 8083 enabled = true diff --git a/rel/config/examples/listeners.wss.conf.example b/rel/config/examples/listeners.wss.conf.example index 799d082d7..cbc632e30 100644 --- a/rel/config/examples/listeners.wss.conf.example +++ b/rel/config/examples/listeners.wss.conf.example @@ -1,16 +1,6 @@ -##-------------------------------------------------------------------- -## WSS Listener -## -## Add a WSS Listener -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## MQTT over Secured Websocket (HTTPS) Listener -## The WSS listener supports all the fields listed in listeners.ws.conf.example -## only the WSS-specific fields are shown here - -## Note: Modifying the 'wssname' to what you need -listeners.wss.wssname { +listeners.wss.my_wss_listener_name = { ## Port or Address to listen on, 0 means disable bind = 8084 ## or with an IP, e.g. "127.0.0.1:8084" enabled = true @@ -76,11 +66,11 @@ listeners.wss.wssname { depth = 10 ## Which versions are to be supported - versions = [tlsv1.3, tlsv1.2] + versions = ["tlsv1.3", "tlsv1.2"] ## TLS cipher suite names ## Note: By default, all available suites are supported, you do not need to set this - ciphers = ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256"] + ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,ECDHE-RSA-AES256-GCM-SHA384" ## Allows a client and a server to renegotiate the parameters of the SSL connection on the fly secure_renegotiate = true diff --git a/rel/config/examples/log.console.conf.example b/rel/config/examples/log.console.conf.example index 1dd30e120..9bb01b0d9 100644 --- a/rel/config/examples/log.console.conf.example +++ b/rel/config/examples/log.console.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- -## Log -## -## Configure the log output location, log level, log file storage path, and parameters -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Log to console log.console { ## set true to enable this diff --git a/rel/config/examples/log.file.conf.example b/rel/config/examples/log.file.conf.example index deb9b16b9..e6f408f61 100644 --- a/rel/config/examples/log.file.conf.example +++ b/rel/config/examples/log.file.conf.example @@ -1,21 +1,14 @@ -##-------------------------------------------------------------------- -## Log -## -## Configure the log output location, log level, log file storage path, and parameters -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## Log to file -## The default-enabled log handler can use all the above fields listed fields log.file { - ## use false to disable this + ## Enable file log handler enable = true ## Log level ## Type: debug | info | notice | warning | error | critical | alert | emergency level = warning - ## Log formatter, text for free text, and json for structured logging + ## Log formatter, text for free text, and json for more structured logging ## Type: text | json formatter = text diff --git a/rel/config/examples/mqtt.conf.example b/rel/config/examples/mqtt.conf.example index c5f81f753..64c73524c 100644 --- a/rel/config/examples/mqtt.conf.example +++ b/rel/config/examples/mqtt.conf.example @@ -1,10 +1,5 @@ -##-------------------------------------------------------------------- -## MQTT -## -## MQTT configuration -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## MQTT protocol related settings +## Settings in this section is applied globally to all MQTT connections/sessions in this node mqtt { ## After the TCP connection is established, @@ -121,4 +116,4 @@ mqtt { ## - hash_clientid :: select the subscribers by hashing the `clientIds` ## - hash_topic :: select the subscribers by hashing the source topic""" shared_subscription_strategy = round_robin - } +} diff --git a/rel/config/examples/node.conf.example b/rel/config/examples/node.conf.example index 65c08b337..b3db9288e 100644 --- a/rel/config/examples/node.conf.example +++ b/rel/config/examples/node.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- -## Node -## -## configuring for current EMQX node -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## EMQX Node ## Note: all fields in this section are immutable after EMQX started, and most of the time you only need to modify the value for the name and cookie. node { @@ -14,29 +8,31 @@ node { ## Note: Make sure the IP resolve from the domain is deterministic and unique and never change name = "emqx@127.0.0.1" - ## Secret cookie is a random string that should be the same on all nodes in the given EMQX cluster, but unique per EMQX cluster - cookie = "Yzc0NGExM2RjYzYxYzM0YzQ5MWQ0NmI1NWM0MWRhMzY4NzgxYmFkMmI2MWJjZWQ5NTQzYTMxNjE1ODVmYmJmMyAgLQo=" + ## Secret cookie is a random string that should be the same on all nodes in the cluster, but unique per EMQX cluster + cookie = "Yzc0NGExM2Rj" ## Select a node role - ## Type: - ## - core :: nodes provide durability of the data, and take care of writes - ## - replicant :: nodes are ephemeral worker nodes + ## Possible values: + ## - core: This is a core node which provides durability of the client states, and takes care of writes + ## - replicant: This is a stateless worker node role = core ## Maximum number of simultaneously existing processes for this Erlang system - ## Type: Range from 1024 to 134217727 process_limit = 2097152 - ## Maximum number of simultaneously existing ports for this Erlang system - ## Type: Range from 1024 to 134217727 + ## Maximum number of simultaneously open files and sockets for this Erlang system max_ports = 1048576 - ## Erlang's distribution buffer busy limit in kilobytes - ## Type: Range from 1 to 2097152 + ## Erlang's distribution buffer busy limit in kilobytes. + ## Range from 1 to 2097152 dist_buffer_size = 8192 - ## Path to the persistent data directory - data_dir = "var/emqx/data" + ## NOTE: keep this the same for all nodes in the cluster. + ## Path to the persistent data directory. + ## This config is pre-filled when the EMQX distribution package is built. + ## You are advised to use the default value. + #data_dir = "data" # when running a zip package or in docker container + #data_dir = "/var/lib/emqx" # when installed from deb/rpm packages ## Type: Periodic garbage collection interval global_gc_interval = 15m diff --git a/rel/config/examples/plugin.conf.example b/rel/config/examples/plugin.conf.example index d704f9981..fe6e25966 100644 --- a/rel/config/examples/plugin.conf.example +++ b/rel/config/examples/plugin.conf.example @@ -1,11 +1,4 @@ -##-------------------------------------------------------------------- -## Plugin -## -## Manage EMQX plugins -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working - +## Plugin management plugin { ## Plugins declaration ## Note: The plugins are started in the defined order diff --git a/rel/config/examples/prometheus-pushgateway.conf.example b/rel/config/examples/prometheus-pushgateway.conf.example new file mode 100644 index 000000000..70b74794a --- /dev/null +++ b/rel/config/examples/prometheus-pushgateway.conf.example @@ -0,0 +1,28 @@ +## Prometheus push-gateway integration + +## EMQX's Prometheus scraping endpoint is enabled by default without authentication. +## e.g. curl -f "127.0.0.1:18083/api/v5/prometheus/stats" +## If you want to use push-gateway + +prometheus { + ## Set to true to make EMQX send metrics to push-gateway + enable = false + + ## URL of push-gateway server + push_gateway_server = "http://127.0.0.1:9091" + + ## Data push interval + interval = 15s + + ## A HTTP Headers when pushing to Push-gateway + headers = { + Authorization = "some-authz-tokens", + Connection = "keep-alive" + } + + ## Job Name that is pushed to the Push-gateway + ## Available variable: + ## - ${name}: Name of EMQX node + ## - ${host}: Host name of EMQX node + job_name = "${name}/instance/${name}~${host}" +} diff --git a/rel/config/examples/prometheus.conf.example b/rel/config/examples/prometheus.conf.example index 645e10364..face3fc1f 100644 --- a/rel/config/examples/prometheus.conf.example +++ b/rel/config/examples/prometheus.conf.example @@ -1,28 +1,10 @@ -##-------------------------------------------------------------------- -## Prometheus -## -## Settings for reporting metrics to Prometheus -##-------------------------------------------------------------------- +## Prometheus push-gateway integration + +## EMQX's Prometheus scraping endpoint is enabled by default without authentication. +## And there is no way to turn it off. +## You can inspect it with a curl command: curl -f "127.0.0.1:18083/api/v5/prometheus/stats" prometheus { - ## URL of Prometheus server - push_gateway_server = "http://127.0.0.1:9091" - - ## Data reporting interval - interval = 15s - - ## A HTTP Headers when pushing to Push Gateway. - headers = { - Authorization = "some-authz-tokens", - Connection = "keep-alive" - } - - ## Job Name that is pushed to the Push Gateway. - ## Available variable: - ## - ${name}: Name of EMQX node - ## - ${host}: Host name of EMQX node - job_name = "${name}/instance/${name}~${host}" - - ## set to false to enable this - enable = true + # turn off this expensive collector + vm_dist_collector = disabled } diff --git a/rel/config/examples/psk_authentication.conf.example b/rel/config/examples/psk_authentication.conf.example index 272eb41b5..6c3482638 100644 --- a/rel/config/examples/psk_authentication.conf.example +++ b/rel/config/examples/psk_authentication.conf.example @@ -1,13 +1,7 @@ -##-------------------------------------------------------------------- ## Pre-Shared Keys authentication -## -## Config to enable TLS-PSK authentication -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working psk_authentication { - ## set to false to disable + ## Set to false to disable enable = true ## If init_file is specified, EMQX will import PSKs from the file into the built-in database at startup for use by the runtime diff --git a/rel/config/examples/retainer.conf.example b/rel/config/examples/retainer.conf.example index 1cef31c30..51101c9c7 100644 --- a/rel/config/examples/retainer.conf.example +++ b/rel/config/examples/retainer.conf.example @@ -3,20 +3,12 @@ ## ## Configuration related to handling PUBLISH packets with a retain flag set to 1 ##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working retainer { - ## set to false to disable this + ## set to false to disable retainer enable = true - ## Message retention time. 0 means message will never be expired - msg_expiry_interval = 0s - - ## Periodic interval for cleaning up expired messages. Never clear if the value is 0 - msg_clear_interval = 0s - - ## Maximum retained message size + ## Maximum message size allowed max_payload_size = 1MB ## When the retained flag of the PUBLISH message is set and Payload is empty, whether to continue to publish the message @@ -27,10 +19,10 @@ retainer { ## Retained messages store backend backend { - ## Backend type + ## Built-in database (Mnesia) type = built_in_database - ## Specifies whether the messages are stored in RAM or persisted on disc + ## Specifies whether the messages are stored in RAM or persisted on disk ## Type: enum: ram | disc storage_type = ram diff --git a/rel/config/examples/sys_topics.conf.example b/rel/config/examples/sys_topics.conf.example index b249efd35..e4d0611b1 100644 --- a/rel/config/examples/sys_topics.conf.example +++ b/rel/config/examples/sys_topics.conf.example @@ -1,10 +1,5 @@ -##-------------------------------------------------------------------- ## System Topic -## ## Publishing client lifecycle events to "$SYS" topics -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working sys_topics { ## Time interval of publishing `$SYS` messages diff --git a/rel/config/examples/sysmon.os.conf.example b/rel/config/examples/sysmon.os.conf.example index 95e7aa1ec..08e025da0 100644 --- a/rel/config/examples/sysmon.os.conf.example +++ b/rel/config/examples/sysmon.os.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- -## System Monitoring For System -## -## System monitoring and introspection -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working +## System Monitoring of the host machine sysmon.os { ## Time interval for the periodic CPU check @@ -17,9 +11,9 @@ sysmon.os { cpu_low_watermark = 60% ## Time interval for the periodic memory check - ## Type: - ## - disabled :: Never check - ## - Time Duration :: The time period + ## Possible values: + ## disabled: Never check + ## Duration: Check every Duration mem_check_interval = 60s ## For how much system memory can be allocated before the corresponding alarm is raised diff --git a/rel/config/examples/sysmon.vm.conf.example b/rel/config/examples/sysmon.vm.conf.example index d68141cea..6ca6ee411 100644 --- a/rel/config/examples/sysmon.vm.conf.example +++ b/rel/config/examples/sysmon.vm.conf.example @@ -1,10 +1,4 @@ -##-------------------------------------------------------------------- ## System Monitoring For Erlang VM -## -## System monitoring and introspection -##-------------------------------------------------------------------- -## Note: This is an example of how to configure this feature -## you should copy and paste the below data into the emqx.conf for working sysmon.vm { ## Time interval for the periodic process limit check @@ -25,7 +19,7 @@ sysmon.vm { ## Generating an alarm is generated when the Erlang VM detect a task scheduled for too long ## Type: ## - disabled :: Never alarm - ## - Time During :: The maximum schedule time for generating an alarm + ## - Duration:: The maximum schedule time for generating an alarm long_schedule = 240ms ## Generating an alarm when an Erlang process consumed a large amount of memory for its heap space diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index 416b3ca76..1fb37cd0a 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -135,7 +135,7 @@ log_overload_kill_restart_after.label: """Handler Restart Timer""" log_file_handler_max_size.desc: -"""This parameter controls log file rotation. The value `infinity` means the log file will grow indefinitely, otherwise the log file will be rotated once it reaches `max_size` in bytes.""" +"""This parameter controls log file rotation. The value `infinity` means the log file will grow indefinitely, otherwise the log file will be rotated once it reaches `rotation_size` in bytes.""" log_file_handler_max_size.label: """Rotation Size""" @@ -514,15 +514,13 @@ cluster_autoclean.label: process_limit.desc: """Maximum number of simultaneously existing processes for this Erlang system. -The actual maximum chosen may be much larger than the Number passed. For more information, see: https://www.erlang.org/doc/man/erl.html""" process_limit.label: """Erlang Process Limit""" max_ports.desc: -"""Maximum number of simultaneously existing ports for this Erlang system. -The actual maximum chosen may be much larger than the Number passed. +"""Maximum number of simultaneously open files and sockets for this Erlang system. For more information, see: https://www.erlang.org/doc/man/erl.html""" max_ports.label: diff --git a/rel/i18n/emqx_prometheus_schema.hocon b/rel/i18n/emqx_prometheus_schema.hocon index d68e1d418..4dd9a2e1e 100644 --- a/rel/i18n/emqx_prometheus_schema.hocon +++ b/rel/i18n/emqx_prometheus_schema.hocon @@ -21,13 +21,15 @@ mnesia_collector.desc: """Enable or disable Mnesia collector, collects Mnesia metrics mainly using mnesia:system_info/1 .""" prometheus.desc: -"""Settings for reporting metrics to Prometheus""" +"""EMQX's Prometheus scraping endpoint is enabled by default without authentication. +You can inspect it with a `curl` command like this: `curl -f "127.0.0.1:18083/api/v5/prometheus/stats"`
+The 'enable' flag is used to turn on and off for the push-gateway integration.""" prometheus.label: """Prometheus""" push_gateway_server.desc: -"""URL of Prometheus server""" +"""URL of Prometheus server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX.""" vm_dist_collector.desc: """Enable or disable VM distribution collector, collects information about the sockets and processes involved in the Erlang distribution mechanism.""" diff --git a/rel/i18n/emqx_retainer_schema.hocon b/rel/i18n/emqx_retainer_schema.hocon index 49b55d259..fc8d4a028 100644 --- a/rel/i18n/emqx_retainer_schema.hocon +++ b/rel/i18n/emqx_retainer_schema.hocon @@ -34,11 +34,11 @@ mnesia_config_type.desc: """Backend type.""" msg_clear_interval.desc: -"""Periodic interval for cleaning up expired messages. -Never clear if the value is 0.""" +"""Interval for EMQX to scan expired messages and delete them. Never scan if the value is 0.""" msg_expiry_interval.desc: -"""Message retention time. 0 means message will never be expired.""" +"""Message retention time. This config is only applicable for messages without the Message Expiry Interval message property. +0 means message will never expir.""" stop_publish_clear_msg.desc: """When the retained flag of the `PUBLISH` message is set and Payload is empty, From 9aa83cb0a236c7a096c057529e22afcd036c820b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 22 Jun 2023 17:28:48 +0200 Subject: [PATCH 03/13] refactor(config-examples): do not include ee examples in ce --- rebar.config.erl | 14 +++++++++++--- .../file_transfer-with-local-exporter.conf.example | 0 .../file_transfer-with-s3-exporter.conf.example | 0 3 files changed, 11 insertions(+), 3 deletions(-) rename rel/config/{examples => ee-examples}/file_transfer-with-local-exporter.conf.example (100%) rename rel/config/{examples => ee-examples}/file_transfer-with-s3-exporter.conf.example (100%) diff --git a/rebar.config.erl b/rebar.config.erl index f8f96e82c..0122751fb 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -520,12 +520,12 @@ relx_overlay(ReleaseType, Edition) -> {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"} ] ++ etc_overlay(ReleaseType, Edition). -etc_overlay(ReleaseType, _Edition) -> +etc_overlay(ReleaseType, Edition) -> Templates = emqx_etc_overlay(ReleaseType), [ {mkdir, "etc/"}, - {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"}, - {copy, "rel/config/examples", "etc/"} + {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"} + | copy_examples(Edition) ] ++ lists:map( fun @@ -535,6 +535,14 @@ etc_overlay(ReleaseType, _Edition) -> Templates ). +copy_examples(ce) -> + [{copy, "rel/config/examples", "etc/"}]; +copy_examples(ee) -> + [ + {copy, "rel/config/examples", "etc/"}, + {copy, "rel/config/ee-examples/*", "etc/examples/"} + ]. + emqx_etc_overlay(ReleaseType) -> emqx_etc_overlay_per_rel(ReleaseType) ++ emqx_etc_overlay(). diff --git a/rel/config/examples/file_transfer-with-local-exporter.conf.example b/rel/config/ee-examples/file_transfer-with-local-exporter.conf.example similarity index 100% rename from rel/config/examples/file_transfer-with-local-exporter.conf.example rename to rel/config/ee-examples/file_transfer-with-local-exporter.conf.example diff --git a/rel/config/examples/file_transfer-with-s3-exporter.conf.example b/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example similarity index 100% rename from rel/config/examples/file_transfer-with-s3-exporter.conf.example rename to rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example From 03d05825db44ceffa9a356cb0b5681c47f0eccb2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 22 Jun 2023 17:12:41 +0200 Subject: [PATCH 04/13] docs: hide core_nodes config doc core_nodes config is no longer required. --- apps/emqx_conf/src/emqx_conf_schema.erl | 2 ++ rel/i18n/emqx_conf_schema.hocon | 11 ----------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index ce7bbb8b2..89fda6de8 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -154,6 +154,8 @@ fields("cluster") -> sc( node_array(), #{ + %% This config is nerver needed (since 5.0.0) + importance => ?IMPORTANCE_HIDDEN, mapping => "mria.core_nodes", default => [], 'readOnly' => true, diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index 1fb37cd0a..bc45fa009 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -389,17 +389,6 @@ cluster_mcast_ttl.desc: cluster_mcast_ttl.label: """Cluster Multicast TTL""" -db_core_nodes.desc: -"""List of core nodes that the replicant will connect to.
-Note: this parameter only takes effect when the backend is set -to rlog and the role is set to replicant.
-This value needs to be defined for manual or static cluster discovery mechanisms.
-If an automatic cluster discovery mechanism is being used (such as etcd), -there is no need to set this value.""" - -db_core_nodes.label: -"""Db Core Node""" - log_file_handler_file.desc: """Name the log file.""" From 0428d91aa4b2e262257499f2a504fcd2f960e839 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 22 Jun 2023 20:21:57 +0200 Subject: [PATCH 05/13] fix(config-examples): fix bad configs prior to this commit, the provided configs are not type-checked --- rel/config/{examples => ee-examples}/license.conf.example | 0 rel/config/examples/listeners.ssl.conf.example | 2 +- rel/config/examples/node.conf.example | 2 +- .../examples/{plugin.conf.example => plugins.conf.example} | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename rel/config/{examples => ee-examples}/license.conf.example (100%) rename rel/config/examples/{plugin.conf.example => plugins.conf.example} (98%) diff --git a/rel/config/examples/license.conf.example b/rel/config/ee-examples/license.conf.example similarity index 100% rename from rel/config/examples/license.conf.example rename to rel/config/ee-examples/license.conf.example diff --git a/rel/config/examples/listeners.ssl.conf.example b/rel/config/examples/listeners.ssl.conf.example index a33ce7efe..4a74f1a27 100644 --- a/rel/config/examples/listeners.ssl.conf.example +++ b/rel/config/examples/listeners.ssl.conf.example @@ -72,7 +72,7 @@ listeners.ssl.my_ssl_listener_name { ocsp { enable_ocsp_stapling = false responder_url = "http://ocsp.example.com" - issuer_pem = true + issuer_pem = "${EMQX_ETC_DIR}/certs/ocsp-issuer-cert.pem" refresh_http_timeout = 15s refresh_interval = 5m } diff --git a/rel/config/examples/node.conf.example b/rel/config/examples/node.conf.example index b3db9288e..596e9884d 100644 --- a/rel/config/examples/node.conf.example +++ b/rel/config/examples/node.conf.example @@ -31,7 +31,7 @@ node { ## Path to the persistent data directory. ## This config is pre-filled when the EMQX distribution package is built. ## You are advised to use the default value. - #data_dir = "data" # when running a zip package or in docker container + data_dir = "data" # when running a zip package or in docker container #data_dir = "/var/lib/emqx" # when installed from deb/rpm packages ## Type: Periodic garbage collection interval diff --git a/rel/config/examples/plugin.conf.example b/rel/config/examples/plugins.conf.example similarity index 98% rename from rel/config/examples/plugin.conf.example rename to rel/config/examples/plugins.conf.example index fe6e25966..6fcc09cbb 100644 --- a/rel/config/examples/plugin.conf.example +++ b/rel/config/examples/plugins.conf.example @@ -1,5 +1,5 @@ ## Plugin management -plugin { +plugins { ## Plugins declaration ## Note: The plugins are started in the defined order states = [ From a069b351fd79e5eb891d2729e47b78278130973a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 22 Jun 2023 20:55:16 +0200 Subject: [PATCH 06/13] test: add test script to verify config example files --- apps/emqx/src/emqx_hocon.erl | 29 +++++++- apps/emqx/src/emqx_schema.erl | 14 +++- .../emqx_conf/test/emqx_conf_schema_tests.erl | 71 ++++++++++++++++++- scripts/test/check-example-configs.sh | 54 ++++++++++++++ 4 files changed, 164 insertions(+), 4 deletions(-) create mode 100755 scripts/test/check-example-configs.sh diff --git a/apps/emqx/src/emqx_hocon.erl b/apps/emqx/src/emqx_hocon.erl index e028ebb14..62573f201 100644 --- a/apps/emqx/src/emqx_hocon.erl +++ b/apps/emqx/src/emqx_hocon.erl @@ -24,7 +24,8 @@ compact_errors/2, format_error/1, format_error/2, - make_schema/1 + make_schema/1, + load_and_check/2 ]). %% @doc Format hocon config field path to dot-separated string in iolist format. @@ -46,7 +47,8 @@ check(SchemaModule, Conf) -> check(SchemaModule, Conf, Opts) when is_map(Conf) -> try - {ok, hocon_tconf:check_plain(SchemaModule, Conf, Opts)} + RootNames = maps:keys(Conf), + {ok, hocon_tconf:check_plain(SchemaModule, Conf, Opts, RootNames)} catch throw:Errors:Stacktrace -> compact_errors(Errors, Stacktrace) @@ -135,3 +137,26 @@ compact_errors(SchemaModule, Error, Stacktrace) -> exception => Error, stacktrace => Stacktrace }}. + +%% @doc This is only used in static check scripts in the CI. +-spec load_and_check(module(), filename:filename_all()) -> {ok, term()} | {error, any()}. +load_and_check(SchemaModule, File) -> + try + do_load_and_check(SchemaModule, File) + catch + throw:Reason -> + {error, Reason} + end. + +do_load_and_check(SchemaModule, File) -> + Conf = + case hocon:load(File, #{format => map}) of + {ok, Conf0} -> + Conf0; + {error, {parse_error, Reason}} -> + throw(Reason); + {error, Reason} -> + throw(Reason) + end, + Opts = #{atom_key => false, required => false}, + check(SchemaModule, Conf, Opts). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 9de6ef34a..6c0c511fb 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3271,7 +3271,19 @@ tombstone() -> tombstone_map(Name, Type) -> %% marked_for_deletion must be the last member of the union %% because we need to first union member to populate the default values - map(Name, ?UNION([Type, ?TOMBSTONE_TYPE])). + map( + Name, + hoconsc:union( + fun + (all_union_members) -> + [Type, ?TOMBSTONE_TYPE]; + ({value, V}) when is_map(V) -> + [Type]; + ({value, _}) -> + [?TOMBSTONE_TYPE] + end + ) + ). %% inverse of mark_del_map get_tombstone_map_value_type(Schema) -> diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 76efd8ec4..855b8ff12 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -1,5 +1,17 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %%-------------------------------------------------------------------- -module(emqx_conf_schema_tests). @@ -488,3 +500,60 @@ check(Config) -> atom_key => false, required => false, format => map }), emqx_utils_maps:unsafe_atom_key_map(Conf). + +with_file(Path, Content, F) -> + ok = file:write_file(Path, Content), + try + F() + after + file:delete(Path) + end. + +load_and_check_test_() -> + [ + {"non-existing file", fun() -> + File = "/tmp/nonexistingfilename.hocon", + ?assertEqual( + {error, {enoent, File}}, + emqx_hocon:load_and_check(emqx_conf_schema, File) + ) + end}, + {"bad syntax", fun() -> + %% use abs path to match error return + File = "/tmp/emqx-conf-bad-syntax-test.hocon", + with_file( + File, + "{", + fun() -> + ?assertMatch( + {error, #{file := File}}, + emqx_hocon:load_and_check(emqx_conf_schema, File) + ) + end + ) + end}, + {"type-check failure", fun() -> + File = "emqx-conf-type-check-failure.hocon", + %% typecheck fail because cookie is required field + with_file( + File, + "node {}", + fun() -> + ?assertMatch( + {error, #{ + kind := validation_error, + path := "node.cookie", + reason := required_field + }}, + emqx_hocon:load_and_check(emqx_conf_schema, File) + ) + end + ) + end}, + {"ok load", fun() -> + File = "emqx-conf-test-tmp-file-load-ok.hocon", + with_file(File, "plugins: {}", fun() -> + ?assertMatch({ok, _}, emqx_hocon:load_and_check(emqx_conf_schema, File)) + end) + end} + ]. diff --git a/scripts/test/check-example-configs.sh b/scripts/test/check-example-configs.sh new file mode 100755 index 000000000..bbf00cf8d --- /dev/null +++ b/scripts/test/check-example-configs.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -euo pipefail +PROJ_DIR="$(git rev-parse --show-toplevel)" + +PROFILE="${PROFILE:-emqx}" +DIR_NAME='examples' +SCHEMA_MOD='emqx_conf_schema' +if [ "${PROFILE}" = 'emqx-enterprise' ]; then + DIR_NAME='ee-examples' + SCHEMA_MOD='emqx_enterprise_schema' + PA="" +fi + +IFS=$'\n' read -r -d '' -a FILES < <(find "${PROJ_DIR}/rel/config/${DIR_NAME}" -name "*.example" 2>/dev/null | sort && printf '\0') + +prepare_erl_libs() { + local libs_dir="$1" + local erl_libs="${ERL_LIBS:-}" + local sep=':' + for app in "${libs_dir}"/*; do + if [ -d "${app}/ebin" ]; then + if [ -n "$erl_libs" ]; then + erl_libs="${erl_libs}${sep}${app}" + else + erl_libs="${app}" + fi + fi + done + export ERL_LIBS="$erl_libs" +} + +# This is needed when checking schema +export EMQX_ETC_DIR="${PROJ_DIR}/apps/emqx/etc" + +prepare_erl_libs "_build/$PROFILE/lib" + +check_file() { + local file="$1" + erl -noshell -eval \ + "File=\"$file\", + case emqx_hocon:load_and_check($SCHEMA_MOD, File) of + {ok, _} -> + io:format(\"check_example_config_ok: ~s~n\", [File]), + halt(0); + {error, Reason} -> + io:format(\"failed_to_check_example_config: ~s~n~p~n\", [File, Reason]), + halt(1) + end." +} + +for file in ${FILES[@]}; do + check_file "$file" +done From 41de679557b282e55e7ea671ab9a612b9412fa7b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 22 Jun 2023 21:01:41 +0200 Subject: [PATCH 07/13] ci: run example config check in ci --- .github/workflows/run_test_cases.yaml | 24 +++++++++++++++++++ .../src/emqx_prometheus.app.src | 2 +- scripts/test/check-example-configs.sh | 3 +-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index a8f65b926..85b38627f 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -88,6 +88,30 @@ jobs: name: source-${{ matrix.profile }}-${{ matrix.otp }} path: source.zip + check_examples: + needs: + - build-matrix + - prepare + runs-on: ${{ needs.build-matrix.outputs.runs-on }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04" + steps: + - uses: AutoModality/action-clean@v1 + - uses: actions/download-artifact@v3 + with: + name: source-${{ matrix.profile }}-${{ matrix.otp }} + path: . + - name: unzip source code + run: unzip -o -q source.zip + - name: check example config files + env: + PROFILE: ${{ matrix.profile }} + working-directory: source + run: ./scripts/test/check-example-configs.sh + static_checks: needs: - build-matrix diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 147dcc28f..7252e4436 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.12"}, + {vsn, "5.0.13"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/scripts/test/check-example-configs.sh b/scripts/test/check-example-configs.sh index bbf00cf8d..f71fb15eb 100755 --- a/scripts/test/check-example-configs.sh +++ b/scripts/test/check-example-configs.sh @@ -9,7 +9,6 @@ SCHEMA_MOD='emqx_conf_schema' if [ "${PROFILE}" = 'emqx-enterprise' ]; then DIR_NAME='ee-examples' SCHEMA_MOD='emqx_enterprise_schema' - PA="" fi IFS=$'\n' read -r -d '' -a FILES < <(find "${PROJ_DIR}/rel/config/${DIR_NAME}" -name "*.example" 2>/dev/null | sort && printf '\0') @@ -49,6 +48,6 @@ check_file() { end." } -for file in ${FILES[@]}; do +for file in "${FILES[@]}"; do check_file "$file" done From 43705a54be226156d44493491c802c6fd576560b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 22 Jun 2023 22:21:35 +0200 Subject: [PATCH 08/13] docs: cosmetic changes in example config files --- rel/config/examples/conn_congestion.conf.example | 2 +- rel/config/examples/prometheus.conf.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/config/examples/conn_congestion.conf.example b/rel/config/examples/conn_congestion.conf.example index 66a47a65f..785ee5ab5 100644 --- a/rel/config/examples/conn_congestion.conf.example +++ b/rel/config/examples/conn_congestion.conf.example @@ -1,6 +1,6 @@ ## Connection Congestion ## -## Generating alarm when MQTT connection congested +## Emit alarms when MQTT connection congested conn_congestion { ## Enable or disable connection congestion alarm diff --git a/rel/config/examples/prometheus.conf.example b/rel/config/examples/prometheus.conf.example index face3fc1f..3b186e93f 100644 --- a/rel/config/examples/prometheus.conf.example +++ b/rel/config/examples/prometheus.conf.example @@ -1,4 +1,4 @@ -## Prometheus push-gateway integration +## Prometheus ## EMQX's Prometheus scraping endpoint is enabled by default without authentication. ## And there is no way to turn it off. From 5425cae6504551c71b9051b9a3f83a47145ed2db Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 25 Jun 2023 22:46:18 +0200 Subject: [PATCH 09/13] chore: delete ipv6_probe from s3 example --- .../ee-examples/file_transfer-with-s3-exporter.conf.example | 3 --- 1 file changed, 3 deletions(-) diff --git a/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example b/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example index cba6fada5..497f5d645 100644 --- a/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example +++ b/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example @@ -57,9 +57,6 @@ file_transfer { ## Timeout for connection attempts connect_timeout = 15s - - ## Attempt to talk through IPv6 first - ipv6_probe = true } } } From 07c80f4031b77551b9eea9b8a9f6c0517f5fddd3 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 25 Jun 2023 22:46:40 +0200 Subject: [PATCH 10/13] build: copy ee-examples for Enterprise edition elixir build --- mix.exs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mix.exs b/mix.exs index 5ded486c9..ac547ecae 100644 --- a/mix.exs +++ b/mix.exs @@ -543,6 +543,19 @@ defmodule EMQXUmbrella.MixProject do force: overwrite? ) + # copy /rel/config/ee-examples if profile is enterprise + case profile do + "emqx-enterprise" -> + File.cp_r!( + "rel/config/ee-examples", + Path.join(etc, "examples"), + force: overwrite? + ) + + _ -> + :ok + end + # this is required by the produced escript / nodetool Mix.Generator.copy_file( Path.join(release.version_path, "start_clean.boot"), From 92ed7d76391114a85594418057922d77edf645e9 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 25 Jun 2023 23:05:04 +0200 Subject: [PATCH 11/13] docs: move some prometheus docs from schema desc to code comments --- apps/emqx_prometheus/src/emqx_prometheus_schema.erl | 6 ++++++ rel/i18n/emqx_prometheus_schema.hocon | 10 +++++----- rel/i18n/emqx_retainer_schema.hocon | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 82d951ab3..27e5790e2 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -94,6 +94,7 @@ fields("prometheus") -> desc => ?DESC(vm_dist_collector) } )}, + %% Mnesia metrics mainly using mnesia:system_info/1 {mnesia_collector, ?HOCON( hoconsc:enum([enabled, disabled]), @@ -104,6 +105,7 @@ fields("prometheus") -> desc => ?DESC(mnesia_collector) } )}, + %% Collects Erlang VM metrics using erlang:statistics/1. {vm_statistics_collector, ?HOCON( hoconsc:enum([enabled, disabled]), @@ -114,6 +116,7 @@ fields("prometheus") -> desc => ?DESC(vm_statistics_collector) } )}, + %% Collects Erlang VM metrics using erlang:system_info/1. {vm_system_info_collector, ?HOCON( hoconsc:enum([enabled, disabled]), @@ -124,6 +127,8 @@ fields("prometheus") -> desc => ?DESC(vm_system_info_collector) } )}, + %% Collects information about memory dynamically allocated by the Erlang VM using erlang:memory/0, + %% it also provides basic (D)ETS statistics. {vm_memory_collector, ?HOCON( hoconsc:enum([enabled, disabled]), @@ -134,6 +139,7 @@ fields("prometheus") -> desc => ?DESC(vm_memory_collector) } )}, + %% Collects microstate accounting metrics using erlang:statistics(microstate_accounting). {vm_msacc_collector, ?HOCON( hoconsc:enum([enabled, disabled]), diff --git a/rel/i18n/emqx_prometheus_schema.hocon b/rel/i18n/emqx_prometheus_schema.hocon index 4dd9a2e1e..d665343e9 100644 --- a/rel/i18n/emqx_prometheus_schema.hocon +++ b/rel/i18n/emqx_prometheus_schema.hocon @@ -18,7 +18,7 @@ For example, when the EMQX node name is emqx@127.0.0.1 then the ${name}/instance/${name}~${host}""" mnesia_collector.desc: -"""Enable or disable Mnesia collector, collects Mnesia metrics mainly using mnesia:system_info/1 .""" +"""Enable or disable Mnesia metrics collector""" prometheus.desc: """EMQX's Prometheus scraping endpoint is enabled by default without authentication. @@ -35,15 +35,15 @@ vm_dist_collector.desc: """Enable or disable VM distribution collector, collects information about the sockets and processes involved in the Erlang distribution mechanism.""" vm_memory_collector.desc: -"""Enable or disable VM memory collector, collects information about memory dynamically allocated by the Erlang emulator using erlang:memory/0 , also provides basic (D)ETS statistics .""" +"""Enable or disable VM memory metrics collector.""" vm_msacc_collector.desc: -"""Enable or disable VM msacc collector, collects microstate accounting metrics using erlang:statistics(microstate_accounting) .""" +"""Enable or disable VM microstate accounting metrics collector.""" vm_statistics_collector.desc: -"""Enable or disable VM statistics collector, collects Erlang VM metrics using erlang:statistics/1 .""" +"""Enable or disable VM statistics collector.""" vm_system_info_collector.desc: -"""Enable or disable VM system info collector, collects Erlang VM metrics using erlang:system_info/1 .""" +"""Enable or disable VM system info collector.""" } diff --git a/rel/i18n/emqx_retainer_schema.hocon b/rel/i18n/emqx_retainer_schema.hocon index fc8d4a028..e6679dbf7 100644 --- a/rel/i18n/emqx_retainer_schema.hocon +++ b/rel/i18n/emqx_retainer_schema.hocon @@ -38,7 +38,7 @@ msg_clear_interval.desc: msg_expiry_interval.desc: """Message retention time. This config is only applicable for messages without the Message Expiry Interval message property. -0 means message will never expir.""" +0 means message will never expire.""" stop_publish_clear_msg.desc: """When the retained flag of the `PUBLISH` message is set and Payload is empty, From 9b6ed09513795dae08515dbcf27b3e6be0a038cf Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 26 Jun 2023 23:08:11 +0200 Subject: [PATCH 12/13] refactor: call hocon_tconf:check_plain/4 directly for example conf check the attemp to re-use emqx_hocon:check failed because hocon_tconf:check_plain/3 and check_plain/4 behaves slightly different for unknown root fields. To keep the old logic unchanged, for example config checks, we call check_plain/4 directly --- apps/emqx/src/emqx_hocon.erl | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/apps/emqx/src/emqx_hocon.erl b/apps/emqx/src/emqx_hocon.erl index 62573f201..4423ad29d 100644 --- a/apps/emqx/src/emqx_hocon.erl +++ b/apps/emqx/src/emqx_hocon.erl @@ -47,8 +47,7 @@ check(SchemaModule, Conf) -> check(SchemaModule, Conf, Opts) when is_map(Conf) -> try - RootNames = maps:keys(Conf), - {ok, hocon_tconf:check_plain(SchemaModule, Conf, Opts, RootNames)} + {ok, hocon_tconf:check_plain(SchemaModule, Conf, Opts)} catch throw:Errors:Stacktrace -> compact_errors(Errors, Stacktrace) @@ -144,19 +143,21 @@ load_and_check(SchemaModule, File) -> try do_load_and_check(SchemaModule, File) catch - throw:Reason -> - {error, Reason} + throw:Reason:Stacktrace -> + compact_errors(Reason, Stacktrace) end. do_load_and_check(SchemaModule, File) -> - Conf = - case hocon:load(File, #{format => map}) of - {ok, Conf0} -> - Conf0; - {error, {parse_error, Reason}} -> - throw(Reason); - {error, Reason} -> - throw(Reason) - end, - Opts = #{atom_key => false, required => false}, - check(SchemaModule, Conf, Opts). + case hocon:load(File, #{format => map}) of + {ok, Conf} -> + Opts = #{atom_key => false, required => false}, + %% here we check only the provided root keys + %% because the examples are all provided only with one (or maybe two) roots + %% and some roots have required fields. + RootKeys = maps:keys(Conf), + {ok, hocon_tconf:check_plain(SchemaModule, Conf, Opts, RootKeys)}; + {error, {parse_error, Reason}} -> + {error, Reason}; + {error, Reason} -> + {error, Reason} + end. From 90c35c3ff8aab5d756d5061b6945ae3ac42ff029 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 27 Jun 2023 14:22:21 +0200 Subject: [PATCH 13/13] chore(prometheus): change default vaue enum the first symbol --- apps/emqx_prometheus/src/emqx_prometheus_schema.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 27e5790e2..3300e8b28 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -86,7 +86,7 @@ fields("prometheus") -> )}, {vm_dist_collector, ?HOCON( - hoconsc:enum([enabled, disabled]), + hoconsc:enum([disabled, enabled]), #{ default => disabled, required => true,