From d1e8fefa1ffefb7b7e69e58ef4091d74553b4506 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 23 Mar 2022 09:51:51 +0100 Subject: [PATCH 1/4] docs(schema): Add descriptions of events --- apps/emqx/src/emqx_schema.erl | 37 ++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index d38feacd3..e502bd548 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1288,22 +1288,34 @@ fields("event_names") -> {"client_connected", sc( boolean(), - #{default => true} + #{ + default => true, + desc => "Connection complete" + } )}, {"client_disconnected", sc( boolean(), - #{default => true} + #{ + default => true, + desc => "Disconnect" + } )}, {"client_subscribed", sc( boolean(), - #{default => false} + #{ + default => false, + desc => "Subscribe" + } )}, {"client_unsubscribed", sc( boolean(), - #{default => false} + #{ + default => false, + desc => "Unsubscribe" + } )} ]; fields("sysmon") -> @@ -1589,10 +1601,8 @@ fields("alarm") -> default => "24h", example => "24h", desc => - "" "Retention time of deactivated alarms. Alarms are not deleted immediately\n" "when deactivated, but after the retention time.\n" - "" } )} ]; @@ -1757,7 +1767,7 @@ common_ssl_opts_schema(Defaults) -> "issue chain. That is, the host's certificate should be placed in the beginning\n" "of the file, followed by the immediate issuer certificate and so on.\n" "Although the root CA certificate is optional, it should be placed at the end of\n" - "the file if it is to be added.\n" + "the file if it is to be added." } )}, {"keyfile", @@ -1767,7 +1777,7 @@ common_ssl_opts_schema(Defaults) -> default => D("keyfile"), required => false, desc => - "PEM format private key file.
\n" + "PEM format private key file." } )}, {"verify", @@ -1805,7 +1815,7 @@ common_ssl_opts_schema(Defaults) -> "All TLS/DTLS versions to be supported.
\n" "NOTE: PSK ciphers are suppressed by 'tlsv1.3' version config
\n" "In case PSK cipher suites are intended, make sure to configured\n" - "['tlsv1.2', 'tlsv1.1'] here.\n", + "['tlsv1.2', 'tlsv1.1'] here.", validator => fun validate_tls_versions/1 } )}, @@ -1827,7 +1837,7 @@ common_ssl_opts_schema(Defaults) -> "SSL parameter renegotiation is a feature that allows a client and a server\n" "to renegotiate the parameters of the SSL connection on the fly.\n" "RFC 5746 defines a more secure way of doing this. By enabling secure renegotiation,\n" - "you drop support for the insecure renegotiation, prone to MitM attacks.\n" + "you drop support for the insecure renegotiation, prone to MitM attacks." } )} ]. @@ -1864,7 +1874,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> "If set to true, the server fails if the client does not have a\n" "certificate to send, that is, sends an empty certificate.\n" "If set to false, it fails only if the client sends an invalid\n" - "certificate (an empty certificate is considered valid).\n" + "certificate (an empty certificate is considered valid)." } )}, {"honor_cipher_order", @@ -1887,7 +1897,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> "this option to false.\n" "The default value is true. Note that disabling renegotiation can result in\n" "long-lived connections becoming unusable due to limits on\n" - "the number of messages the underlying cipher suite can encipher.\n" + "the number of messages the underlying cipher suite can encipher." } )} | [ @@ -1980,8 +1990,7 @@ ciphers_schema(Default) -> "PSK cipher suites: \"RSA-PSK-AES256-GCM-SHA384,RSA-PSK-AES256-CBC-SHA384,\n" "RSA-PSK-AES128-GCM-SHA256,RSA-PSK-AES128-CBC-SHA256,\n" "RSA-PSK-AES256-CBC-SHA,RSA-PSK-AES128-CBC-SHA,\n" - "RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA\"
\n" - "" ++ + "RSA-PSK-DES-CBC3-SHA,RSA-PSK-RC4-SHA\"
\n" ++ case Default of quic -> "NOTE: QUIC listener supports only 'tlsv1.3' ciphers
"; _ -> "" From ee805f12a610f5095a78f5ac2a80a2742b5a970a Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 23 Mar 2022 10:51:54 +0100 Subject: [PATCH 2/4] docs(schema): Add documentation for prometheus and logger --- apps/emqx_conf/src/emqx_conf_schema.erl | 35 ++++++++++++++++--- .../src/emqx_prometheus_schema.erl | 18 ++++++++-- apps/emqx_statsd/src/emqx_statsd_schema.erl | 26 ++++++++++---- 3 files changed, 64 insertions(+), 15 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 7fb9d730a..826b73461 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -651,10 +651,12 @@ fields("log_rotation") -> [ {"enable", sc(boolean(), #{ default => true + , desc => "Enable log rotation feature." })} , {"count", sc(range(1, 2048), #{ default => 10 + , desc => "Maximum number of log files." })} ]; @@ -662,18 +664,23 @@ fields("log_overload_kill") -> [ {"enable", sc(boolean(), #{ default => true + , desc => "Enable log handler overload kill feature." })} , {"mem_size", sc(emqx_schema:bytesize(), #{ default => "30MB" + , desc => "Maximum memory size that the handler process is allowed to use." })} , {"qlen", sc(integer(), #{ default => 20000 + , desc => "Maximum allowed queue length." })} , {"restart_after", sc(hoconsc:union([emqx_schema:duration(), infinity]), #{ default => "5s" + , desc => "If the handler is terminated, it restarts automatically after a " + "delay specified in milliseconds. The value `infinity` prevents restarts." })} ]; @@ -681,14 +688,20 @@ fields("log_burst_limit") -> [ {"enable", sc(boolean(), #{ default => true + , desc => "Enable log burst control feature." })} , {"max_count", sc(integer(), #{ default => 10000 + , desc => "Maximum number of log events to handle within a `window_time` interval. " + "After the limit is reached, successive events are dropped " + "until the end of the `window_time`." })} , {"window_time", sc(emqx_schema:duration(), - #{default => "1s"})} + #{ default => "1s" + , desc => "See `max_count`." + })} ]; fields("authorization") -> @@ -801,6 +814,7 @@ log_handler_common_confs() -> [ {"enable", sc(boolean(), #{ default => false + , desc => "Enable this log handler." })} , {"level", sc(log_level(), @@ -811,6 +825,7 @@ log_handler_common_confs() -> , {"time_offset", sc(string(), #{ default => "system" + , desc => "The time offset to be used when formatting the timestamp." })} , {"chars_limit", sc(hoconsc:union([unlimited, range(1, inf)]), @@ -833,24 +848,34 @@ log_handler_common_confs() -> , {"sync_mode_qlen", sc(integer(), #{ default => 100 + , desc => "As long as the number of buffered log events is lower than this value, " + "all log events are handled asynchronously." })} , {"drop_mode_qlen", sc(integer(), #{ default => 3000 + , desc => "When the number of buffered log events is larger than this value, " + "the new log events are dropped.
" + "When drop mode is activated or deactivated, a message is printed in " + "the logs." })} , {"flush_qlen", sc(integer(), #{ default => 8000 + , desc => "If the number of buffered log events grows larger than this threshold, " + "a flush (delete) operation takes place. " + "To flush events, the handler discards the buffered log messages without logging." })} , {"overload_kill", - sc(ref("log_overload_kill"), - #{})} + sc(ref("log_overload_kill"), #{})} , {"burst_limit", - sc(ref("log_burst_limit"), - #{})} + sc(ref("log_burst_limit"), #{})} , {"supervisor_reports", sc(hoconsc:enum([error, progress]), #{ default => error + , desc => "Type of supervisor reports that are logged.\n" + " - `error`: only log errors in the Erlang processes.\n" + " - `progress`: log process startup." })} , {"max_depth", sc(hoconsc:union([unlimited, non_neg_integer()]), diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 17387e06d..5434ef1f1 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -28,9 +28,21 @@ namespace() -> "prometheus". roots() -> ["prometheus"]. fields("prometheus") -> - [ {push_gateway_server, sc(string(), #{default => "http://127.0.0.1:9091", required => true})} - , {interval, sc(emqx_schema:duration_ms(), #{default => "15s", required => true})} - , {enable, sc(boolean(), #{default => false, required => true})} + [ {push_gateway_server, sc(string(), + #{ default => "http://127.0.0.1:9091" + , required => true + , desc => "URL of Prometheus pushgateway." + })} + , {interval, sc(emqx_schema:duration_ms(), + #{ default => "15s" + , required => true + , desc => "Data reporting interval in milliseconds." + })} + , {enable, sc(boolean(), + #{ default => false + , required => true + , desc => "Enable reporting of metrics via Prometheus Pushgateway." + })} ]. sc(Type, Meta) -> hoconsc:mk(Type, Meta). diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl index 09522870d..0d6eea3fc 100644 --- a/apps/emqx_statsd/src/emqx_statsd_schema.erl +++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl @@ -33,21 +33,33 @@ namespace() -> "statsd". roots() -> ["statsd"]. fields("statsd") -> - [ {enable, hoconsc:mk(boolean(), #{default => false, required => true})} + [ {enable, hoconsc:mk(boolean(), + #{ default => false + , required => true + , desc => "Enable statsd" + })} , {server, fun server/1} - , {sample_time_interval, fun duration_ms/1} - , {flush_time_interval, fun duration_ms/1} + , {sample_time_interval, fun sample_interval/1} + , {flush_time_interval, fun flush_interval/1} ]. server(type) -> emqx_schema:ip_port(); server(required) -> true; server(default) -> "127.0.0.1:8125"; +server(desc) -> "URL of the statsd gateway."; server(_) -> undefined. -duration_ms(type) -> emqx_schema:duration_ms(); -duration_ms(required) -> true; -duration_ms(default) -> "10s"; -duration_ms(_) -> undefined. +sample_interval(type) -> emqx_schema:duration_ms(); +sample_interval(required) -> true; +sample_interval(default) -> "10s"; +sample_interval(desc) -> "Data collection interval in milliseconds."; +sample_interval(_) -> undefined. + +flush_interval(type) -> emqx_schema:duration_ms(); +flush_interval(required) -> true; +flush_interval(default) -> "10s"; +flush_interval(desc) -> "Flush interval in milliseconds."; +flush_interval(_) -> undefined. to_ip_port(Str) -> case string:tokens(Str, ":") of From 8967e13eeeaf9cd710075b67dd4e0324191bb7ee Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 23 Mar 2022 13:42:11 +0100 Subject: [PATCH 3/4] docs(schema): Move descriptions from emqx.conf.all --- apps/emqx/src/emqx_schema.erl | 37 ++++++++++--- apps/emqx_conf/src/emqx_conf_schema.erl | 6 ++- .../src/emqx_dashboard_schema.erl | 5 +- apps/emqx_exhook/src/emqx_exhook_schema.erl | 31 ++++++++--- apps/emqx_gateway/src/emqx_gateway_schema.erl | 24 ++++++++- .../src/emqx_retainer_schema.erl | 54 ++++++++++++++----- 6 files changed, 126 insertions(+), 31 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index e502bd548..4219f8463 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -659,22 +659,40 @@ fields("flapping_detect") -> {"enable", sc( boolean(), - #{default => false} + #{ + default => false, + desc => + "Enable flapping connection detection feature.
\n" + "This config controls the allowed maximum number of `CONNECT` packets received\n" + "from the same clientid in a time frame defined by `window_time`.\n" + "After the limit is reached, successive `CONNECT` requests are forbidden\n" + "(banned) until the end of the time period defined by `ban_time`." + } )}, {"max_count", sc( integer(), - #{default => 15} + #{ + default => 15, + desc => + "The maximum number of disconnects allowed for a MQTT Client in `window_time`" + } )}, {"window_time", sc( duration(), - #{default => "1m"} + #{ + default => "1m", + desc => "The time window for flapping detection." + } )}, {"ban_time", sc( duration(), - #{default => "5m"} + #{ + default => "5m", + desc => "How long the flapping clientid will be banned." + } )} ]; fields("force_shutdown") -> @@ -682,18 +700,25 @@ fields("force_shutdown") -> {"enable", sc( boolean(), - #{default => true} + #{ + default => true, + desc => "Enable `force_shutdown` feature." + } )}, {"max_message_queue_len", sc( range(0, inf), - #{default => 1000} + #{ + default => 1000, + desc => "Maximum message queue length." + } )}, {"max_heap_size", sc( wordsize(), #{ default => "32MB", + desc => "Total heap size", validator => fun ?MODULE:validate_heap_size/1 } )} diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 826b73461..e1fdf2361 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -637,13 +637,17 @@ fields("console_handler") -> fields("log_file_handler") -> [ {"file", sc(file(), - #{})} + #{ desc => "Name the log file." + })} , {"rotation", sc(ref("log_rotation"), #{})} , {"max_size", sc(hoconsc:union([infinity, emqx_schema:bytesize()]), #{ default => "10MB" + , desc => "This parameter controls log file rotation. " + "The value `infinity` means the log file will grow indefinitely, " + "otherwise the log file will be rotated once it reaches `max_size` in bytes." })} ] ++ log_handler_common_confs(); diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index a2c5b3c26..36ec75963 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -29,14 +29,13 @@ fields("dashboard") -> sc(hoconsc:array(hoconsc:union([hoconsc:ref(?MODULE, "http"), hoconsc:ref(?MODULE, "https")])), #{ desc => -"""HTTP(s) listeners are identified by their protocol type and are +"HTTP(s) listeners are identified by their protocol type and are used to serve dashboard UI and restful HTTP API.
Listeners must have a unique combination of port number and IP address.
For example, an HTTP listener can listen on all configured IP addresses on a given port for a machine by specifying the IP address 0.0.0.0.
Alternatively, the HTTP listener can specify a unique IP address for each listener, -but use the same port. -""" })} +but use the same port."})} , {default_username, fun default_username/1} , {default_password, fun default_password/1} , {sample_interval, sc(emqx_schema:duration_s(), #{default => "10s"})} diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 77ae90195..e95a46beb 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -45,19 +45,36 @@ fields(exhook) -> ]; fields(server) -> - [ {name, sc(binary(), #{})} - , {enable, sc(boolean(), #{default => true})} - , {url, sc(binary(), #{})} - , {request_timeout, - sc(duration(), #{default => "5s"})} + [ {name, sc(binary(), + #{ desc => "Name of the exhook server." + })} + , {enable, sc(boolean(), + #{ default => true + , desc => "Enable the exhook server." + })} + , {url, sc(binary(), + #{ desc => "URL of the gRPC server." + })} + , {request_timeout, sc(duration(), + #{ default => "5s" + , desc => "The timeout to request gRPC server." + })} , {failed_action, failed_action()} , {ssl, sc(ref(ssl_conf), #{})} , {auto_reconnect, sc(hoconsc:union([false, duration()]), - #{default => "60s"})} + #{ default => "60s" + , desc => "Whether to automatically reconnect (initialize) the gRPC server.
" + "When gRPC is not available, exhook tries to request the gRPC service at " + "that interval and reinitialize the list of mounted hooks." + })} , {pool_size, - sc(integer(), #{default => 8, example => 8})} + sc(integer(), + #{ default => 8 + , example => 8 + , desc => "The process pool size for gRPC client." + })} ]; fields(ssl_conf) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 4440aaba8..09b0b7d1c 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -473,26 +473,48 @@ common_listener_opts() -> [ {enable, sc(boolean(), #{ default => true + , desc => "Enable the listener." })} , {bind, sc(hoconsc:union([ip_port(), integer()]), - #{})} + #{ desc => "The IP address and port that the listener will bind." + })} , {max_connections, sc(integer(), #{ default => 1024 + , desc => "Maximum number of concurrent connections." })} , {max_conn_rate, sc(integer(), #{ default => 1000 + , desc => "Maximum connections per second." })} , {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM, authentication_schema()} , {mountpoint, sc(binary(), #{ default => undefined + , desc => + "When publishing or subscribing, prefix all topics with a mountpoint string.\n" + " The prefixed string will be removed from the topic name when the message\n" + " is delivered to the subscriber. The mountpoint is a way that users can use\n" + " to implement isolation of message routing between different listeners.\n" + " For example if a client A subscribes to `t` with `listeners.tcp..mountpoint`\n" + " set to `some_tenant`, then the client actually subscribes to the topic\n" + " `some_tenant/t`. Similarly, if another client B (connected to the same listener\n" + " as the client A) sends a message to topic `t`, the message is routed\n" + " to all the clients subscribed `some_tenant/t`, so client A will receive the\n" + " message, with topic name `t`.
\n" + " Set to `\"\"` to disable the feature.
\n" + "\n" + " Variables in mountpoint string:\n" + " - ${clientid}: clientid\n" + " - ${username}: username" })} , {access_rules, sc(hoconsc:array(string()), #{ default => [] + , desc => "The access control rules for this listener.
" + "See: https://github.com/emqtt/esockd#allowdeny" })} ]. diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 12189c737..306ab5642 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -11,35 +11,63 @@ namespace() -> "retainer". roots() -> ["retainer"]. fields("retainer") -> - [ {enable, sc(boolean(), false)} - , {msg_expiry_interval, sc(emqx_schema:duration_ms(), "0s")} - , {msg_clear_interval, sc(emqx_schema:duration_ms(), "0s")} + [ {enable, sc(boolean(), "Enable retainer feature.", false)} + , {msg_expiry_interval, sc(emqx_schema:duration_ms(), + "Message retention time. 0 means message will never be expired.", + "0s")} + , {msg_clear_interval, sc(emqx_schema:duration_ms(), + "Periodic interval for cleaning up expired messages. " + "Never clear if the value is 0.", + "0s")} , {flow_control, ?TYPE(hoconsc:ref(?MODULE, flow_control))} - , {max_payload_size, sc(emqx_schema:bytesize(), "1MB")} - , {stop_publish_clear_msg, sc(boolean(), false)} + , {max_payload_size, sc(emqx_schema:bytesize(), + "Maximum retained message size.", + "1MB")} + , {stop_publish_clear_msg, sc(boolean(), + "When the retained flag of the `PUBLISH` message is set and Payload is empty, " + "whether to continue to publish the message.
" + "See: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718038", + false)} , {backend, backend_config()} ]; fields(mnesia_config) -> [ {type, ?TYPE(hoconsc:union([built_in_database]))} - , {storage_type, sc(hoconsc:union([ram, disc]), ram)} - , {max_retained_messages, sc(integer(), 0, fun is_pos_integer/1)} + , {storage_type, sc(hoconsc:union([ram, disc]), + "Specifies whether the messages are stored in RAM or persisted on disc.", + ram)} + , {max_retained_messages, sc(integer(), + "Maximum number of retained messages. 0 means no limit.", + 0, + fun is_pos_integer/1)} ]; fields(flow_control) -> - [ {batch_read_number, sc(integer(), 0, fun is_pos_integer/1)} - , {batch_deliver_number, sc(range(0, 1000), 0)} - , {batch_deliver_limiter, sc(emqx_limiter_schema:bucket_name(), undefined)} + [ {batch_read_number, sc(integer(), + "Size of the batch when reading messages from storage. 0 means no limit.", + 0, + fun is_pos_integer/1)} + , {batch_deliver_number, sc(range(0, 1000), + "The number of retained messages can be delivered per batch.", + 0)} + , {batch_deliver_limiter, sc(emqx_limiter_schema:bucket_name(), + "The rate limiter name for retained messages' delivery.
" + "Limiter helps to avoid delivering too many messages to the client at once, which may cause the client " + "to block or crash, or drop messages due to exceeding the size of the message queue.
" + "The names of the available rate limiters are taken from the existing rate limiters under `limiter.batch`.
" + "If this field is empty, limiter is not used.", + undefined)} ]. %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- -sc(Type, Default) -> - hoconsc:mk(Type, #{default => Default}). +sc(Type, Desc, Default) -> + hoconsc:mk(Type, #{default => Default, desc => Desc}). -sc(Type, Default, Validator) -> +sc(Type, Desc, Default, Validator) -> hoconsc:mk(Type, #{default => Default, + desc => Desc, validator => Validator}). is_pos_integer(V) -> From 93d1c4ec87d2012e80bfe645689b7a7aefc87e4e Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 23 Mar 2022 14:24:00 +0100 Subject: [PATCH 4/4] ci(spellcheck): Bump container version to 0.2.3 --- .github/workflows/build_slim_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index a814595db..4166514cf 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -218,7 +218,7 @@ jobs: - emqx - emqx-enterprise runs-on: ubuntu-20.04 - container: "ghcr.io/iequ1/emqx-schema-validate:0.2.2" + container: "ghcr.io/iequ1/emqx-schema-validate:0.2.3" steps: - uses: actions/download-artifact@v2 name: Download schema dump