From a9cca84595dc1e8372c75969feadfbc7138a0af5 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Fri, 3 Sep 2021 23:24:37 +0200 Subject: [PATCH 01/12] fix(emqx_schema): allow listener to be null --- apps/emqx/src/emqx_schema.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 01989c5a1..0c9f19d79 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -377,26 +377,31 @@ fields("listeners") -> [ {"tcp", sc(ref("tcp_listeners"), #{ desc => "TCP listeners" + , nullable => {true, recursive} }) } , {"ssl", sc(ref("ssl_listeners"), #{ desc => "SSL listeners" + , nullable => {true, recursive} }) } , {"ws", sc(ref("ws_listeners"), #{ desc => "HTTP websocket listeners" + , nullable => {true, recursive} }) } , {"wss", sc(ref("wss_listeners"), #{ desc => "HTTPS websocket listeners" + , nullable => {true, recursive} }) } , {"quic", sc(ref("quic_listeners"), #{ desc => "QUIC listeners" + , nullable => {true, recursive} }) } ]; From bbd1c142dea2575bc59b458bc5da9d43cf728b4b Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 4 Sep 2021 08:03:01 +0200 Subject: [PATCH 02/12] refactor(emqx_schema): use hoconsc:map type --- apps/emqx/src/emqx_schema.erl | 51 ++++++++++++++--------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 0c9f19d79..64ccbc58f 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -76,7 +76,11 @@ namespace() -> undefined. roots() -> - ["zones", + [{"zones", + sc(map("name", ref("zone")), + #{ desc => "A zones is a set configs grouped per the zone's name, there is a builtin zone named 'default' " + "The name of a zone can be bound to listners to apply zone settings to connections accepted by the bound listener." + })}, "mqtt", "flapping_detect", "force_shutdown", @@ -94,8 +98,14 @@ roots() -> "stats", "sysmon", "alarm", + {"authentication", + sc(hoconsc:lazy(hoconsc:array(map())), + #{ desc => "Default authentication configs for all MQTT listeners.
" + "For per-listener overrides see authentication " + "in listener configs" + })}, "authorization", - {"authentication", sc(hoconsc:lazy(hoconsc:array(map())), #{})} + "flapping_detect" ]. fields("stats") -> @@ -270,14 +280,7 @@ fields("mqtt") -> })} ]; -fields("zones") -> - [ {"$name", - sc(ref("zone_settings"), - #{ - } - )}]; - -fields("zone_settings") -> +fields("zone") -> Fields = ["mqtt", "stats", "flapping_detect", "force_shutdown", "conn_congestion", "rate_limit", "quota", "force_gc"], [{F, ref(emqx_zone_schema, F)} || F <- Fields]; @@ -375,53 +378,37 @@ fields("force_gc") -> fields("listeners") -> [ {"tcp", - sc(ref("tcp_listeners"), + sc(map(name, ref("mqtt_tcp_listener")), #{ desc => "TCP listeners" , nullable => {true, recursive} }) } , {"ssl", - sc(ref("ssl_listeners"), + sc(map(name, ref("mqtt_ssl_listener")), #{ desc => "SSL listeners" , nullable => {true, recursive} }) } , {"ws", - sc(ref("ws_listeners"), + sc(map(name, ref("mqtt_ws_listener")), #{ desc => "HTTP websocket listeners" , nullable => {true, recursive} }) } , {"wss", - sc(ref("wss_listeners"), + sc(map(name, ref("mqtt_wss_listener")), #{ desc => "HTTPS websocket listeners" , nullable => {true, recursive} }) } , {"quic", - sc(ref("quic_listeners"), + sc(map(ref("mqtt_quic_listener")), #{ desc => "QUIC listeners" , nullable => {true, recursive} }) } ]; -fields("tcp_listeners") -> - [ {"$name", ref("mqtt_tcp_listener")} - ]; -fields("ssl_listeners") -> - [ {"$name", ref("mqtt_ssl_listener")} - ]; -fields("ws_listeners") -> - [ {"$name", ref("mqtt_ws_listener")} - ]; -fields("wss_listeners") -> - [ {"$name", ref("mqtt_wss_listener")} - ]; -fields("quic_listeners") -> - [ {"$name", ref("mqtt_quic_listener")} - ]; - fields("mqtt_tcp_listener") -> [ {"tcp", sc(ref("tcp_opts"), @@ -1016,6 +1003,8 @@ ceiling(X) -> sc(Type, Meta) -> hoconsc:mk(Type, Meta). +map(Name, Type) -> hoconsc:map(Name, Type). + ref(Field) -> hoconsc:ref(?MODULE, Field). ref(Module, Field) -> hoconsc:ref(Module, Field). From 0039bfca6b06460620cf5c4f8551e31e8c771712 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 4 Sep 2021 08:07:11 +0200 Subject: [PATCH 03/12] refactor(emqx_machine_schema): use hoconsc:map type --- apps/emqx/src/emqx_schema.erl | 7 +++++-- apps/emqx_machine/src/emqx_machine_schema.erl | 10 +++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 64ccbc58f..c8698eb35 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -78,8 +78,11 @@ namespace() -> undefined. roots() -> [{"zones", sc(map("name", ref("zone")), - #{ desc => "A zones is a set configs grouped per the zone's name, there is a builtin zone named 'default' " - "The name of a zone can be bound to listners to apply zone settings to connections accepted by the bound listener." + #{ desc => "A zone is a set of configs grouped by the zone `$name`.
" + "The `$name` can be set to a listner's `zone` config for " + "flexible configuration mapping.
" + "NOTE: A builtin zone named `default` is auto created " + "and can not be deleted." })}, "mqtt", "flapping_detect", diff --git a/apps/emqx_machine/src/emqx_machine_schema.erl b/apps/emqx_machine/src/emqx_machine_schema.erl index 614609875..348832556 100644 --- a/apps/emqx_machine/src/emqx_machine_schema.erl +++ b/apps/emqx_machine/src/emqx_machine_schema.erl @@ -381,7 +381,7 @@ fields("rpc") -> fields("log") -> [ {"console_handler", ref("console_handler")} , {"file_handlers", - sc(ref("file_handlers"), + sc(map(name, ref("log_file_handler")), #{})} , {"error_logger", sc(atom(), @@ -396,12 +396,6 @@ fields("console_handler") -> })} ] ++ log_handler_common_confs(); -fields("file_handlers") -> - [ {"$name", - sc(ref("log_file_handler"), - #{})} - ]; - fields("log_file_handler") -> [ {"file", sc(file(), @@ -701,6 +695,8 @@ keys(Parent, Conf) -> sc(Type, Meta) -> hoconsc:mk(Type, Meta). +map(Name, Type) -> hoconsc:map(Name, Type). + ref(Field) -> hoconsc:ref(?MODULE, Field). options(static, Conf) -> From 3a60f0064389e0a973f23688a5aef63ac5ecc486 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 4 Sep 2021 08:34:05 +0200 Subject: [PATCH 04/12] refactor(emqx_gateway_schema): use hoconsc:map --- apps/emqx/src/emqx_schema.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 141 ++++++++---------- 2 files changed, 62 insertions(+), 81 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index c8698eb35..e31ab2bf1 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -405,7 +405,7 @@ fields("listeners") -> }) } , {"quic", - sc(map(ref("mqtt_quic_listener")), + sc(map(name, ref("mqtt_quic_listener")), #{ desc => "QUIC listeners" , nullable => {true, recursive} }) diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 7fb945ba0..70528d2bc 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -50,16 +50,16 @@ namespace() -> gateway. roots() -> [gateway]. fields(gateway) -> - [{stomp, sc(ref(stomp_structs))}, - {mqttsn, sc(ref(mqttsn_structs))}, - {coap, sc(ref(coap_structs))}, - {lwm2m, sc(ref(lwm2m_structs))}, - {exproto, sc(ref(exproto_structs))} + [{stomp, sc(ref(stomp))}, + {mqttsn, sc(ref(mqttsn))}, + {coap, sc(ref(coap))}, + {lwm2m, sc(ref(lwm2m))}, + {exproto, sc(ref(exproto))} ]; -fields(stomp_structs) -> +fields(stomp) -> [ {frame, sc(ref(stomp_frame))} - , {listeners, sc(ref(tcp_listener_group))} + , {listeners, sc(ref(tcp_listeners))} ] ++ gateway_common_options(); fields(stomp_frame) -> @@ -68,12 +68,12 @@ fields(stomp_frame) -> , {max_body_length, sc(integer(), 8192)} ]; -fields(mqttsn_structs) -> +fields(mqttsn) -> [ {gateway_id, sc(integer())} , {broadcast, sc(boolean())} , {enable_qos3, sc(boolean())} , {predefined, hoconsc:array(ref(mqttsn_predefined))} - , {listeners, sc(ref(udp_listener_group))} + , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); fields(mqttsn_predefined) -> @@ -81,34 +81,34 @@ fields(mqttsn_predefined) -> , {topic, sc(binary())} ]; -fields(coap_structs) -> +fields(coap) -> [ {heartbeat, sc(duration(), <<"30s">>)} , {connection_required, sc(boolean(), false)} - , {notify_type, sc(union([non, con, qos]), qos)} - , {subscribe_qos, sc(union([qos0, qos1, qos2, coap]), coap)} - , {publish_qos, sc(union([qos0, qos1, qos2, coap]), coap)} - , {listeners, sc(ref(udp_listener_group))} + , {notify_type, sc(hoconsc:union([non, con, qos]), qos)} + , {subscribe_qos, sc(hoconsc:union([qos0, qos1, qos2, coap]), coap)} + , {publish_qos, sc(hoconsc:union([qos0, qos1, qos2, coap]), coap)} + , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); -fields(lwm2m_structs) -> +fields(lwm2m) -> [ {xml_dir, sc(binary())} , {lifetime_min, sc(duration())} , {lifetime_max, sc(duration())} , {qmode_time_windonw, sc(integer())} , {auto_observe, sc(boolean())} - , {update_msg_publish_condition, sc(union([always, contains_object_list]))} + , {update_msg_publish_condition, sc(hoconsc:union([always, contains_object_list]))} , {translators, sc(ref(translators))} - , {listeners, sc(ref(udp_listener_group))} + , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); -fields(exproto_structs) -> +fields(exproto) -> [ {server, sc(ref(exproto_grpc_server))} , {handler, sc(ref(exproto_grpc_handler))} - , {listeners, sc(ref(udp_tcp_listener_group))} + , {listeners, sc(ref(udp_tcp_listeners))} ] ++ gateway_common_options(); fields(exproto_grpc_server) -> - [ {bind, sc(union(ip_port(), integer()))} + [ {bind, sc(hoconsc:union([ip_port(), integer()]))} %% TODO: ssl options ]; @@ -136,62 +136,45 @@ fields(translator) -> , {qos, sc(range(0, 2))} ]; -fields(udp_listener_group) -> - [ {udp, sc(ref(udp_listener))} - , {dtls, sc(ref(dtls_listener))} +fields(udp_listeners) -> + [ {udp, sc(map(name, ref(udp_listener)))} + , {dtls, sc(map(name, ref(dtls_listener)))} ]; -fields(tcp_listener_group) -> - [ {tcp, sc(ref(tcp_listener))} - , {ssl, sc(ref(ssl_listener))} +fields(tcp_listeners) -> + [ {tcp, sc(map(name, ref(tcp_listener)))} + , {ssl, sc(map(name, ref(ssl_listener)))} ]; -fields(udp_tcp_listener_group) -> - [ {udp, sc(ref(udp_listener))} - , {dtls, sc(ref(dtls_listener))} - , {tcp, sc(ref(tcp_listener))} - , {ssl, sc(ref(ssl_listener))} +fields(udp_tcp_listeners) -> + [ {udp, sc(map(name, ref(udp_listener)))} + , {dtls, sc(map(name, ref(dtls_listener)))} + , {tcp, sc(map(name, ref(tcp_listener)))} + , {ssl, sc(map(name, ref(ssl_listener)))} ]; fields(tcp_listener) -> - [ {"$name", sc(ref(tcp_listener_settings))}]; - -fields(ssl_listener) -> - [ {"$name", sc(ref(ssl_listener_settings))}]; - -fields(udp_listener) -> - [ {"$name", sc(ref(udp_listener_settings))}]; - -fields(dtls_listener) -> - [ {"$name", sc(ref(dtls_listener_settings))}]; - -fields(tcp_listener_settings) -> [ %% some special confs for tcp listener - ] ++ tcp_opts() - ++ proxy_protocol_opts() - ++ common_listener_opts(); + ] ++ + tcp_opts() ++ + proxy_protocol_opts() ++ + common_listener_opts(); -fields(ssl_listener_settings) -> - [ - %% some special confs for ssl listener - ] ++ tcp_opts() - ++ ssl_opts() - ++ proxy_protocol_opts() - ++ common_listener_opts(); +fields(ssl_listener) -> + fields(tcp_listener) ++ + ssl_opts(); -fields(udp_listener_settings) -> +fields(udp_listener) -> [ %% some special confs for udp listener - ] ++ udp_opts() - ++ common_listener_opts(); + ] ++ + udp_opts() ++ + common_listener_opts(); -fields(dtls_listener_settings) -> - [ - %% some special confs for dtls listener - ] ++ udp_opts() - ++ dtls_opts() - ++ common_listener_opts(); +fields(dtls_listener) -> + fields(udp_listener) ++ + dtls_opts(); fields(udp_opts) -> [ {active_n, sc(integer(), 100)} @@ -218,11 +201,7 @@ fields(dtls_listener_ssl_opts) -> lists:keyreplace("versions", 1, Base, {"versions", DtlsVers}), {"ciphers", Ciphers} ) - ); - -fields(ExtraField) -> - Mod = list_to_atom(ExtraField++"_schema"), - Mod:fields(ExtraField). + ). default_ciphers() -> ["ECDHE-ECDSA-AES256-GCM-SHA384", @@ -286,16 +265,16 @@ common_listener_opts() -> ]. tcp_opts() -> - [{tcp, sc(ref(emqx_schema, "tcp_opts"), #{})}]. + [{tcp, sc_meta(ref(emqx_schema, "tcp_opts"), #{})}]. udp_opts() -> - [{udp, sc(ref(udp_opts), #{})}]. + [{udp, sc_meta(ref(udp_opts), #{})}]. ssl_opts() -> - [{ssl, sc(ref(emqx_schema, "listener_ssl_opts"), #{})}]. + [{ssl, sc_meta(ref(emqx_schema, "listener_ssl_opts"), #{})}]. dtls_opts() -> - [{dtls, sc(ref(dtls_listener_ssl_opts), #{})}]. + [{dtls, sc_meta(ref(dtls_listener_ssl_opts), #{})}]. proxy_protocol_opts() -> [ {proxy_protocol, sc(boolean())} @@ -308,18 +287,20 @@ default_dtls_vsns() -> dtls_vsn(<<"dtlsv1.2">>) -> 'dtlsv1.2'; dtls_vsn(<<"dtlsv1">>) -> 'dtlsv1'. -%%-------------------------------------------------------------------- -%% Helpers - -%% types - -sc(Type) -> #{type => Type}. +sc(Type) -> + sc_meta(Type, #{}). sc(Type, Default) -> - hoconsc:mk(Type, #{default => Default}). + sc_meta(Type, #{default => Default}). -ref(Field) -> - hoconsc:ref(?MODULE, Field). +sc_meta(Type, Meta) -> + hoconsc:mk(Type, Meta). + +map(Name, Type) -> + hoconsc: map(Name, Type). + +ref(StructName) -> + ref(?MODULE, StructName). ref(Mod, Field) -> hoconsc:ref(Mod, Field). From 6b7d3bcf986d7b42c29c4860d0fe8cb19d3a26b1 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 4 Sep 2021 13:39:58 +0200 Subject: [PATCH 05/12] chore(hocon): upgrade to 0.17.0 --- apps/emqx/rebar.config | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index bb3a588a9..a8462ad82 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -15,7 +15,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.15.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} diff --git a/rebar.config b/rebar.config index 1abeef9a6..7210f11b0 100644 --- a/rebar.config +++ b/rebar.config @@ -60,7 +60,7 @@ , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.15.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.0"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} From 6f99f145404e41110ae8f5c63cd8d5368d634b1f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 4 Sep 2021 23:05:41 +0200 Subject: [PATCH 06/12] feat(bin/emqx): add cold_eval nodetool command --- bin/emqx | 19 ++++++++-------- bin/nodetool | 63 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 49 insertions(+), 33 deletions(-) diff --git a/bin/emqx b/bin/emqx index 41f2c0db5..e6cd421f1 100755 --- a/bin/emqx +++ b/bin/emqx @@ -14,6 +14,11 @@ ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" # shellcheck disable=SC1090 . "$ROOT_DIR"/releases/emqx_vars +# defined in emqx_vars +export RUNNER_ROOT_DIR +export RUNNER_ETC_DIR +export REL_VSN + RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME" CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}" REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN" @@ -109,7 +114,7 @@ relx_usage() { echo " don't make it permanent" ;; *) - echo "Usage: $REL_NAME {start|start_boot |ertspath|foreground|stop|pid|ping|console|console_clean|console_boot |attach|remote_console|upgrade|downgrade|install|uninstall|versions|escript|ctl|rpc|rpcterms|eval|root_dir}" + echo "Usage: $REL_NAME {start|start_boot |ertspath|foreground|stop|pid|ping|console|console_clean|console_boot |attach|remote_console|upgrade|downgrade|install|uninstall|versions|escript|ctl|rpc|rpcterms|eval|cold_eval|root_dir}" ;; esac } @@ -198,18 +203,12 @@ relx_gen_id() { # Control a node relx_nodetool() { command="$1"; shift - export RUNNER_ROOT_DIR - export REL_VSN - ERL_FLAGS="$ERL_FLAGS $EPMD_ARG" \ "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \ -setcookie "$COOKIE" "$command" "$@" } call_hocon() { - export RUNNER_ROOT_DIR - export RUNNER_ETC_DIR - export REL_VSN "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" hocon "$@" \ || die "call_hocon_failed: $*" $? } @@ -217,8 +216,6 @@ call_hocon() { # Run an escript in the node's environment relx_escript() { shift; scriptpath="$1"; shift - export RUNNER_ROOT_DIR - "$ERTS_DIR/bin/escript" "$ROOTDIR/$scriptpath" "$@" } @@ -709,6 +706,10 @@ case "$1" in shift relx_nodetool "eval" "$@" ;; + cold_eval) + shift; + "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" cold_eval "$@" + ;; *) relx_usage "$1" exit 1 diff --git a/bin/nodetool b/bin/nodetool index 377ade040..298d15a0b 100755 --- a/bin/nodetool +++ b/bin/nodetool @@ -24,6 +24,8 @@ main(Args) -> ["hocon" | Rest] -> %% forward the call to hocon_cli hocon_cli:main(Rest); + ["cold_eval" | Rest] -> + code_eval(Rest); _ -> do(Args) end. @@ -117,40 +119,53 @@ do(Args) -> io:format("~p\n", [Other]) end; ["eval" | ListOfArgs] -> - % shells may process args into more than one, and end up stripping - % spaces, so this converts all of that to a single string to parse - String = binary_to_list( - list_to_binary( - join(ListOfArgs," ") - ) - ), - - % then just as a convenience to users, if they forgot a trailing - % '.' add it for them. - Normalized = - case lists:reverse(String) of - [$. | _] -> String; - R -> lists:reverse([$. | R]) - end, - - % then scan and parse the string - {ok, Scanned, _} = erl_scan:string(Normalized), - {ok, Parsed } = erl_parse:parse_exprs(Scanned), - + Parsed = parse_eval_args(ListOfArgs), % and evaluate it on the remote node case rpc:call(TargetNode, erl_eval, exprs, [Parsed, [] ]) of {value, Value, _} -> - io:format ("~p\n",[Value]); + io:format ("~p~n",[Value]); {badrpc, Reason} -> - io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), + io:format("RPC to ~p failed: ~p~n", [TargetNode, Reason]), halt(1) end; Other -> - io:format("Other: ~p\n", [Other]), - io:format("Usage: nodetool {genconfig, chkconfig|getpid|ping|stop|rpc|rpc_infinity|rpcterms|eval [Terms]} [RPC]\n") + io:format("Other: ~p~n", [Other]), + io:format("Usage: nodetool chkconfig|getpid|ping|stop|rpc|rpc_infinity|rpcterms|eval|cold_eval [Terms] [RPC]\n") end, net_kernel:stop(). +code_eval(Args) -> + Parsed = parse_eval_args(Args), + case erl_eval:exprs(Parsed, []) of + {value, Value, _} -> + io:format ("~p~n", [Value]); + Other -> + io:format("cold_eval_failed_with: ~p~n", [Other]), + halt(1) + end. + +parse_eval_args(Args) -> + % shells may process args into more than one, and end up stripping + % spaces, so this converts all of that to a single string to parse + String = binary_to_list( + list_to_binary( + join(Args," ") + ) + ), + + % then just as a convenience to users, if they forgot a trailing + % '.' add it for them. + Normalized = + case lists:reverse(String) of + [$. | _] -> String; + R -> lists:reverse([$. | R]) + end, + + % then scan and parse the string + {ok, Scanned, _} = erl_scan:string(Normalized), + {ok, Parsed } = erl_parse:parse_exprs(Scanned), + Parsed. + do_with_ret(Args, Name, Handler) -> {arity, Arity} = erlang:fun_info(Handler, arity), case take_args(Args, Name, Arity) of From 687114eeb17f70672b38bce0c53aec2407746073 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Sat, 4 Sep 2021 23:06:12 +0200 Subject: [PATCH 07/12] fix(bridge_schema): ensure atom exists --- .../src/emqx_data_bridge_schema.erl | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl index e3c6d8ee9..fe2d3947d 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl @@ -14,13 +14,12 @@ fields("emqx_data_bridge") -> [{bridges, #{type => hoconsc:array(hoconsc:union(?BRIDGES)), default => []}}]; -fields(mysql) -> connector_fields(mysql); -fields(pgsql) -> connector_fields(pgsql); -fields(mongo) -> connector_fields(mongo); -fields(redis) -> connector_fields(redis); -fields(ldap) -> connector_fields(ldap). +fields(mysql) -> connector_fields(emqx_connector_mysql, mysql); +fields(pgsql) -> connector_fields(emqx_connector_pgsql, pgsql); +fields(mongo) -> connector_fields(emqx_connector_mongo, mongo); +fields(redis) -> connector_fields(emqx_connector_redis, redis); +fields(ldap) -> connector_fields(emqx_connector_ldap, ldap). -connector_fields(DB) -> - Mod = list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])), +connector_fields(ConnectModule, DB) -> [{name, hoconsc:mk(typerefl:binary())}, - {type, #{type => DB}}] ++ Mod:roots(). + {type, #{type => DB}}] ++ ConnectModule:roots(). From 01166a8bfbdfcaa3c33c4db6a9d9ec52806ffb02 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 8 Sep 2021 10:29:25 +0200 Subject: [PATCH 08/12] feat(schema): generate document when building the release --- apps/emqx/src/emqx_schema.erl | 38 ++++++++++++------- apps/emqx_machine/src/emqx_machine_schema.erl | 36 +++++++++++++----- build | 4 ++ 3 files changed, 55 insertions(+), 23 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index e31ab2bf1..f9ee7e7e0 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -76,31 +76,41 @@ namespace() -> undefined. roots() -> - [{"zones", + [ {"listeners", + sc(ref("listeners"), + #{ desc => "MQTT listeners identified by their protocol type and assigned names" + }) + }, + {"zones", sc(map("name", ref("zone")), - #{ desc => "A zone is a set of configs grouped by the zone `$name`.
" - "The `$name` can be set to a listner's `zone` config for " - "flexible configuration mapping.
" - "NOTE: A builtin zone named `default` is auto created " + #{ desc => "A zone is a set of configs grouped by the zone name.
" + "For flexible configuration mapping, the name " + "can be set to a listener's zone config .
" + "NOTE: A builtin zone named default is auto created " "and can not be deleted." })}, - "mqtt", - "flapping_detect", + {"mqtt", + sc(ref("mqtt"), + #{ desc => "Global MQTT configuration.
" + "The configs here work as default values which can be overriden " + "in zone configs" + })}, + "rate_limit", "force_shutdown", "force_gc", "conn_congestion", - "rate_limit", "quota", - {"listeners", - sc(ref("listeners"), - #{ desc => "MQTT listeners identified by their protocol type and assigned names. " - "The listeners enabled by default are named with 'default'"}) - }, "broker", - "plugins", + "plugins", %% TODO: move to emqx_machine_schema "stats", "sysmon", "alarm", + {"authentication", + sc(hoconsc:lazy(hoconsc:array(map())), + #{ desc => "Default authentication configs for all MQTT listeners.
" + "For per-listener overrides see authentication " + "in listener configs" + })}, {"authentication", sc(hoconsc:lazy(hoconsc:array(map())), #{ desc => "Default authentication configs for all MQTT listeners.
" diff --git a/apps/emqx_machine/src/emqx_machine_schema.erl b/apps/emqx_machine/src/emqx_machine_schema.erl index 348832556..5ef9df7c6 100644 --- a/apps/emqx_machine/src/emqx_machine_schema.erl +++ b/apps/emqx_machine/src/emqx_machine_schema.erl @@ -59,13 +59,32 @@ namespace() -> undefined. roots() -> - %% This is a temp workaround to define part of authorization config - %% in emqx_schema and part of it in emqx_authz_schema but then - %% merged here in this module - %% The proper fix should be to make connection (channel, session) state - %% extendable by e.g. allow hooks be stateful. - ["cluster", "node", "rpc", "log", "authorization"] ++ - lists:keydelete("authorization", 1, lists:flatmap(fun roots/1, ?MERGED_CONFIGS)). + lists:flatmap(fun roots/1, ?MERGED_CONFIGS) ++ + [ {"node", + sc(hoconsc:ref("node"), + #{ desc => "Node name, cookie, config & data directories " + "and the Eralng virtual machine (beam) boot parameters." + })} + , {"cluster", + sc(hoconsc:ref("cluster"), + #{ desc => "EMQ X nodes can form a cluster to scale up the total capacity.
" + "Here holds the configs to instruct how individual nodes " + "can discover each other, also the database replication " + "role of this node etc." + })} + , {"log", + sc(hoconsc:ref("log"), + #{ desc => "Configure logging backends (to console or to file), " + "and logging level for each logger backend." + })} + , {"rpc", + sc(hoconsc:ref("rpc"), + #{ desc => "EMQ X uses a library called gen_rpc for " + "inter-broker RPCs.
Most of the time the default config " + "should work, but in case you need to do performance " + "fine-turning or experiment a bit, this is where to look." + })} + ]. fields("cluster") -> [ {"name", @@ -738,5 +757,4 @@ to_atom(Bin) when is_binary(Bin) -> binary_to_atom(Bin, utf8). roots(Module) -> - lists:map(fun({_BinName, Root}) -> Root end, - maps:to_list(hocon_schema:roots(Module))). + lists:map(fun({_BinName, Root}) -> Root end, hocon_schema:roots(Module)). diff --git a/build b/build index 27626387e..481249b7c 100755 --- a/build +++ b/build @@ -69,6 +69,10 @@ make_rel() { echo "gpb should not be included in the release" exit 1 fi + local conf_doc + conf_doc="$(pwd)/_build/${PROFILE}/rel/emqx/etc/emqx-config-doc.html" + echo "===< Generating config document $conf_doc" + ./_build/"$PROFILE"/rel/emqx/bin/emqx cold_eval "file:write_file('$conf_doc', hocon_schema_html:gen(emqx_machine_schema, \"EMQ X ${PKG_VSN}\"))" } ## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup From 0b11ab2d596dfa0a94b092c871aef8ddeb8e816f Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Wed, 8 Sep 2021 11:01:07 +0200 Subject: [PATCH 09/12] refactor(schema): reorder config root fields so more important configs are ordered before less important --- apps/emqx/src/emqx_schema.erl | 95 ++++++++++++------- apps/emqx_machine/src/emqx_machine_schema.erl | 17 +++- 2 files changed, 73 insertions(+), 39 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index f9ee7e7e0..e9a4385c7 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -69,56 +69,82 @@ cipher/0, comma_separated_atoms/0]). --export([namespace/0, roots/0, fields/1]). +-export([namespace/0, roots/0, roots/1, fields/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([ssl/1]). namespace() -> undefined. roots() -> + %% TODO change config importance to a field metadata + roots(high) ++ roots(medium) ++ roots(low). + +roots(high) -> [ {"listeners", sc(ref("listeners"), #{ desc => "MQTT listeners identified by their protocol type and assigned names" }) - }, - {"zones", - sc(map("name", ref("zone")), - #{ desc => "A zone is a set of configs grouped by the zone name.
" - "For flexible configuration mapping, the name " - "can be set to a listener's zone config .
" - "NOTE: A builtin zone named default is auto created " - "and can not be deleted." - })}, - {"mqtt", - sc(ref("mqtt"), + } + , {"zones", + sc(map("name", ref("zone")), + #{ desc => "A zone is a set of configs grouped by the zone name.
" + "For flexible configuration mapping, the name " + "can be set to a listener's zone config .
" + "NOTE: A builtin zone named default is auto created " + "and can not be deleted." + })} + , {"mqtt", + sc(ref("mqtt"), #{ desc => "Global MQTT configuration.
" "The configs here work as default values which can be overriden " "in zone configs" - })}, - "rate_limit", - "force_shutdown", - "force_gc", - "conn_congestion", - "quota", - "broker", - "plugins", %% TODO: move to emqx_machine_schema - "stats", - "sysmon", - "alarm", - {"authentication", + })} + , {"authentication", sc(hoconsc:lazy(hoconsc:array(map())), #{ desc => "Default authentication configs for all MQTT listeners.
" "For per-listener overrides see authentication " "in listener configs" - })}, - {"authentication", - sc(hoconsc:lazy(hoconsc:array(map())), - #{ desc => "Default authentication configs for all MQTT listeners.
" - "For per-listener overrides see authentication " - "in listener configs" - })}, - "authorization", - "flapping_detect" + })} + , {"authorization", + sc(ref("authorization"), + #{})} + ]; +roots(medium) -> + [ {"broker", + sc(ref("broker"), + #{})} + , {"rate_limit", + sc(ref("rate_limit"), + #{})} + , {"force_shutdown", + sc(ref("force_shutdown"), + #{})} + ]; +roots(low) -> + [ {"force_gc", + sc(ref("force_gc"), + #{})} + , {"conn_congestion", + sc(ref("conn_congestion"), + #{})} + , {"quota", + sc(ref("quota"), + #{})} + , {"plugins", %% TODO: move to emqx_machine_schema + sc(ref("plugins"), + #{})} + , {"stats", + sc(ref("stats"), + #{})} + , {"sysmon", + sc(ref("sysmon"), + #{})} + , {"alarm", + sc(ref("alarm"), + #{})} + , {"flapping_detect", + sc(ref("flapping_detect"), + #{})} ]. fields("stats") -> @@ -140,8 +166,7 @@ fields("authorization") -> , {"cache", sc(ref(?MODULE, "cache"), #{ - }) - } + })} ]; fields("cache") -> diff --git a/apps/emqx_machine/src/emqx_machine_schema.erl b/apps/emqx_machine/src/emqx_machine_schema.erl index 5ef9df7c6..e4f7a8d1a 100644 --- a/apps/emqx_machine/src/emqx_machine_schema.erl +++ b/apps/emqx_machine/src/emqx_machine_schema.erl @@ -42,8 +42,7 @@ %% The list can not be made a dynamic read at run-time as it is used %% by nodetool to generate app.