From 497e08448d0d85d0a3f9f493709f7193159fad2f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 2 Oct 2023 20:29:25 +0200 Subject: [PATCH] feat(cluster): support ipv6 and tls on ipv6 for clustering Made possible to configure inet6_tls for Erlang distribution Also, added support to configure ipv6 listener for gen_rpc --- apps/emqx_conf/src/emqx_conf_schema.erl | 24 +++++++++++++++++++++--- bin/emqx | 18 ++++++++---------- rel/i18n/emqx_conf_schema.hocon | 11 ++++++++++- 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 051e6885d..481eb59f2 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -195,7 +195,7 @@ fields("cluster") -> )}, {"proto_dist", sc( - hoconsc:enum([inet_tcp, inet6_tcp, inet_tls]), + hoconsc:enum([inet_tcp, inet6_tcp, inet_tls, inet6_tls]), #{ mapping => "ekka.proto_dist", default => inet_tcp, @@ -948,7 +948,16 @@ fields("rpc") -> } )}, {"ciphers", emqx_schema:ciphers_schema(tls_all_available)}, - {"tls_versions", emqx_schema:tls_versions_schema(tls_all_available)} + {"tls_versions", emqx_schema:tls_versions_schema(tls_all_available)}, + {"listen_address", + sc( + string(), + #{ + default => "0.0.0.0", + desc => ?DESC(rpc_listen_address), + importance => ?IMPORTANCE_MEDIUM + } + )} ]; fields("log") -> [ @@ -1133,7 +1142,16 @@ translation("gen_rpc") -> [ {"default_client_driver", fun tr_default_config_driver/1}, {"ssl_client_options", fun tr_gen_rpc_ssl_options/1}, - {"ssl_server_options", fun tr_gen_rpc_ssl_options/1} + {"ssl_server_options", fun tr_gen_rpc_ssl_options/1}, + {"socket_ip", fun(Conf) -> + Addr = conf_get("rpc.listen_address", Conf), + case inet:parse_address(Addr) of + {ok, Tuple} -> + Tuple; + {error, _Reason} -> + throw(#{bad_ip_address => Addr}) + end + end} ]; translation("prometheus") -> [ diff --git a/bin/emqx b/bin/emqx index 5484a82e5..f24210cdf 100755 --- a/bin/emqx +++ b/bin/emqx @@ -522,17 +522,13 @@ else ## only one emqx node is running, get running args from 'ps -ef' output tmp_nodename=$(echo -e "$PS_LINE" | $GREP -oE "\s-s?name.*" | awk '{print $2}' || true) tmp_cookie=$(echo -e "$PS_LINE" | $GREP -oE "\s-setcookie.*" | awk '{print $2}' || true) + tmp_proto_dist=$(echo -e "$PS_LINE" | $GREP -oE '\s-ekka_proto_dist.*' | awk '{print $2}' || echo 'inet_tcp') SSL_DIST_OPTFILE="$(echo -e "$PS_LINE" | $GREP -oE '\-ssl_dist_optfile\s.+\s' | awk '{print $2}' || true)" tmp_ticktime="$(echo -e "$PS_LINE" | $GREP -oE '\s-kernel\snet_ticktime\s.+\s' | awk '{print $3}' || true)" # data_dir is actually not needed, but kept anyway tmp_datadir="$(echo -e "$PS_LINE" | $GREP -oE "\-emqx_data_dir.*" | sed -E 's#.+emqx_data_dir[[:blank:]]##g' | sed -E 's#[[:blank:]]--$##g' || true)" - if [ -z "$SSL_DIST_OPTFILE" ]; then - tmp_proto='inet_tcp' - else - tmp_proto='inet_tls' - fi ## Make the format like what call_hocon multi_get prints out, but only need 4 args - EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}" + EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto_dist}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}" else if [ "$RUNNING_NODES_COUNT" -gt 1 ]; then if [ -z "${EMQX_NODE__NAME:-}" ]; then @@ -567,7 +563,7 @@ TICKTIME="$(get_boot_config 'node.dist_net_ticktime' || echo '120')" # this environment variable is required by ekka_dist module # because proto_dist is overriden to ekka, and there is a lack of ekka_tls module export EKKA_PROTO_DIST_MOD="${PROTO_DIST:-inet_tcp}" -if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ]; then +if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ] || [ "$EKKA_PROTO_DIST_MOD" = 'inet6_tls' ]; then if [ "$IS_BOOT_COMMAND" = 'yes' ]; then SSL_DIST_OPTFILE=${EMQX_SSL_DIST_OPTFILE:-"$EMQX_ETC_DIR/ssl_dist.conf"} case "$SSL_DIST_OPTFILE" in @@ -1216,7 +1212,6 @@ case "${COMMAND}" in export PROGNAME # Store passed arguments since they will be erased by `set` - # add emqx_data_dir to boot command so it is visible from 'ps -ef' ARGS="$*" # shellcheck disable=SC2086 @@ -1247,10 +1242,13 @@ case "${COMMAND}" in fi # Log the startup - logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS} -emqx_data_dir ${DATA_DIR}" + logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS} -ekka_proto_dist ${EKKA_PROTO_DIST_MOD} -emqx_data_dir ${DATA_DIR}" # Start the VM - exec "$@" -- ${1+$ARGS} -emqx_data_dir "${DATA_DIR}" + # add ekka_proto_dist emqx_data_dir to boot command so it is visible from 'ps -ef' + # NTOE: order matters! emqx_data_dir has to be positioned at the end of the line to simplify the + # line parsing when file path contains spaces + exec "$@" -- ${1+$ARGS} -ekka_proto_dist "${EKKA_PROTO_DIST_MOD}" -emqx_data_dir "${DATA_DIR}" ;; ctl) diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon index e1f2120ca..13f852b3f 100644 --- a/rel/i18n/emqx_conf_schema.hocon +++ b/rel/i18n/emqx_conf_schema.hocon @@ -60,7 +60,9 @@ node_etc_dir.label: cluster_proto_dist.desc: """The Erlang distribution protocol for the cluster.
- inet_tcp: IPv4 TCP
-- inet_tls: IPv4 TLS, works together with etc/ssl_dist.conf""" +- inet_tls: IPv4 TLS, works together with etc/ssl_dist.conf
+- inet6_tcp: IPv6 TCP
+- inet6_tls: IPv6 TLS, works together with etc/ssl_dist.conf""" cluster_proto_dist.label: """Cluster Protocol Distribution""" @@ -192,6 +194,13 @@ rpc_insecure_fallback.desc: rpc_insecure_fallback.label: """RPC insecure fallback""" +rpc_listen_address.desc: +"""Specify which IP address the RPC server should listen on. +For example "0.0.0.0" (default) for IPv4 and "::" for IPv6""" + +rpc_listen_address.label: +"""RPC Listen Address""" + cluster_mcast_buffer.desc: """Size of the user-level buffer."""