diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index a69060b9d..544848889 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -28,8 +28,8 @@
{gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
{cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}},
- {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.15"}}},
- {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.1"}}},
+ {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}},
+ {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.0"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.16"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl
index 051e6885d..7ec5348fc 100644
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@@ -195,7 +195,7 @@ fields("cluster") ->
)},
{"proto_dist",
sc(
- hoconsc:enum([inet_tcp, inet6_tcp, inet_tls]),
+ hoconsc:enum([inet_tcp, inet6_tcp, inet_tls, inet6_tls]),
#{
mapping => "ekka.proto_dist",
default => inet_tcp,
@@ -948,7 +948,26 @@ fields("rpc") ->
}
)},
{"ciphers", emqx_schema:ciphers_schema(tls_all_available)},
- {"tls_versions", emqx_schema:tls_versions_schema(tls_all_available)}
+ {"tls_versions", emqx_schema:tls_versions_schema(tls_all_available)},
+ {"listen_address",
+ sc(
+ string(),
+ #{
+ default => "0.0.0.0",
+ desc => ?DESC(rpc_listen_address),
+ importance => ?IMPORTANCE_MEDIUM
+ }
+ )},
+ {"ipv6_only",
+ sc(
+ boolean(),
+ #{
+ default => false,
+ mapping => "gen_rpc.ipv6_only",
+ desc => ?DESC(rpc_ipv6_only),
+ importance => ?IMPORTANCE_LOW
+ }
+ )}
];
fields("log") ->
[
@@ -1133,7 +1152,16 @@ translation("gen_rpc") ->
[
{"default_client_driver", fun tr_default_config_driver/1},
{"ssl_client_options", fun tr_gen_rpc_ssl_options/1},
- {"ssl_server_options", fun tr_gen_rpc_ssl_options/1}
+ {"ssl_server_options", fun tr_gen_rpc_ssl_options/1},
+ {"socket_ip", fun(Conf) ->
+ Addr = conf_get("rpc.listen_address", Conf),
+ case inet:parse_address(Addr) of
+ {ok, Tuple} ->
+ Tuple;
+ {error, _Reason} ->
+ throw(#{bad_ip_address => Addr})
+ end
+ end}
];
translation("prometheus") ->
[
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index ae8e29593..c353742ae 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -2,7 +2,7 @@
{application, emqx_rule_engine, [
{description, "EMQX Rule Engine"},
% strict semver, bump manually!
- {vsn, "5.0.27"},
+ {vsn, "5.0.28"},
{modules, []},
{registered, [emqx_rule_engine_sup, emqx_rule_engine]},
{applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
index 37797cb4f..aa5e271eb 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl
@@ -80,14 +80,18 @@
tanh/1
]).
-%% Bits Funcs
+%% Bitwise operations
-export([
bitnot/1,
bitand/2,
bitor/2,
bitxor/2,
bitsl/2,
- bitsr/2,
+ bitsr/2
+]).
+
+%% binary and bitstring Funcs
+-export([
bitsize/1,
bytesize/1,
subbits/2,
diff --git a/bin/emqx b/bin/emqx
index 5484a82e5..f24210cdf 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -522,17 +522,13 @@ else
## only one emqx node is running, get running args from 'ps -ef' output
tmp_nodename=$(echo -e "$PS_LINE" | $GREP -oE "\s-s?name.*" | awk '{print $2}' || true)
tmp_cookie=$(echo -e "$PS_LINE" | $GREP -oE "\s-setcookie.*" | awk '{print $2}' || true)
+ tmp_proto_dist=$(echo -e "$PS_LINE" | $GREP -oE '\s-ekka_proto_dist.*' | awk '{print $2}' || echo 'inet_tcp')
SSL_DIST_OPTFILE="$(echo -e "$PS_LINE" | $GREP -oE '\-ssl_dist_optfile\s.+\s' | awk '{print $2}' || true)"
tmp_ticktime="$(echo -e "$PS_LINE" | $GREP -oE '\s-kernel\snet_ticktime\s.+\s' | awk '{print $3}' || true)"
# data_dir is actually not needed, but kept anyway
tmp_datadir="$(echo -e "$PS_LINE" | $GREP -oE "\-emqx_data_dir.*" | sed -E 's#.+emqx_data_dir[[:blank:]]##g' | sed -E 's#[[:blank:]]--$##g' || true)"
- if [ -z "$SSL_DIST_OPTFILE" ]; then
- tmp_proto='inet_tcp'
- else
- tmp_proto='inet_tls'
- fi
## Make the format like what call_hocon multi_get prints out, but only need 4 args
- EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}"
+ EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto_dist}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}"
else
if [ "$RUNNING_NODES_COUNT" -gt 1 ]; then
if [ -z "${EMQX_NODE__NAME:-}" ]; then
@@ -567,7 +563,7 @@ TICKTIME="$(get_boot_config 'node.dist_net_ticktime' || echo '120')"
# this environment variable is required by ekka_dist module
# because proto_dist is overriden to ekka, and there is a lack of ekka_tls module
export EKKA_PROTO_DIST_MOD="${PROTO_DIST:-inet_tcp}"
-if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ]; then
+if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ] || [ "$EKKA_PROTO_DIST_MOD" = 'inet6_tls' ]; then
if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
SSL_DIST_OPTFILE=${EMQX_SSL_DIST_OPTFILE:-"$EMQX_ETC_DIR/ssl_dist.conf"}
case "$SSL_DIST_OPTFILE" in
@@ -1216,7 +1212,6 @@ case "${COMMAND}" in
export PROGNAME
# Store passed arguments since they will be erased by `set`
- # add emqx_data_dir to boot command so it is visible from 'ps -ef'
ARGS="$*"
# shellcheck disable=SC2086
@@ -1247,10 +1242,13 @@ case "${COMMAND}" in
fi
# Log the startup
- logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS} -emqx_data_dir ${DATA_DIR}"
+ logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS} -ekka_proto_dist ${EKKA_PROTO_DIST_MOD} -emqx_data_dir ${DATA_DIR}"
# Start the VM
- exec "$@" -- ${1+$ARGS} -emqx_data_dir "${DATA_DIR}"
+ # add ekka_proto_dist emqx_data_dir to boot command so it is visible from 'ps -ef'
+ # NTOE: order matters! emqx_data_dir has to be positioned at the end of the line to simplify the
+ # line parsing when file path contains spaces
+ exec "$@" -- ${1+$ARGS} -ekka_proto_dist "${EKKA_PROTO_DIST_MOD}" -emqx_data_dir "${DATA_DIR}"
;;
ctl)
diff --git a/build b/build
index 874e4088c..5c2bb556b 100755
--- a/build
+++ b/build
@@ -407,10 +407,10 @@ make_docker() {
PRODUCT_DESCRIPTION='Official docker image for EMQX Enterprise, an enterprise MQTT platform at scale. '
DOCUMENTATION_URL='https://docs.emqx.com/en/enterprise/latest/'
fi
- # shellcheck disable=SC2155
- local ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
- # shellcheck disable=SC2155
- local GIT_REVISION="$(git rev-parse HEAD)"
+ local ISO_8601_DATE GIT_REVISION
+ ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+ GIT_REVISION="$(git rev-parse HEAD)"
+ export BUILDX_NO_DEFAULT_ATTESTATIONS=1
local DOCKER_BUILDX_ARGS=(
--build-arg BUILD_FROM="${EMQX_BUILDER}" \
--build-arg RUN_FROM="${EMQX_RUNNER}" \
@@ -430,7 +430,6 @@ make_docker() {
--label org.opencontainers.image.licenses="${LICENSE}" \
--label org.opencontainers.image.otp.version="${EMQX_BUILDER_OTP}" \
--tag "${EMQX_IMAGE_TAG}" \
- --provenance false \
--pull
)
if [ "${DOCKER_BUILD_NOCACHE:-false}" = true ]; then
diff --git a/changes/ce/fix-11734.en.md b/changes/ce/fix-11734.en.md
new file mode 100644
index 000000000..576799ecf
--- /dev/null
+++ b/changes/ce/fix-11734.en.md
@@ -0,0 +1,3 @@
+Fix clustering in IPv6 network.
+
+Added new configurations `rpc.listen_address` and `rpc.ipv6_only` to allow EMQX cluster RPC server and client to use IPv6.
diff --git a/mix.exs b/mix.exs
index 6f727e911..07f6b0209 100644
--- a/mix.exs
+++ b/mix.exs
@@ -55,8 +55,8 @@ defmodule EMQXUmbrella.MixProject do
{:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true},
{:esockd, github: "emqx/esockd", tag: "5.9.7", override: true},
{:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true},
- {:ekka, github: "emqx/ekka", tag: "0.15.15", override: true},
- {:gen_rpc, github: "emqx/gen_rpc", tag: "3.1.1", override: true},
+ {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true},
+ {:gen_rpc, github: "emqx/gen_rpc", tag: "3.2.0", override: true},
{:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true},
{:minirest, github: "emqx/minirest", tag: "1.3.13", override: true},
{:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true},
diff --git a/rebar.config b/rebar.config
index 70341ef31..e2e1a7cf0 100644
--- a/rebar.config
+++ b/rebar.config
@@ -62,8 +62,8 @@
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}}
, {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}}
- , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.15"}}}
- , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.1"}}}
+ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}}
+ , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.0"}}}
, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.8"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.13"}}}
, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}}
diff --git a/rel/i18n/emqx_conf_schema.hocon b/rel/i18n/emqx_conf_schema.hocon
index e1f2120ca..fde0f7ff3 100644
--- a/rel/i18n/emqx_conf_schema.hocon
+++ b/rel/i18n/emqx_conf_schema.hocon
@@ -60,7 +60,9 @@ node_etc_dir.label:
cluster_proto_dist.desc:
"""The Erlang distribution protocol for the cluster.
- inet_tcp: IPv4 TCP
-- inet_tls: IPv4 TLS, works together with etc/ssl_dist.conf
"""
+- inet_tls: IPv4 TLS, works together with etc/ssl_dist.conf
+- inet6_tcp: IPv6 TCP
+- inet6_tls: IPv6 TLS, works together with etc/ssl_dist.conf
"""
cluster_proto_dist.label:
"""Cluster Protocol Distribution"""
@@ -192,6 +194,20 @@ rpc_insecure_fallback.desc:
rpc_insecure_fallback.label:
"""RPC insecure fallback"""
+rpc_listen_address.desc:
+"""Indicates the IP address for the RPC server to listen on. For example, use "0.0.0.0"
for IPv4 or "::"
for IPv6."""
+
+rpc_listen_address.label:
+"""RPC Listen IP Address"""
+
+rpc_ipv6_only.desc:
+"""This setting is effective only when rpc.listen_address
is assigned an IPv6 address.
+If set to true
, the RPC client will exclusively use IPv6 for connections.
+Otherwise, the client might opt for IPv4, even if the server is on IPv6."""
+
+rpc_ipv6_only.label:
+"""Use IPv6 Only"""
+
cluster_mcast_buffer.desc:
"""Size of the user-level buffer."""
diff --git a/scripts/test/start-two-nodes-in-docker.sh b/scripts/test/start-two-nodes-in-docker.sh
index 53689b207..a6fe4d522 100755
--- a/scripts/test/start-two-nodes-in-docker.sh
+++ b/scripts/test/start-two-nodes-in-docker.sh
@@ -16,6 +16,7 @@ NET='emqx.io'
NODE1="node1.$NET"
NODE2="node2.$NET"
COOKIE='this-is-a-secret'
+IPV6=0
cleanup() {
docker rm -f haproxy >/dev/null 2>&1 || true
@@ -24,31 +25,61 @@ cleanup() {
docker network rm "$NET" >/dev/null 2>&1 || true
}
-while getopts ":Pc" opt
+show_help() {
+ echo "Usage: $0 [options] EMQX_IMAGE1 [EMQX_IAMGE2]"
+ echo ""
+ echo "Specifiy which docker image to run with EMQX_IMAGE1"
+ echo "EMQX_IMAGE2 is the same as EMQX_IMAGE1 if not set"
+ echo ""
+ echo "Options:"
+ echo " -h, --help Show this help message and exit."
+ echo " -P Add -p options for docker run to expose more HAProxy container ports."
+ echo " -6 Test with IPv6"
+ echo " -c Cleanup: delete docker network, force delete the containers."
+}
+
+while getopts "hc6P:" opt
do
case $opt in
# -P option is treated similarly to docker run -P:
# publish ports to random available host ports
P) HAPROXY_PORTS=(-p 18083 -p 8883 -p 8084);;
c) cleanup; exit 0;;
+ h) show_help; exit 0;;
+ 6) IPV6=1;;
*) ;;
esac
done
shift $((OPTIND - 1))
-IMAGE1="${1}"
+IMAGE1="${1:-}"
IMAGE2="${2:-${IMAGE1}}"
+if [ -z "${IMAGE1:-}" ] || [ -z "${IMAGE2:-}" ]; then
+ show_help
+ exit 1
+fi
+
cleanup
-docker network create "$NET"
+if [ ${IPV6} = 1 ]; then
+ docker network create --ipv6 --subnet 2001:0DB8::/112 "$NET"
+ RPC_ADDRESS="::"
+ PROTO_DIST='inet6_tls'
+else
+ docker network create "$NET"
+ RPC_ADDRESS="0.0.0.0"
+ PROTO_DIST='inet_tls'
+fi
docker run -d -t --restart=always --name "$NODE1" \
--net "$NET" \
-e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \
-e EMQX_NODE_NAME="emqx@$NODE1" \
-e EMQX_NODE_COOKIE="$COOKIE" \
- -e EMQX_CLUSTER__PROTO_DIST='inet_tls' \
+ -e EMQX_CLUSTER__PROTO_DIST="${PROTO_DIST}" \
+ -e EMQX_RPC__LISTEN_ADDRESS="${RPC_ADDRESS}" \
+ -e EMQX_RPC__IPV6_ONLY="true" \
-e EMQX_listeners__ssl__default__enable=false \
-e EMQX_listeners__wss__default__enable=false \
-e EMQX_listeners__tcp__default__proxy_protocol=true \
@@ -60,7 +91,9 @@ docker run -d -t --restart=always --name "$NODE2" \
-e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \
-e EMQX_NODE_NAME="emqx@$NODE2" \
-e EMQX_NODE_COOKIE="$COOKIE" \
- -e EMQX_CLUSTER__PROTO_DIST='inet_tls' \
+ -e EMQX_CLUSTER__PROTO_DIST="${PROTO_DIST}" \
+ -e EMQX_RPC__LISTEN_ADDRESS="${RPC_ADDRESS}" \
+ -e EMQX_RPC__IPV6_ONLY="true" \
-e EMQX_listeners__ssl__default__enable=false \
-e EMQX_listeners__wss__default__enable=false \
-e EMQX_listeners__tcp__default__proxy_protocol=true \