yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #11734 from zmstone/1002-support-ipv6-only-clustering 

Make IPv6 clustering options configurable
This commit is contained in:
Zaiming (Stone) Shi 2023-10-10 14:29:52 +02:00 committed by GitHub
parent 9b573834f9 d651764ff4
commit 0a9e1dfa88
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 114 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/emqx/rebar.config
View File

@ -28,8 +28,8 @@
{gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
{cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.15"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.1"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.0"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.16"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},

34
apps/emqx_conf/src/emqx_conf_schema.erl
View File

@ -195,7 +195,7 @@ fields("cluster") ->
)},
{"proto_dist",
sc(
hoconsc:enum([inet_tcp, inet6_tcp, inet_tls]),
hoconsc:enum([inet_tcp, inet6_tcp, inet_tls, inet6_tls]),
#{
mapping => "ekka.proto_dist",
default => inet_tcp,
@ -948,7 +948,26 @@ fields("rpc") ->
}
)},
{"ciphers", emqx_schema:ciphers_schema(tls_all_available)},
{"tls_versions", emqx_schema:tls_versions_schema(tls_all_available)}
{"tls_versions", emqx_schema:tls_versions_schema(tls_all_available)},
{"listen_address",
sc(
string(),
#{
default => "0.0.0.0",
desc => ?DESC(rpc_listen_address),
importance => ?IMPORTANCE_MEDIUM
}
)},
{"ipv6_only",
sc(
boolean(),
#{
default => false,
mapping => "gen_rpc.ipv6_only",
desc => ?DESC(rpc_ipv6_only),
importance => ?IMPORTANCE_LOW
}
)}
];
fields("log") ->
[
@ -1133,7 +1152,16 @@ translation("gen_rpc") ->
[
{"default_client_driver", fun tr_default_config_driver/1},
{"ssl_client_options", fun tr_gen_rpc_ssl_options/1},
{"ssl_server_options", fun tr_gen_rpc_ssl_options/1}
{"ssl_server_options", fun tr_gen_rpc_ssl_options/1},
{"socket_ip", fun(Conf) ->
Addr = conf_get("rpc.listen_address", Conf),
case inet:parse_address(Addr) of
{ok, Tuple} ->
Tuple;
{error, _Reason} ->
throw(#{bad_ip_address => Addr})
end
end}
];
translation("prometheus") ->
[

2
apps/emqx_rule_engine/src/emqx_rule_engine.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_rule_engine, [
{description, "EMQX Rule Engine"},
% strict semver, bump manually!
{vsn, "5.0.27"},
{vsn, "5.0.28"},
{modules, []},
{registered, [emqx_rule_engine_sup, emqx_rule_engine]},
{applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]},

8
apps/emqx_rule_engine/src/emqx_rule_funcs.erl
View File

@ -80,14 +80,18 @@
tanh/1
]).
%% Bits Funcs
%% Bitwise operations
-export([
bitnot/1,
bitand/2,
bitor/2,
bitxor/2,
bitsl/2,
bitsr/2,
bitsr/2
]).
%% binary and bitstring Funcs
-export([
bitsize/1,
bytesize/1,
subbits/2,

18
bin/emqx
View File

@ -522,17 +522,13 @@ else
## only one emqx node is running, get running args from 'ps -ef' output
tmp_nodename=$(echo -e "$PS_LINE" | $GREP -oE "\s-s?name.*" | awk '{print $2}' || true)
tmp_cookie=$(echo -e "$PS_LINE" | $GREP -oE "\s-setcookie.*" | awk '{print $2}' || true)
tmp_proto_dist=$(echo -e "$PS_LINE" | $GREP -oE '\s-ekka_proto_dist.*' | awk '{print $2}' || echo 'inet_tcp')
SSL_DIST_OPTFILE="$(echo -e "$PS_LINE" | $GREP -oE '\-ssl_dist_optfile\s.+\s' | awk '{print $2}' || true)"
tmp_ticktime="$(echo -e "$PS_LINE" | $GREP -oE '\s-kernel\snet_ticktime\s.+\s' | awk '{print $3}' || true)"
# data_dir is actually not needed, but kept anyway
tmp_datadir="$(echo -e "$PS_LINE" | $GREP -oE "\-emqx_data_dir.*" | sed -E 's#.+emqx_data_dir[[:blank:]]##g' | sed -E 's#[[:blank:]]--$##g' || true)"
if [ -z "$SSL_DIST_OPTFILE" ]; then
tmp_proto='inet_tcp'
else
tmp_proto='inet_tls'
fi
## Make the format like what call_hocon multi_get prints out, but only need 4 args
EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}"
EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto_dist}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}"
else
if [ "$RUNNING_NODES_COUNT" -gt 1 ]; then
if [ -z "${EMQX_NODE__NAME:-}" ]; then
@ -567,7 +563,7 @@ TICKTIME="$(get_boot_config 'node.dist_net_ticktime' || echo '120')"
# this environment variable is required by ekka_dist module
# because proto_dist is overriden to ekka, and there is a lack of ekka_tls module
export EKKA_PROTO_DIST_MOD="${PROTO_DIST:-inet_tcp}"
if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ]; then
if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ] || [ "$EKKA_PROTO_DIST_MOD" = 'inet6_tls' ]; then
if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
SSL_DIST_OPTFILE=${EMQX_SSL_DIST_OPTFILE:-"$EMQX_ETC_DIR/ssl_dist.conf"}
case "$SSL_DIST_OPTFILE" in
@ -1216,7 +1212,6 @@ case "${COMMAND}" in
export PROGNAME
# Store passed arguments since they will be erased by `set`
# add emqx_data_dir to boot command so it is visible from 'ps -ef'
ARGS="$*"
# shellcheck disable=SC2086
@ -1247,10 +1242,13 @@ case "${COMMAND}" in
fi
# Log the startup
logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS} -emqx_data_dir ${DATA_DIR}"
logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS} -ekka_proto_dist ${EKKA_PROTO_DIST_MOD} -emqx_data_dir ${DATA_DIR}"
# Start the VM
exec "$@" -- ${1+$ARGS} -emqx_data_dir "${DATA_DIR}"
# add ekka_proto_dist emqx_data_dir to boot command so it is visible from 'ps -ef'
# NTOE: order matters! emqx_data_dir has to be positioned at the end of the line to simplify the
# line parsing when file path contains spaces
exec "$@" -- ${1+$ARGS} -ekka_proto_dist "${EKKA_PROTO_DIST_MOD}" -emqx_data_dir "${DATA_DIR}"
;;
ctl)

9
build
View File

@ -407,10 +407,10 @@ make_docker() {
PRODUCT_DESCRIPTION='Official docker image for EMQX Enterprise, an enterprise MQTT platform at scale. '
DOCUMENTATION_URL='https://docs.emqx.com/en/enterprise/latest/'
fi
# shellcheck disable=SC2155
local ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
# shellcheck disable=SC2155
local GIT_REVISION="$(git rev-parse HEAD)"
local ISO_8601_DATE GIT_REVISION
ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
GIT_REVISION="$(git rev-parse HEAD)"
export BUILDX_NO_DEFAULT_ATTESTATIONS=1
local DOCKER_BUILDX_ARGS=(
--build-arg BUILD_FROM="${EMQX_BUILDER}" \
--build-arg RUN_FROM="${EMQX_RUNNER}" \
@ -430,7 +430,6 @@ make_docker() {
--label org.opencontainers.image.licenses="${LICENSE}" \
--label org.opencontainers.image.otp.version="${EMQX_BUILDER_OTP}" \
--tag "${EMQX_IMAGE_TAG}" \
--provenance false \
--pull
)
if [ "${DOCKER_BUILD_NOCACHE:-false}" = true ]; then

3
changes/ce/fix-11734.en.md Normal file
View File

@ -0,0 +1,3 @@
Fix clustering in IPv6 network.
Added new configurations `rpc.listen_address` and `rpc.ipv6_only` to allow EMQX cluster RPC server and client to use IPv6.

4
mix.exs
View File

@ -55,8 +55,8 @@ defmodule EMQXUmbrella.MixProject do
{:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true},
{:esockd, github: "emqx/esockd", tag: "5.9.7", override: true},
{:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true},
{:ekka, github: "emqx/ekka", tag: "0.15.15", override: true},
{:gen_rpc, github: "emqx/gen_rpc", tag: "3.1.1", override: true},
{:ekka, github: "emqx/ekka", tag: "0.15.16", override: true},
{:gen_rpc, github: "emqx/gen_rpc", tag: "3.2.0", override: true},
{:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true},
{:minirest, github: "emqx/minirest", tag: "1.3.13", override: true},
{:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true},

4
rebar.config
View File

@ -62,8 +62,8 @@
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}}
, {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.15"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.1"}}}
, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.0"}}}
, {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.8"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.13"}}}
, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}}

18
rel/i18n/emqx_conf_schema.hocon
View File

@ -60,7 +60,9 @@ node_etc_dir.label:
cluster_proto_dist.desc:
"""The Erlang distribution protocol for the cluster.<br/>
- inet_tcp: IPv4 TCP <br/>
- inet_tls: IPv4 TLS, works together with <code>etc/ssl_dist.conf</code>"""
- inet_tls: IPv4 TLS, works together with <code>etc/ssl_dist.conf</code> <br/>
- inet6_tcp: IPv6 TCP <br/>
- inet6_tls: IPv6 TLS, works together with <code>etc/ssl_dist.conf</code>"""
cluster_proto_dist.label:
"""Cluster Protocol Distribution"""
@ -192,6 +194,20 @@ rpc_insecure_fallback.desc:
rpc_insecure_fallback.label:
"""RPC insecure fallback"""
rpc_listen_address.desc:
"""Indicates the IP address for the RPC server to listen on. For example, use <code>"0.0.0.0"</code> for IPv4 or <code>"::"</code> for IPv6."""
rpc_listen_address.label:
"""RPC Listen IP Address"""
rpc_ipv6_only.desc:
"""This setting is effective only when <code>rpc.listen_address</code> is assigned an IPv6 address.
If set to <code>true</code>, the RPC client will exclusively use IPv6 for connections.
Otherwise, the client might opt for IPv4, even if the server is on IPv6."""
rpc_ipv6_only.label:
"""Use IPv6 Only"""
cluster_mcast_buffer.desc:
"""Size of the user-level buffer."""

43
scripts/test/start-two-nodes-in-docker.sh
View File

@ -16,6 +16,7 @@ NET='emqx.io'
NODE1="node1.$NET"
NODE2="node2.$NET"
COOKIE='this-is-a-secret'
IPV6=0
cleanup() {
docker rm -f haproxy >/dev/null 2>&1 || true
@ -24,31 +25,61 @@ cleanup() {
docker network rm "$NET" >/dev/null 2>&1 || true
}
while getopts ":Pc" opt
show_help() {
echo "Usage: $0 [options] EMQX_IMAGE1 [EMQX_IAMGE2]"
echo ""
echo "Specifiy which docker image to run with EMQX_IMAGE1"
echo "EMQX_IMAGE2 is the same as EMQX_IMAGE1 if not set"
echo ""
echo "Options:"
echo " -h, --help Show this help message and exit."
echo " -P Add -p options for docker run to expose more HAProxy container ports."
echo " -6 Test with IPv6"
echo " -c Cleanup: delete docker network, force delete the containers."
}
while getopts "hc6P:" opt
do
case $opt in
# -P option is treated similarly to docker run -P:
# publish ports to random available host ports
P) HAPROXY_PORTS=(-p 18083 -p 8883 -p 8084);;
c) cleanup; exit 0;;
h) show_help; exit 0;;
6) IPV6=1;;
*) ;;
esac
done
shift $((OPTIND - 1))
IMAGE1="${1}"
IMAGE1="${1:-}"
IMAGE2="${2:-${IMAGE1}}"
if [ -z "${IMAGE1:-}" ] || [ -z "${IMAGE2:-}" ]; then
show_help
exit 1
fi
cleanup
docker network create "$NET"
if [ ${IPV6} = 1 ]; then
docker network create --ipv6 --subnet 2001:0DB8::/112 "$NET"
RPC_ADDRESS="::"
PROTO_DIST='inet6_tls'
else
docker network create "$NET"
RPC_ADDRESS="0.0.0.0"
PROTO_DIST='inet_tls'
fi
docker run -d -t --restart=always --name "$NODE1" \
--net "$NET" \
-e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \
-e EMQX_NODE_NAME="emqx@$NODE1" \
-e EMQX_NODE_COOKIE="$COOKIE" \
-e EMQX_CLUSTER__PROTO_DIST='inet_tls' \
-e EMQX_CLUSTER__PROTO_DIST="${PROTO_DIST}" \
-e EMQX_RPC__LISTEN_ADDRESS="${RPC_ADDRESS}" \
-e EMQX_RPC__IPV6_ONLY="true" \
-e EMQX_listeners__ssl__default__enable=false \
-e EMQX_listeners__wss__default__enable=false \
-e EMQX_listeners__tcp__default__proxy_protocol=true \
@ -60,7 +91,9 @@ docker run -d -t --restart=always --name "$NODE2" \
-e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \
-e EMQX_NODE_NAME="emqx@$NODE2" \
-e EMQX_NODE_COOKIE="$COOKIE" \
-e EMQX_CLUSTER__PROTO_DIST='inet_tls' \
-e EMQX_CLUSTER__PROTO_DIST="${PROTO_DIST}" \
-e EMQX_RPC__LISTEN_ADDRESS="${RPC_ADDRESS}" \
-e EMQX_RPC__IPV6_ONLY="true" \
-e EMQX_listeners__ssl__default__enable=false \
-e EMQX_listeners__wss__default__enable=false \
-e EMQX_listeners__tcp__default__proxy_protocol=true \