From 67b8e6c3bdb6271cb087af9465d7c088e1c8c4b4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 15 Jun 2022 23:57:24 +0200 Subject: [PATCH] test: add a script to test dns cluster node discovery --- apps/emqx/rebar.config | 2 +- apps/emqx_conf/i18n/emqx_conf_schema.conf | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- scripts/make-docker-image-from-host-build.sh | 3 +- scripts/test-node-discovery-dns.sh | 77 ++++++++++++++++++++ 5 files changed, 82 insertions(+), 4 deletions(-) create mode 100755 scripts/test-node-discovery-dns.sh diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index bf85ed511..79fda9cc5 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.9"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.0"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index d0d08cb55..a7dc12d90 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -342,7 +342,7 @@ It is refreshed automatically, as long as the node is alive. cluster_k8s_address_type { desc { en: """Address type used for connecting to the discovered nodes. -setting cluster.k8s.address_type to ip will +Setting cluster.k8s.address_type to ip will make EMQX to discover IP addresses of peer nodes from Kubernetes API. """ zh: """当使用 k8s 方式集群时，address_type 用来从 Kubernetes 接口的应答里获取什么形式的 Host 列表。 diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 7960eba7f..3acf4cb45 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -287,7 +287,7 @@ fields(cluster_dns) -> desc => ?DESC(cluster_dns_name), 'readOnly' => true } - )}, + )} ]; fields(cluster_etcd) -> [ diff --git a/scripts/make-docker-image-from-host-build.sh b/scripts/make-docker-image-from-host-build.sh index 8911ed251..1791013f4 100755 --- a/scripts/make-docker-image-from-host-build.sh +++ b/scripts/make-docker-image-from-host-build.sh @@ -22,7 +22,8 @@ if [ "$COMPILE" = '--compile' ]; then sync fi -export DOCKER_BUILDKIT=1 +# cannot enable DOCKER_BUILDKIT because the COPY often gets stale layers +#export DOCKER_BUILDKIT=1 docker build --build-arg PROFILE="${PROFILE}" \ -t "emqx/emqx:${PKG_VSN}-${DISTRO}" \ -f "$EMQX_DOCKERFILE" . diff --git a/scripts/test-node-discovery-dns.sh b/scripts/test-node-discovery-dns.sh new file mode 100755 index 000000000..09701dab6 --- /dev/null +++ b/scripts/test-node-discovery-dns.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +## Test two nodes-cluster discover each other using DNS A records lookup result. + +set -euo pipefail + +cd -P -- "$(dirname -- "$0")/.." + +IMAGE="${1}" + +NET='test_node_discovery_dns' +NODE1='emqx1' +NODE2='emqx2' +COOKIE='this-is-a-secret' + +# cleanup +docker rm -f dnsmasq >/dev/null 2>&1 || true +docker rm -f "$NODE1" >/dev/null 2>&1 || true +docker rm -f "$NODE2" >/dev/null 2>&1 || true +docker network rm "$NET" >/dev/null 2>&1 || true + +docker network create --subnet=172.18.0.0/16 $NET + +IP0="172.18.0.100" +IP1="172.18.0.101" +IP2="172.18.0.102" +DOMAIN="dnstest.mynet" + +# create configs for dnsmasq +cat <<-EOF > "/tmp/dnsmasq.conf" +conf-dir=/etc/dnsmasq,*.conf +addn-hosts=/etc/hosts.$DOMAIN +EOF + +cat <<-EOF > "/tmp/hosts.$DOMAIN" +$IP1 $DOMAIN +$IP2 $DOMAIN +EOF + +cat <<-EOF > /tmp/dnsmasq.base.conf +domain-needed +bogus-priv +no-hosts +keep-in-foreground +no-resolv +expand-hosts +server=8.8.8.8 +EOF + +docker run -d -t --name dnsmasq \ + --net "$NET" \ + --ip "$IP0" \ + -v /tmp/dnsmasq.conf:/etc/dnsmasq.conf \ + -v "/tmp/hosts.$DOMAIN:/etc/hosts.$DOMAIN" \ + -v "/tmp/dnsmasq.base.conf:/etc/dnsmasq/0.base.conf" \ + --cap-add=NET_ADMIN \ + storytel/dnsmasq dnsmasq --no-daemon --log-queries + +start_emqx() { + NAME="$1" + IP="$2" + DASHBOARD_PORT="$3" + docker run -d -t \ + --name "$NAME" \ + --net "$NET" \ + --ip "$IP" \ + --dns "$IP0" \ + -p "$DASHBOARD_PORT:18083" \ + -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \ + -e EMQX_NODE_COOKIE="$COOKIE" \ + -e EMQX_cluster__discovery_strategy='dns' \ + -e EMQX_cluster__dns__name="$DOMAIN" \ + "$IMAGE" +} + +start_emqx "$NODE1" "$IP1" 18083 +start_emqx "$NODE2" "$IP2" 18084