From d635d08c7af913fd5a06a6d4ff1efc45608c0595 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 15 Jun 2022 09:04:38 +0200 Subject: [PATCH 1/4] chore: pin ekka 0.13.0 ekka-0.13.0 has the dns.app and k8s.app_name configs made optional the node names of peer nodes must be the same as the connecting node --- mix.exs | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index a2d6f6322..29286973c 100644 --- a/mix.exs +++ b/mix.exs @@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.3", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.12.9", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.13.0", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.4", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, diff --git a/rebar.config b/rebar.config index 709c2a039..daa45abe7 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.9"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.0"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.4"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} From 05782c4e4e18ef9dc62c896a5dcd4c0b50fc3afa Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 15 Jun 2022 09:12:55 +0200 Subject: [PATCH 2/4] refactor: delete k8s.app_name config --- apps/emqx_conf/i18n/emqx_conf_schema.conf | 33 ++++--------------- apps/emqx_conf/src/emqx_conf_schema.erl | 10 ------ deploy/charts/emqx/templates/StatefulSet.yaml | 2 -- 3 files changed, 7 insertions(+), 38 deletions(-) diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index 8c02a4912..edef14a9c 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -353,17 +353,14 @@ It is refreshed automatically, as long as the node is alive. cluster_k8s_address_type { desc { - en: """Address type used for connecting to the discovered nodes.""" + en: """Address type used for connecting to the discovered nodes. +setting cluster.k8s.address_type to ip will +make EMQX to discover IP addresses of peer nodes from Kubernetes API. +""" zh: """当使用 k8s 方式集群时，address_type 用来从 Kubernetes 接口的应答里获取什么形式的 Host 列表。 -指定 cluster.k8s.address_type 为 ip，则将从 Kubernetes 接口中获取 emqx 服务的 IP 地址列表: -- 172.16.122.31
-- 172.16.122.32
-- 172.16.122.33
-然后与 cluster.k8s.app_name 配置指定的 app name 拼接，得到 emqx 节点列表:
-- emqx@172.16.122.31 -- emqx@172.16.122.32 -- emqx@172.16.122.33 - """ +指定 cluster.k8s.address_typeip，则将从 Kubernetes 接口中获取集群中其他节点 +的IP地址。 +""" } label { en: "K8s Address Type" @@ -371,22 +368,6 @@ It is refreshed automatically, as long as the node is alive. } } - cluster_k8s_app_name { - desc { - en: """This parameter should be set to the part of the node.name -before the '@'.
-For example, if the node.name is emqx@127.0.0.1, then this parameter -should be set to emqx.""" - zh: """app_name 用来跟获取的 Host 列表拼接，得到节点列表。
-当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。 - """ - } - label { - en: "K8s App Name" - zh: "K8s 节点名前缀" - } - } - cluster_k8s_namespace { desc { en: """Kubernetes namespace.""" diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index c7fdc3b98..ce4d112ac 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -364,15 +364,6 @@ fields(cluster_k8s) -> 'readOnly' => true } )}, - {"app_name", - sc( - string(), - #{ - default => "emqx", - 'readOnly' => true, - desc => ?DESC(cluster_k8s_app_name) - } - )}, {"namespace", sc( string(), @@ -1320,7 +1311,6 @@ options(k8s, Conf) -> {apiserver, conf_get("cluster.k8s.apiserver", Conf)}, {service_name, conf_get("cluster.k8s.service_name", Conf)}, {address_type, conf_get("cluster.k8s.address_type", Conf, ip)}, - {app_name, conf_get("cluster.k8s.app_name", Conf)}, {namespace, conf_get("cluster.k8s.namespace", Conf)}, {suffix, conf_get("cluster.k8s.suffix", Conf, "")} ]; diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 02c2ec02c..5138cf40f 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -117,8 +117,6 @@ spec: env: - name: EMQX_NAME value: {{ .Release.Name }} - - name: EMQX_CLUSTER__K8S__APP_NAME - value: {{ .Release.Name }} - name: EMQX_CLUSTER__DISCOVERY_STRATEGY value: k8s - name: EMQX_CLUSTER__K8S__SERVICE_NAME From d35b95d00ff094839416add58af3dcb1df0297fb Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 15 Jun 2022 09:25:09 +0200 Subject: [PATCH 3/4] refactor: remove dns.app config --- apps/emqx_conf/i18n/emqx_conf_schema.conf | 24 ++++++----------------- apps/emqx_conf/src/emqx_conf_schema.erl | 12 +----------- 2 files changed, 7 insertions(+), 29 deletions(-) diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index edef14a9c..d0d08cb55 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -245,11 +245,12 @@ Note: probe messages are broadcast to all the specified ports. cluster_dns_name { desc { - en: """The domain name of the EMQX cluster.""" - zh: """指定 DNS A 记录的名字。emqx 会通过访问这个 DNS A 记录来获取 IP 地址列表， -然后拼接 cluster.dns.app 里指定的 APP 名得到集群中所有节点的列表。 -当 cluster.discovery_strategy 为 dns 时，此配置项才有效。 - """ + en: """The domain name from which to discover peer EMQX nodes' IP addresses. +Applicable when cluster.discovery_strategy = dns +""" + zh: """指定 DNS A 记录的名字。emqx 会通过访问这个 DNS A 记录来获取 IP 地址列表。 +当cluster.discovery_strategy 为 dns 时有效。 +""" } label { en: "Cluster Dns Name" @@ -257,19 +258,6 @@ Note: probe messages are broadcast to all the specified ports. } } - cluster_dns_app { - desc { - en: """The symbolic name of the EMQX service.""" - zh: """用来与从 cluster.dns.name 获取的 IP 列表拼接得到节点名列表。 -当 cluster.discovery_strategy 为 dns 时，此配置项才有效。 - """ - } - label { - en: "Cluster DNS App" - zh: "DNS前缀标识" - } - } - cluster_etcd_server { desc { en: """List of endpoint URLs of the etcd cluster""" diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index ce4d112ac..7960eba7f 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -288,15 +288,6 @@ fields(cluster_dns) -> 'readOnly' => true } )}, - {"app", - sc( - string(), - #{ - default => "emqx", - desc => ?DESC(cluster_dns_app), - 'readOnly' => true - } - )} ]; fields(cluster_etcd) -> [ @@ -1291,8 +1282,7 @@ options(mcast, Conf) -> ]; options(dns, Conf) -> [ - {name, conf_get("cluster.dns.name", Conf)}, - {app, conf_get("cluster.dns.app", Conf)} + {name, conf_get("cluster.dns.name", Conf)} ]; options(etcd, Conf) -> Namespace = "cluster.etcd.ssl", From 67b8e6c3bdb6271cb087af9465d7c088e1c8c4b4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 15 Jun 2022 23:57:24 +0200 Subject: [PATCH 4/4] test: add a script to test dns cluster node discovery --- apps/emqx/rebar.config | 2 +- apps/emqx_conf/i18n/emqx_conf_schema.conf | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 2 +- scripts/make-docker-image-from-host-build.sh | 3 +- scripts/test-node-discovery-dns.sh | 77 ++++++++++++++++++++ 5 files changed, 82 insertions(+), 4 deletions(-) create mode 100755 scripts/test-node-discovery-dns.sh diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index bf85ed511..79fda9cc5 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.9"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.0"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/apps/emqx_conf/i18n/emqx_conf_schema.conf b/apps/emqx_conf/i18n/emqx_conf_schema.conf index d0d08cb55..a7dc12d90 100644 --- a/apps/emqx_conf/i18n/emqx_conf_schema.conf +++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf @@ -342,7 +342,7 @@ It is refreshed automatically, as long as the node is alive. cluster_k8s_address_type { desc { en: """Address type used for connecting to the discovered nodes. -setting cluster.k8s.address_type to ip will +Setting cluster.k8s.address_type to ip will make EMQX to discover IP addresses of peer nodes from Kubernetes API. """ zh: """当使用 k8s 方式集群时，address_type 用来从 Kubernetes 接口的应答里获取什么形式的 Host 列表。 diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 7960eba7f..3acf4cb45 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -287,7 +287,7 @@ fields(cluster_dns) -> desc => ?DESC(cluster_dns_name), 'readOnly' => true } - )}, + )} ]; fields(cluster_etcd) -> [ diff --git a/scripts/make-docker-image-from-host-build.sh b/scripts/make-docker-image-from-host-build.sh index 8911ed251..1791013f4 100755 --- a/scripts/make-docker-image-from-host-build.sh +++ b/scripts/make-docker-image-from-host-build.sh @@ -22,7 +22,8 @@ if [ "$COMPILE" = '--compile' ]; then sync fi -export DOCKER_BUILDKIT=1 +# cannot enable DOCKER_BUILDKIT because the COPY often gets stale layers +#export DOCKER_BUILDKIT=1 docker build --build-arg PROFILE="${PROFILE}" \ -t "emqx/emqx:${PKG_VSN}-${DISTRO}" \ -f "$EMQX_DOCKERFILE" . diff --git a/scripts/test-node-discovery-dns.sh b/scripts/test-node-discovery-dns.sh new file mode 100755 index 000000000..09701dab6 --- /dev/null +++ b/scripts/test-node-discovery-dns.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +## Test two nodes-cluster discover each other using DNS A records lookup result. + +set -euo pipefail + +cd -P -- "$(dirname -- "$0")/.." + +IMAGE="${1}" + +NET='test_node_discovery_dns' +NODE1='emqx1' +NODE2='emqx2' +COOKIE='this-is-a-secret' + +# cleanup +docker rm -f dnsmasq >/dev/null 2>&1 || true +docker rm -f "$NODE1" >/dev/null 2>&1 || true +docker rm -f "$NODE2" >/dev/null 2>&1 || true +docker network rm "$NET" >/dev/null 2>&1 || true + +docker network create --subnet=172.18.0.0/16 $NET + +IP0="172.18.0.100" +IP1="172.18.0.101" +IP2="172.18.0.102" +DOMAIN="dnstest.mynet" + +# create configs for dnsmasq +cat <<-EOF > "/tmp/dnsmasq.conf" +conf-dir=/etc/dnsmasq,*.conf +addn-hosts=/etc/hosts.$DOMAIN +EOF + +cat <<-EOF > "/tmp/hosts.$DOMAIN" +$IP1 $DOMAIN +$IP2 $DOMAIN +EOF + +cat <<-EOF > /tmp/dnsmasq.base.conf +domain-needed +bogus-priv +no-hosts +keep-in-foreground +no-resolv +expand-hosts +server=8.8.8.8 +EOF + +docker run -d -t --name dnsmasq \ + --net "$NET" \ + --ip "$IP0" \ + -v /tmp/dnsmasq.conf:/etc/dnsmasq.conf \ + -v "/tmp/hosts.$DOMAIN:/etc/hosts.$DOMAIN" \ + -v "/tmp/dnsmasq.base.conf:/etc/dnsmasq/0.base.conf" \ + --cap-add=NET_ADMIN \ + storytel/dnsmasq dnsmasq --no-daemon --log-queries + +start_emqx() { + NAME="$1" + IP="$2" + DASHBOARD_PORT="$3" + docker run -d -t \ + --name "$NAME" \ + --net "$NET" \ + --ip "$IP" \ + --dns "$IP0" \ + -p "$DASHBOARD_PORT:18083" \ + -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \ + -e EMQX_NODE_COOKIE="$COOKIE" \ + -e EMQX_cluster__discovery_strategy='dns' \ + -e EMQX_cluster__dns__name="$DOMAIN" \ + "$IMAGE" +} + +start_emqx "$NODE1" "$IP1" 18083 +start_emqx "$NODE2" "$IP2" 18084