Merge pull request #8229 from zmstone/0615-delete-appname-for-k8s-and-dns-discovery

Delete app name config for k8s and dns discovery
2022-06-16 07:29:02 +01:00 · 2022-06-16 07:29:02 +01:00 · d0709798e7
parent 5287dc322e 67b8e6c3bd
commit d0709798e7
8 changed files with 96 additions and 71 deletions
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@ -27,7 +27,7 @@
    {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
    {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
    {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}},
-    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.9"}}},
+    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.0"}}},
    {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.2"}}},
    {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
--- a/apps/emqx_conf/i18n/emqx_conf_schema.conf
+++ b/apps/emqx_conf/i18n/emqx_conf_schema.conf
@ -245,10 +245,11 @@ Note: probe messages are broadcast to all the specified ports.

  cluster_dns_name {
    desc {
-      en: """The domain name of the EMQX cluster."""
-      zh: """指定 DNS A 记录的名字。emqx 会通过访问这个 DNS A 记录来获取 IP 地址列表，
-然后拼接 cluster.dns.app 里指定的 APP 名得到集群中所有节点的列表。
-当 cluster.discovery_strategy 为 dns 时，此配置项才有效。
+      en: """The domain name from which to discover peer EMQX nodes' IP addresses.
+Applicable when <code>cluster.discovery_strategy = dns</code>
+"""
+      zh: """指定 DNS A 记录的名字。emqx 会通过访问这个 DNS A 记录来获取 IP 地址列表。
+当<code>cluster.discovery_strategy</code> 为 <code>dns</code> 时有效。
 """
    }
    label {
@ -257,19 +258,6 @@ Note: probe messages are broadcast to all the specified ports.
    }
  }

-  cluster_dns_app {
-    desc {
-      en: """The symbolic name of the EMQX service."""
-      zh: """用来与从 cluster.dns.name 获取的 IP 列表拼接得到节点名列表。
-当 cluster.discovery_strategy 为 dns 时，此配置项才有效。
-         """
-    }
-    label {
-      en: "Cluster DNS App"
-      zh: "DNS前缀标识"
-    }
-  }
-
  cluster_etcd_server {
    desc {
      en: """List of endpoint URLs of the etcd cluster"""
@ -353,16 +341,13 @@ It is refreshed automatically, as long as the node is alive.

  cluster_k8s_address_type {
    desc {
-      en: """Address type used for connecting to the discovered nodes."""
+      en: """Address type used for connecting to the discovered nodes.
+Setting <code>cluster.k8s.address_type</code> to <code>ip</code> will
+make EMQX to discover IP addresses of peer nodes from Kubernetes API.
+"""
      zh: """当使用 k8s 方式集群时，address_type 用来从 Kubernetes 接口的应答里获取什么形式的 Host 列表。
-指定 cluster.k8s.address_type 为 ip，则将从 Kubernetes 接口中获取 emqx 服务的 IP 地址列表:
- 172.16.122.31 </br>
- 172.16.122.32 </br>
- 172.16.122.33 </br>
-然后与 cluster.k8s.app_name 配置指定的 app name 拼接，得到 emqx 节点列表: </br>
- emqx@172.16.122.31
- emqx@172.16.122.32
- emqx@172.16.122.33
+指定 <code>cluster.k8s.address_type</code. 为 <code>ip</code>，则将从 Kubernetes 接口中获取集群中其他节点
+的IP地址。
 """
    }
    label {
@ -371,22 +356,6 @@ It is refreshed automatically, as long as the node is alive.
    }
  }

-  cluster_k8s_app_name {
-    desc {
-      en: """This parameter should be set to the part of the <code>node.name</code>
-before the '@'.</br>
-For example, if the <code>node.name</code> is <code>emqx@127.0.0.1</code>, then this parameter
-should be set to <code>emqx</code>."""
-      zh: """app_name 用来跟获取的 Host 列表拼接，得到节点列表。</br>
-当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。
-          """
-    }
-    label {
-      en: "K8s App Name"
-      zh: "K8s 节点名前缀"
-    }
-  }
-
  cluster_k8s_namespace {
    desc {
      en: """Kubernetes namespace."""
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@ -287,15 +287,6 @@ fields(cluster_dns) ->
                    desc => ?DESC(cluster_dns_name),
                    'readOnly' => true
                }
-            )},
-        {"app",
-            sc(
-                string(),
-                #{
-                    default => "emqx",
-                    desc => ?DESC(cluster_dns_app),
-                    'readOnly' => true
-                }
            )}
    ];
 fields(cluster_etcd) ->
@ -364,15 +355,6 @@ fields(cluster_k8s) ->
                    'readOnly' => true
                }
            )},
-        {"app_name",
-            sc(
-                string(),
-                #{
-                    default => "emqx",
-                    'readOnly' => true,
-                    desc => ?DESC(cluster_k8s_app_name)
-                }
-            )},
        {"namespace",
            sc(
                string(),
@ -1300,8 +1282,7 @@ options(mcast, Conf) ->
    ];
 options(dns, Conf) ->
    [
-        {name, conf_get("cluster.dns.name", Conf)},
-        {app, conf_get("cluster.dns.app", Conf)}
+        {name, conf_get("cluster.dns.name", Conf)}
    ];
 options(etcd, Conf) ->
    Namespace = "cluster.etcd.ssl",
@ -1320,7 +1301,6 @@ options(k8s, Conf) ->
        {apiserver, conf_get("cluster.k8s.apiserver", Conf)},
        {service_name, conf_get("cluster.k8s.service_name", Conf)},
        {address_type, conf_get("cluster.k8s.address_type", Conf, ip)},
-        {app_name, conf_get("cluster.k8s.app_name", Conf)},
        {namespace, conf_get("cluster.k8s.namespace", Conf)},
        {suffix, conf_get("cluster.k8s.suffix", Conf, "")}
    ];
--- a/deploy/charts/emqx/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx/templates/StatefulSet.yaml
@ -117,8 +117,6 @@ spec:
          env:
          - name: EMQX_NAME
            value: {{ .Release.Name }}
-          - name: EMQX_CLUSTER__K8S__APP_NAME
-            value: {{ .Release.Name }}
          - name: EMQX_CLUSTER__DISCOVERY_STRATEGY
            value: k8s
          - name: EMQX_CLUSTER__K8S__SERVICE_NAME
--- a/mix.exs
+++ b/mix.exs
@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do
      {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
      {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
      {:esockd, github: "emqx/esockd", tag: "5.9.3", override: true},
-      {:ekka, github: "emqx/ekka", tag: "0.12.9", override: true},
+      {:ekka, github: "emqx/ekka", tag: "0.13.0", override: true},
      {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
      {:minirest, github: "emqx/minirest", tag: "1.3.4", override: true},
      {:ecpool, github: "emqx/ecpool", tag: "0.5.2"},
--- a/rebar.config
+++ b/rebar.config
@ -54,7 +54,7 @@
    , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
    , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}
-    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.9"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.0"}}}
    , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.4"}}}
    , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
--- a/scripts/make-docker-image-from-host-build.sh
+++ b/scripts/make-docker-image-from-host-build.sh
@ -22,7 +22,8 @@ if [ "$COMPILE" = '--compile' ]; then
    sync
 fi

-export DOCKER_BUILDKIT=1
+# cannot enable DOCKER_BUILDKIT because the COPY often gets stale layers
+#export DOCKER_BUILDKIT=1
 docker build --build-arg PROFILE="${PROFILE}" \
    -t "emqx/emqx:${PKG_VSN}-${DISTRO}" \
    -f "$EMQX_DOCKERFILE" .
--- a/scripts/test-node-discovery-dns.sh
+++ b/scripts/test-node-discovery-dns.sh
@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+## Test two nodes-cluster discover each other using DNS A records lookup result.
+
+set -euo pipefail
+
+cd -P -- "$(dirname -- "$0")/.."
+
+IMAGE="${1}"
+
+NET='test_node_discovery_dns'
+NODE1='emqx1'
+NODE2='emqx2'
+COOKIE='this-is-a-secret'
+
+# cleanup
+docker rm -f dnsmasq >/dev/null 2>&1 || true
+docker rm -f "$NODE1" >/dev/null 2>&1 || true
+docker rm -f "$NODE2" >/dev/null 2>&1 || true
+docker network rm "$NET" >/dev/null 2>&1 || true
+
+docker network create --subnet=172.18.0.0/16 $NET
+
+IP0="172.18.0.100"
+IP1="172.18.0.101"
+IP2="172.18.0.102"
+DOMAIN="dnstest.mynet"
+
+# create configs for dnsmasq
+cat <<-EOF > "/tmp/dnsmasq.conf"
+conf-dir=/etc/dnsmasq,*.conf
+addn-hosts=/etc/hosts.$DOMAIN
+EOF
+
+cat <<-EOF > "/tmp/hosts.$DOMAIN"
+$IP1 $DOMAIN
+$IP2 $DOMAIN
+EOF
+
+cat <<-EOF > /tmp/dnsmasq.base.conf
+domain-needed
+bogus-priv
+no-hosts
+keep-in-foreground
+no-resolv
+expand-hosts
+server=8.8.8.8
+EOF
+
+docker run -d -t --name dnsmasq \
+    --net "$NET" \
+    --ip "$IP0" \
+    -v /tmp/dnsmasq.conf:/etc/dnsmasq.conf \
+    -v "/tmp/hosts.$DOMAIN:/etc/hosts.$DOMAIN" \
+    -v "/tmp/dnsmasq.base.conf:/etc/dnsmasq/0.base.conf" \
+    --cap-add=NET_ADMIN \
+    storytel/dnsmasq dnsmasq --no-daemon --log-queries
+
+start_emqx() {
+    NAME="$1"
+    IP="$2"
+    DASHBOARD_PORT="$3"
+    docker run -d -t \
+        --name "$NAME" \
+        --net "$NET" \
+        --ip "$IP" \
+        --dns "$IP0" \
+        -p "$DASHBOARD_PORT:18083" \
+        -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \
+        -e EMQX_NODE_COOKIE="$COOKIE" \
+        -e EMQX_cluster__discovery_strategy='dns' \
+        -e EMQX_cluster__dns__name="$DOMAIN" \
+        "$IMAGE"
+}
+
+start_emqx "$NODE1" "$IP1" 18083
+start_emqx "$NODE2" "$IP2" 18084