chore(helm): support cluster discovery by dns

2021-12-07 14:30:16 +08:00 · 2021-12-07 14:30:16 +08:00 · e7c765aaa3
parent 26fa06b071
commit e7c765aaa3
6 changed files with 80 additions and 25 deletions
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@ -65,6 +65,12 @@ jobs:

    helm_test:
        runs-on: ubuntu-20.04
+        strategy:
+          fail-fast: false
+          matrix:
+            discovery:
+              - k8s
+              - dns

        steps:
        - uses: actions/checkout@v1
@ -104,17 +110,18 @@ jobs:
            sudo chmod 700 get_helm.sh
            sudo ./get_helm.sh
            helm version
-        - name: run emqx on chart
-          env:
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
-          timeout-minutes: 5
+        - name: setup emqx chart
          run: |
            sudo docker save ${TARGET}:${EMQX_TAG} -o emqx.tar.gz
            sudo k3s ctr image import emqx.tar.gz

            sed -i -r "s/^appVersion: .*$/appVersion: \"${EMQX_TAG}\"/g" deploy/charts/emqx/Chart.yaml
            sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml
-
+        - name: run emqx on chart
+          if: matrix.discovery == 'k8s'
+          env:
+            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
+          run: |
            helm install emqx \
                --set image.repository=${TARGET} \
                --set image.pullPolicy=Never \
@ -124,7 +131,29 @@ jobs:
                --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
                deploy/charts/emqx \
                --debug
-
+        - name: run emqx on chart
+          if: matrix.discovery == 'dns'
+          env:
+            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
+          run: |
+            helm install emqx \
+                --set emqxConfig.EMQX_CLUSTER__DISCOVERY="dns" \
+                --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \
+                --set emqxConfig.EMQX_CLUSTER__DNS__APP="emqx" \
+                --set emqxConfig.EMQX_CLUSTER__DNS__TYPE="srv" \
+                --set image.repository=${TARGET} \
+                --set image.pullPolicy=Never \
+                --set emqxAclConfig="" \
+                --set image.pullPolicy=Never \
+                --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \
+                --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \
+                deploy/charts/emqx \
+                --debug
+        - name: waiting emqx started
+          env:
+            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
+          timeout-minutes: 5
+          run: |
            while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \
              != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
              echo "==============================";
@ -133,6 +162,18 @@ jobs:
              echo "waiting emqx started";
              sleep 10;
            done
+        - name: Check ${{ matrix.kind[0]}} cluster
+          env:
+            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
+          timeout-minutes: 10
+          run: |
+            while
+              nodes="$(kubectl exec -i emqx-0 -- curl --silent --basic -u admin:public -X GET http://localhost:8081/api/v4/brokers | jq '.data|length')";
+              [ "$nodes" != "3" ];
+            do
+              echo "waiting emqx cluster scale"
+              sleep 1
+            done
        - name: get emqx-0 pods log
          if: failure()
          env:
--- a/deploy/charts/emqx/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx/templates/StatefulSet.yaml
@ -87,7 +87,9 @@ spec:
        secret:
          secretName: {{ .Values.emqxLicneseSecretName }}
      {{- end }}
+      {{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s"  }}
      serviceAccountName:  {{ include "emqx.fullname" . }}
+      {{- end }}
      {{- if .Values.podSecurityContext.enabled }}
      securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
      {{- end }}
@ -134,17 +136,6 @@ spec:
          envFrom:
            - configMapRef:
                name: {{ include "emqx.fullname" . }}-env
-          env:
-          - name: EMQX_NAME
-            value: {{ .Release.Name }}
-          - name: EMQX_CLUSTER__K8S__APP_NAME
-            value: {{ .Release.Name }}
-          - name: EMQX_CLUSTER__DISCOVERY
-            value: k8s
-          - name: EMQX_CLUSTER__K8S__SERVICE_NAME
-            value: {{ include "emqx.fullname" . }}-headless
-          - name: EMQX_CLUSTER__K8S__NAMESPACE
-            value: {{ .Release.Namespace }}
          resources:
 {{ toYaml .Values.resources | indent 12 }}
          volumeMounts:
--- a/deploy/charts/emqx/templates/configmap.yaml
+++ b/deploy/charts/emqx/templates/configmap.yaml
@ -10,7 +10,7 @@ metadata:
    app.kubernetes.io/managed-by: {{ .Release.Service }}
 data:
  {{- range $index, $value := .Values.emqxConfig}}
-    {{$index}}: "{{ $value }}" 
+    {{$index}}: "{{ tpl (printf "%v" $value) $ }}"
  {{- end}}

 ---
--- a/deploy/charts/emqx/templates/rbac.yaml
+++ b/deploy/charts/emqx/templates/rbac.yaml
@ -1,3 +1,4 @@
+{{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s"  }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@ -39,4 +40,5 @@ subjects:
 roleRef:
  kind: Role
  name: {{ include "emqx.fullname" . }}
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@ -50,7 +50,20 @@ initContainers: {}

 ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
 emqxConfig:
+  EMQX_NAME:  "{{ .Release.Name }}"
+
+  ## Cluster discovery by dns
+  # EMQX_CLUSTER__DISCOVERY: "dns"
+  # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
+  # EMQX_CLUSTER__DNS__APP: "{{ .Release.Name }}"
+  # EMQX_CLUSTER__DNS__TYPE: "srv"
+
+  ## Cluster discovery by k8s
+  EMQX_CLUSTER__DISCOVERY: "k8s"
+  EMQX_CLUSTER__K8S__APP_NAME: "{{ .Release.Name }}"
  EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
+  EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless"
+  EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}"
  ## The address type is used to extract host from k8s service.
  ## Value: ip | dns | hostname
  ## Note：Hostname is only supported after v4.0-rc.2
--- a/deploy/docker/docker-entrypoint.sh
+++ b/deploy/docker/docker-entrypoint.sh
@ -28,12 +28,20 @@ if [[ -z "$EMQX_NAME" ]]; then
 fi

 if [[ -z "$EMQX_HOST" ]]; then
-    if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
-        EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"}
-        EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX"
-    elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
-        EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'}
-        EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX")
+    if [[ "$EMQX_CLUSTER__DISCOVERY" == "dns" ]] && \
+        [[ "$EMQX_CLUSTER__DNS__TYPE" == "srv" ]] && \
+        grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then
+            EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME"
+    elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \
+        [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \
+        [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
+            EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"}
+            EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX"
+    elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \
+        [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \
+        [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
+            EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'}
+            EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX")
    else
        EMQX_HOST="$LOCAL_IP"
    fi