From 780bda2803c7dd67a8efc068bb52f461dc308f75 Mon Sep 17 00:00:00 2001 From: Rory Z Date: Thu, 16 Jun 2022 15:52:37 +0800 Subject: [PATCH] feat(helm): support dns cluster --- .github/workflows/run_fvt_tests.yaml | 52 +++++++++++++++---- deploy/charts/emqx/templates/StatefulSet.yaml | 11 +--- deploy/charts/emqx/templates/configmap.yaml | 11 ++-- deploy/charts/emqx/templates/rbac.yaml | 2 + deploy/charts/emqx/values.yaml | 6 +++ 5 files changed, 61 insertions(+), 21 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index d5fbc32b6..2728f22d8 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -126,16 +126,19 @@ jobs: strategy: fail-fast: false matrix: + discovery: + - k8s + - dns profile: - emqx os: - - ["debian11", "debian:11-slim"] + - ["debian11", "debian:11-slim"] otp: - - 24.2.1-1 + - 24.2.1-1 elixir: - - 1.13.4 + - 1.13.4 arch: - - amd64 + - amd64 # - emqx-enterprise # TODO test enterprise steps: @@ -171,14 +174,15 @@ jobs: echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - run: minikube start - - name: run emqx on chart - timeout-minutes: 5 + - name: setup helm chart working-directory: source run: | minikube image load $TARGET:$EMQX_TAG - sed -i -r "s/^appVersion: .*$/appVersion: \"$EMQX_TAG\"/g" deploy/charts/emqx/Chart.yaml - + - name: run emqx on chart + working-directory: source + if: matrix.discovery == 'k8s' + run: | helm install emqx \ --set image.repository=$TARGET \ --set image.pullPolicy=Never \ @@ -190,7 +194,27 @@ jobs: --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ deploy/charts/emqx \ --debug - + - name: run emqx on chart + working-directory: source + if: matrix.discovery == 'dns' + run: | + helm install emqx \ + --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \ + --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \ + --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \ + --set image.repository=$TARGET \ + --set image.pullPolicy=Never \ + --set emqxAclConfig="" \ + --set image.pullPolicy=Never \ + --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ + --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ + --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ + deploy/charts/emqx \ + --debug + - name: waiting emqx started + timeout-minutes: 10 + run: | while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \ != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do echo "=============================="; @@ -199,6 +223,16 @@ jobs: echo "waiting emqx started"; sleep 10; done + - name: Check cluster + timeout-minutes: 10 + run: | + kubectl port-forward svc/emqx 18083:18083 & + while + [ "$(curl --silent --basic -u admin:public -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ]; + do + echo "waiting emqx cluster scale" + sleep 1 + done - name: get emqx-0 pods log if: failure() run: | diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 5138cf40f..9710db75e 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -65,7 +65,9 @@ spec: secret: secretName: {{ .Values.emqxLicenseSecretName }} {{- end }} + {{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} serviceAccountName: {{ include "emqx.fullname" . }} + {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} @@ -114,15 +116,6 @@ spec: - secretRef: name: {{ .Values.envFromSecret }} {{- end }} - env: - - name: EMQX_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__DISCOVERY_STRATEGY - value: k8s - - name: EMQX_CLUSTER__K8S__SERVICE_NAME - value: {{ include "emqx.fullname" . }}-headless - - name: EMQX_CLUSTER__K8S__NAMESPACE - value: {{ .Release.Namespace }} resources: {{ toYaml .Values.resources | indent 12 }} volumeMounts: diff --git a/deploy/charts/emqx/templates/configmap.yaml b/deploy/charts/emqx/templates/configmap.yaml index ffd1b66dc..e0563d02a 100644 --- a/deploy/charts/emqx/templates/configmap.yaml +++ b/deploy/charts/emqx/templates/configmap.yaml @@ -1,3 +1,4 @@ +{{- if .Values.emqxConfig }} apiVersion: v1 kind: ConfigMap metadata: @@ -9,6 +10,10 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} data: - {{- range $index, $value := .Values.emqxConfig}} - {{$index}}: "{{ $value }}" - {{- end}} + {{- range $index, $value := .Values.emqxConfig }} + {{- if $value }} + {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }} + {{ print "EMQX_" $key }}: "{{ tpl (printf "%v" $value) $ }}" + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/charts/emqx/templates/rbac.yaml b/deploy/charts/emqx/templates/rbac.yaml index d2a0f35f4..f2cdd3601 100644 --- a/deploy/charts/emqx/templates/rbac.yaml +++ b/deploy/charts/emqx/templates/rbac.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} apiVersion: v1 kind: ServiceAccount metadata: @@ -40,3 +41,4 @@ roleRef: kind: Role name: {{ include "emqx.fullname" . }} apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 72c98c3fd..fb99a4c64 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -89,7 +89,13 @@ initContainers: {} ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) emqxConfig: + EMQX_CLUSTER__DISCOVERY_STRATEGY: "k8s" + # EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns" + # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local" + # EMQX_CLUSTER__DNS__RECORD_TYPE: "srv" EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" + EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless" + EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}" ## The address type is used to extract host from k8s service. ## Value: ip | dns | hostname ## Noteļ¼šHostname is only supported after v4.0-rc.2