diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index a16586bd9..772fd4fa2 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -65,6 +65,12 @@ jobs: helm_test: runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + discovery: + - k8s + - dns steps: - uses: actions/checkout@v1 @@ -104,17 +110,18 @@ jobs: sudo chmod 700 get_helm.sh sudo ./get_helm.sh helm version - - name: run emqx on chart - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - timeout-minutes: 5 + - name: setup emqx chart run: | sudo docker save ${TARGET}:${EMQX_TAG} -o emqx.tar.gz sudo k3s ctr image import emqx.tar.gz sed -i -r "s/^appVersion: .*$/appVersion: \"${EMQX_TAG}\"/g" deploy/charts/emqx/Chart.yaml sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml - + - name: run emqx on chart + if: matrix.discovery == 'k8s' + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | helm install emqx \ --set image.repository=${TARGET} \ --set image.pullPolicy=Never \ @@ -124,7 +131,29 @@ jobs: --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ deploy/charts/emqx \ --debug - + - name: run emqx on chart + if: matrix.discovery == 'dns' + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + run: | + helm install emqx \ + --set emqxConfig.EMQX_CLUSTER__DISCOVERY="dns" \ + --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \ + --set emqxConfig.EMQX_CLUSTER__DNS__APP="emqx" \ + --set emqxConfig.EMQX_CLUSTER__DNS__TYPE="srv" \ + --set image.repository=${TARGET} \ + --set image.pullPolicy=Never \ + --set emqxAclConfig="" \ + --set image.pullPolicy=Never \ + --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ + deploy/charts/emqx \ + --debug + - name: waiting emqx started + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + timeout-minutes: 5 + run: | while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \ != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do echo "=============================="; @@ -133,6 +162,18 @@ jobs: echo "waiting emqx started"; sleep 10; done + - name: Check ${{ matrix.kind[0]}} cluster + env: + KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" + timeout-minutes: 10 + run: | + while + nodes="$(kubectl exec -i emqx-0 -- curl --silent --basic -u admin:public -X GET http://localhost:8081/api/v4/brokers | jq '.data|length')"; + [ "$nodes" != "3" ]; + do + echo "waiting emqx cluster scale" + sleep 1 + done - name: get emqx-0 pods log if: failure() env: diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index dcc03b996..7ad870b6d 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -87,7 +87,9 @@ spec: secret: secretName: {{ .Values.emqxLicneseSecretName }} {{- end }} + {{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s" }} serviceAccountName: {{ include "emqx.fullname" . }} + {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} @@ -134,17 +136,6 @@ spec: envFrom: - configMapRef: name: {{ include "emqx.fullname" . }}-env - env: - - name: EMQX_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__K8S__APP_NAME - value: {{ .Release.Name }} - - name: EMQX_CLUSTER__DISCOVERY - value: k8s - - name: EMQX_CLUSTER__K8S__SERVICE_NAME - value: {{ include "emqx.fullname" . }}-headless - - name: EMQX_CLUSTER__K8S__NAMESPACE - value: {{ .Release.Namespace }} resources: {{ toYaml .Values.resources | indent 12 }} volumeMounts: diff --git a/deploy/charts/emqx/templates/configmap.yaml b/deploy/charts/emqx/templates/configmap.yaml index c9c4b4770..328df2000 100644 --- a/deploy/charts/emqx/templates/configmap.yaml +++ b/deploy/charts/emqx/templates/configmap.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} data: {{- range $index, $value := .Values.emqxConfig}} - {{$index}}: "{{ $value }}" + {{$index}}: "{{ tpl (printf "%v" $value) $ }}" {{- end}} --- diff --git a/deploy/charts/emqx/templates/rbac.yaml b/deploy/charts/emqx/templates/rbac.yaml index 87cd18178..79b431442 100644 --- a/deploy/charts/emqx/templates/rbac.yaml +++ b/deploy/charts/emqx/templates/rbac.yaml @@ -1,3 +1,4 @@ +{{- if eq (.Values.emqxConfig.EMQX_CLUSTER__DISCOVERY | default "k8s") "k8s" }} apiVersion: v1 kind: ServiceAccount metadata: @@ -39,4 +40,5 @@ subjects: roleRef: kind: Role name: {{ include "emqx.fullname" . }} - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 62cf779ea..373ba0849 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -50,7 +50,20 @@ initContainers: {} ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) emqxConfig: + EMQX_NAME: "{{ .Release.Name }}" + + ## Cluster discovery by dns + # EMQX_CLUSTER__DISCOVERY: "dns" + # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local" + # EMQX_CLUSTER__DNS__APP: "{{ .Release.Name }}" + # EMQX_CLUSTER__DNS__TYPE: "srv" + + ## Cluster discovery by k8s + EMQX_CLUSTER__DISCOVERY: "k8s" + EMQX_CLUSTER__K8S__APP_NAME: "{{ .Release.Name }}" EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" + EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless" + EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}" ## The address type is used to extract host from k8s service. ## Value: ip | dns | hostname ## Note:Hostname is only supported after v4.0-rc.2 diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index 16b6cb077..1abef430b 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -28,12 +28,20 @@ if [[ -z "$EMQX_NAME" ]]; then fi if [[ -z "$EMQX_HOST" ]]; then - if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then - EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"} - EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX" - elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then - EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'} - EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX") + if [[ "$EMQX_CLUSTER__DISCOVERY" == "dns" ]] && \ + [[ "$EMQX_CLUSTER__DNS__TYPE" == "srv" ]] && \ + grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then + EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME" + elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \ + [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \ + [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then + EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"} + EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX" + elif [[ "$EMQX_CLUSTER__DISCOVERY" == "k8s" ]] && \ + [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \ + [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then + EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'} + EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX") else EMQX_HOST="$LOCAL_IP" fi