From a58529edec7255c1510c77df079961a753110500 Mon Sep 17 00:00:00 2001 From: Rory Z Date: Thu, 8 Sep 2022 09:32:59 +0800 Subject: [PATCH 1/4] ci(helm): fix helm push error --- .github/workflows/release.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 87c3f0c55..e62d06d52 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -58,7 +58,6 @@ jobs: -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - uses: emqx/push-helm-action@v1 - if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' with: charts_dir: "${{ github.workspace }}/deploy/charts/emqx" version: ${{ github.ref_name }} @@ -66,15 +65,6 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-west-2" aws_bucket_name: "repos-emqx-io" - - uses: emqx/push-helm-action@v1 - if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' - with: - charts_dir: "${{ github.workspace }}/deploy/charts/emqx-ee" - version: ${{ github.ref_name }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-west-2" - aws_bucket_name: "repos-emqx-io" - name: update homebrew packages if: github.event_name == 'release' run: | From 874414793f23121f9dab5f99f0e688b830bbf6a4 Mon Sep 17 00:00:00 2001 From: Rory Z Date: Thu, 8 Sep 2022 09:41:00 +0800 Subject: [PATCH 2/4] chore: add emqx enterprise helm chart --- .github/workflows/release.yaml | 10 + .github/workflows/run_fvt_tests.yaml | 22 +- deploy/charts/emqx-ee/Chart.yaml | 21 ++ deploy/charts/emqx-ee/README.md | 121 ++++++++++ .../charts/emqx-ee/templates/StatefulSet.yaml | 168 ++++++++++++++ deploy/charts/emqx-ee/templates/_helpers.tpl | 32 +++ .../charts/emqx-ee/templates/certificate.yaml | 16 ++ .../charts/emqx-ee/templates/configmap.yaml | 19 ++ deploy/charts/emqx-ee/templates/ingress.yaml | 50 ++++ deploy/charts/emqx-ee/templates/rbac.yaml | 44 ++++ deploy/charts/emqx-ee/templates/secret.yaml | 19 ++ .../emqx-ee/templates/service-monitor.yaml | 39 ++++ deploy/charts/emqx-ee/templates/service.yaml | 149 ++++++++++++ deploy/charts/emqx-ee/values.yaml | 214 ++++++++++++++++++ 14 files changed, 919 insertions(+), 5 deletions(-) create mode 100644 deploy/charts/emqx-ee/Chart.yaml create mode 100644 deploy/charts/emqx-ee/README.md create mode 100644 deploy/charts/emqx-ee/templates/StatefulSet.yaml create mode 100644 deploy/charts/emqx-ee/templates/_helpers.tpl create mode 100644 deploy/charts/emqx-ee/templates/certificate.yaml create mode 100644 deploy/charts/emqx-ee/templates/configmap.yaml create mode 100644 deploy/charts/emqx-ee/templates/ingress.yaml create mode 100644 deploy/charts/emqx-ee/templates/rbac.yaml create mode 100644 deploy/charts/emqx-ee/templates/secret.yaml create mode 100644 deploy/charts/emqx-ee/templates/service-monitor.yaml create mode 100644 deploy/charts/emqx-ee/templates/service.yaml create mode 100644 deploy/charts/emqx-ee/values.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e62d06d52..1ffe9a2cb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -58,6 +58,7 @@ jobs: -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - uses: emqx/push-helm-action@v1 + if: github.event_name == 'release' && startsWith(github.ref_name, 'v') with: charts_dir: "${{ github.workspace }}/deploy/charts/emqx" version: ${{ github.ref_name }} @@ -65,6 +66,15 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-west-2" aws_bucket_name: "repos-emqx-io" + - uses: emqx/push-helm-action@v1 + if: github.event_name == 'release' && startsWith(github.ref_name, 'e') + with: + charts_dir: "${{ github.workspace }}/deploy/charts/emqx-ee" + version: ${{ github.ref_name }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-west-2" + aws_bucket_name: "repos-emqx-io" - name: update homebrew packages if: github.event_name == 'release' run: | diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 9e56dd6b3..214603665 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -116,6 +116,7 @@ jobs: - dns profile: - emqx + - emqx-enterprise os: - ["debian11", "debian:11-slim"] otp: @@ -156,6 +157,12 @@ jobs: working-directory: source if: matrix.discovery == 'k8s' run: | + if [ ${{ matrix.profile }} = "emqx-enterprise" ]; then + chart_name="emqx-ee" + else + chart_name="emqx" + fi + helm install emqx \ --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \ --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \ @@ -169,12 +176,18 @@ jobs: --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ - deploy/charts/emqx \ + deploy/charts/${chart_name} \ --debug - name: run emqx on chart working-directory: source if: matrix.discovery == 'dns' run: | + if [ ${{ matrix.profile }} = "emqx-enterprise" ]; then + chart_name="emqx-ee" + else + chart_name="emqx" + fi + helm install emqx \ --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \ --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \ @@ -182,18 +195,17 @@ jobs: --set image.repository=$TARGET \ --set image.pullPolicy=Never \ --set emqxAclConfig="" \ - --set image.pullPolicy=Never \ --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \ --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ - deploy/charts/emqx \ + deploy/charts/${chart_name} \ --debug - name: waiting emqx started timeout-minutes: 10 run: | - while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \ - != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do + while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=emqx -o jsonpath='{.items[0].status.replicas}')" \ + != "$(kubectl get StatefulSet -l app.kubernetes.io/instance=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do echo "=============================="; kubectl get pods; echo "=============================="; diff --git a/deploy/charts/emqx-ee/Chart.yaml b/deploy/charts/emqx-ee/Chart.yaml new file mode 100644 index 000000000..24323b4ef --- /dev/null +++ b/deploy/charts/emqx-ee/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: emqx-ee +icon: https://github.com/emqx.png +description: A Helm chart for EMQX +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 5.0.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 5.0.0 diff --git a/deploy/charts/emqx-ee/README.md b/deploy/charts/emqx-ee/README.md new file mode 100644 index 000000000..9c3762fdd --- /dev/null +++ b/deploy/charts/emqx-ee/README.md @@ -0,0 +1,121 @@ +# Introduction + +This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager. + +# Prerequisites + ++ Kubernetes 1.6+ ++ Helm + +# Installing the Chart + +To install the chart with the release name `my-emqx`: + ++ From github + ``` + $ git clone https://github.com/emqx/emqx.git + $ cd emqx/deploy/charts/emqx + $ helm install my-emqx . + ``` + ++ From chart repos + ``` + helm repo add emqx https://repos.emqx.io/charts + helm install my-emqx emqx/emqx + ``` + > If you want to install an unstable version, you need to add `--devel` when you execute the `helm install` command. + +# Uninstalling the Chart + +To uninstall/delete the `my-emqx` deployment: + +``` +$ helm del my-emqx +``` + +# Configuration + +The following table lists the configurable parameters of the emqx chart and their default values. + +| Parameter | Description | Default Value | +|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | 3 | +| `image.repository` | EMQX Image name | emqx/emqx | +| `image.pullPolicy` | The image pull policy | IfNotPresent | +| `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | +| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | +| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | +| `podAnnotations ` | Annotations for pod | `{}` | +| `podManagementPolicy` | To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock | `Parallel` | +| `persistence.enabled` | Enable EMQX persistence using PVC | false | +| `persistence.storageClass` | Storage class of backing PVC | `nil` (uses alpha storage class annotation) | +| `persistence.existingClaim` | EMQX data Persistent Volume existing claim name, evaluated as a template | "" | +| `persistence.accessMode` | PVC Access Mode for EMQX volume | ReadWriteOnce | +| `persistence.size` | PVC Storage Request for EMQX volume | 20Mi | +| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. | `{}` | +| `resources` | CPU/Memory resource requests/limits | {} | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Map of node/pod affinities | `{}` | +| `service.type` | Kubernetes Service type. | ClusterIP | +| `service.mqtt` | Port for MQTT. | 1883 | +| `service.mqttssl` | Port for MQTT(SSL). | 8883 | +| `service.mgmt` | Port for mgmt API. | 8081 | +| `service.ws` | Port for WebSocket/HTTP. | 8083 | +| `service.wss` | Port for WSS/HTTPS. | 8084 | +| `service.dashboard` | Port for dashboard. | 18083 | +| `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil | +| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil | +| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil | +| `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | nil | +| `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | nil | +| `service.nodePorts.dashboard` | Kubernetes node port for dashboard. | nil | +| `service.loadBalancerIP` | loadBalancerIP for Service | nil | +| `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | +| `service.externalIPs` | ExternalIPs for the service | [] | +| `service.annotations` | Service annotations | {}(evaluated as a template) | +| `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | +| `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | +| `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` | +| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | +| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | +| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | +| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | +| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | +| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | +| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | +| `metrics.type` | Now we only supported "prometheus" | "prometheus" | +| `ssl.enabled` | Enable SSL support | false | +| `ssl.useExisting` | Use existing certificate or let cert-manager generate one | false | +| `ssl.existingName` | Name of existing certificate | emqx-tls | +| `ssl.dnsnames` | DNS name(s) for certificate to be generated | {} | +| `ssl.issuer.name` | Issuer name for certificate generation | letsencrypt-dns | +| `ssl.issuer.kind` | Issuer kind for certificate generation | ClusterIssuer | + +## EMQX specific settings + +The following table lists the configurable [EMQX](https://www.emqx.io/)-specific parameters of the chart and their +default values. +Parameter | Description | Default Value +--- | --- | --- +`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items +expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix +can be omitted) or using the configuration +files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil` +`emqxLicenseSecretName` | Name of the secret that holds the license information | `nil` + +## SSL settings +`cert-manager` generates secrets with certificate data using the keys `tls.crt` and `tls.key`. The helm chart always mounts those keys as files to `/tmp/ssl/` +which needs to explicitly configured by either changing the emqx config file or by passing the following environment variables: + +``` + EMQX_LISTENERS__SSL__DEFAULT__SSL_OPTIONS__CERTFILE: /tmp/ssl/tls.crt + EMQX_LISTENERS__SSL__DEFAULT__SSL_OPTIONS__KEYFILE: /tmp/ssl/tls.key +``` + +If you chose to use an existing certificate, make sure, you update the filenames accordingly. + diff --git a/deploy/charts/emqx-ee/templates/StatefulSet.yaml b/deploy/charts/emqx-ee/templates/StatefulSet.yaml new file mode 100644 index 000000000..3af9fd62d --- /dev/null +++ b/deploy/charts/emqx-ee/templates/StatefulSet.yaml @@ -0,0 +1,168 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "emqx.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + serviceName: {{ include "emqx.fullname" . }}-headless + podManagementPolicy: {{ .Values.podManagementPolicy }} + {{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} + volumeClaimTemplates: + - metadata: + name: emqx-data + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + spec: + {{- if .Values.persistence.storageClassName }} + storageClassName: {{ .Values.persistence.storageClassName | quote }} + {{- end }} + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- end }} + updateStrategy: + type: RollingUpdate + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ include "emqx.name" . }} + version: {{ .Chart.AppVersion }} + app.kubernetes.io/name: {{ include "emqx.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.recreatePods }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }} + {{- end }} + spec: + volumes: + {{- if .Values.ssl.enabled }} + - name: ssl-cert + secret: + secretName: {{ include "emqx.fullname" . }}-tls + {{- end }} + {{- if not .Values.persistence.enabled }} + - name: emqx-data + emptyDir: {} + {{- else if .Values.persistence.existingClaim }} + - name: emqx-data + persistentVolumeClaim: + {{- with .Values.persistence.existingClaim }} + claimName: {{ tpl . $ }} + {{- end }} + {{- end }} + {{- if .Values.emqxLicenseSecretName }} + - name: emqx-license + secret: + secretName: {{ .Values.emqxLicenseSecretName }} + {{- end }} + {{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} + serviceAccountName: {{ include "emqx.fullname" . }} + {{- end }} + {{- if .Values.podSecurityContext.enabled }} + securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.initContainers }} + initContainers: +{{ toYaml .Values.initContainers | indent 8 }} + {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + containers: + - name: emqx + image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + ports: + - name: mqtt + containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT | default 1883 }} + - name: mqttssl + containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__SSL__DEFAULT | default 8883 }} + - name: ws + containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WS__DEFAULT | default 8083 }} + - name: wss + containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT | default 8084 }} + - name: dashboard + containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP | default 18083 }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + - name: internalmqtt + containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT }} + {{- end }} + {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS) }} + - name: dashboardtls + containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS }} + {{- end }} + - name: ekka + containerPort: 4370 + envFrom: + - configMapRef: + name: {{ include "emqx.fullname" . }}-env + {{- if .Values.envFromSecret }} + - secretRef: + name: {{ .Values.envFromSecret }} + {{- end }} + resources: +{{ toYaml .Values.resources | indent 12 }} + volumeMounts: + - name: emqx-data + mountPath: "/opt/emqx/data" + {{- if .Values.ssl.enabled }} + - name: ssl-cert + mountPath: /tmp/ssl + readOnly: true + {{- end}} + {{ if .Values.emqxLicenseSecretName }} + - name: emqx-license + mountPath: "/opt/emqx/etc/emqx.lic" + subPath: "emqx.lic" + readOnly: true + {{- end }} + readinessProbe: + httpGet: + path: /status + port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP | default 18083 }} + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 30 + livenessProbe: + httpGet: + path: /status + port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP | default 18083 }} + initialDelaySeconds: 60 + periodSeconds: 30 + failureThreshold: 10 + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/charts/emqx-ee/templates/_helpers.tpl b/deploy/charts/emqx-ee/templates/_helpers.tpl new file mode 100644 index 000000000..ad01d5862 --- /dev/null +++ b/deploy/charts/emqx-ee/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "emqx.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "emqx.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "emqx.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/deploy/charts/emqx-ee/templates/certificate.yaml b/deploy/charts/emqx-ee/templates/certificate.yaml new file mode 100644 index 000000000..36b7f6521 --- /dev/null +++ b/deploy/charts/emqx-ee/templates/certificate.yaml @@ -0,0 +1,16 @@ +{{- if and (.Values.ssl.enable) (not .Values.ssl.useExisting) -}} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "emqx.fullname" . }}-tls +spec: + secretName: {{ include "emqx.fullname" . }}-tls + issuerRef: + name: {{ default "letsencrypt-staging" .Values.ssl.issuer.name }} + kind: {{ default "ClusterIssuer" .Values.ssl.issuer.kind }} + dnsNames: + {{- range .Values.ssl.dnsnames }} + - {{ . }} + {{- end }} +{{- end -}} diff --git a/deploy/charts/emqx-ee/templates/configmap.yaml b/deploy/charts/emqx-ee/templates/configmap.yaml new file mode 100644 index 000000000..e0563d02a --- /dev/null +++ b/deploy/charts/emqx-ee/templates/configmap.yaml @@ -0,0 +1,19 @@ +{{- if .Values.emqxConfig }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "emqx.fullname" . }}-env + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +data: + {{- range $index, $value := .Values.emqxConfig }} + {{- if $value }} + {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }} + {{ print "EMQX_" $key }}: "{{ tpl (printf "%v" $value) $ }}" + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/charts/emqx-ee/templates/ingress.yaml b/deploy/charts/emqx-ee/templates/ingress.yaml new file mode 100644 index 000000000..b6f496d88 --- /dev/null +++ b/deploy/charts/emqx-ee/templates/ingress.yaml @@ -0,0 +1,50 @@ +{{- if .Values.ingress.dashboard.enabled -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ printf "%s-%s" (include "emqx.fullname" .) "dashboard" }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.ingress.dashboard.annotations }} + annotations: + {{- toYaml .Values.ingress.dashboard.annotations | nindent 4 }} + {{- end }} +spec: +{{- if and .Values.ingress.dashboard.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.dashboard.ingressClassName }} +{{- end }} + rules: + {{- range $host := .Values.ingress.dashboard.hosts }} + - host: {{ $host }} + http: + paths: + - path: {{ $.Values.ingress.dashboard.path | default "/" }} + {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ $.Values.ingress.dashboard.pathType | default "ImplementationSpecific" }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "emqx.fullname" $ }} + port: + number: {{ $.Values.service.dashboard }} + {{- else }} + serviceName: {{ include "emqx.fullname" $ }} + servicePort: {{ $.Values.service.dashboard }} + {{- end }} + {{- end -}} + {{- if .Values.ingress.dashboard.tls }} + tls: + {{- toYaml .Values.ingress.dashboard.tls | nindent 4 }} + {{- end }} +--- +{{- end }} diff --git a/deploy/charts/emqx-ee/templates/rbac.yaml b/deploy/charts/emqx-ee/templates/rbac.yaml new file mode 100644 index 000000000..f2cdd3601 --- /dev/null +++ b/deploy/charts/emqx-ee/templates/rbac.yaml @@ -0,0 +1,44 @@ +{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: {{ .Release.Namespace }} + name: {{ include "emqx.fullname" . }} +--- +kind: Role +{{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- else }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +{{- end }} +metadata: + namespace: {{ .Release.Namespace }} + name: {{ include "emqx.fullname" . }} +rules: +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - watch + - list +--- +kind: RoleBinding +{{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- else }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +{{- end }} +metadata: + namespace: {{ .Release.Namespace }} + name: {{ include "emqx.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "emqx.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "emqx.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/deploy/charts/emqx-ee/templates/secret.yaml b/deploy/charts/emqx-ee/templates/secret.yaml new file mode 100644 index 000000000..447326769 --- /dev/null +++ b/deploy/charts/emqx-ee/templates/secret.yaml @@ -0,0 +1,19 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "emqx.fullname" . }}-basic-auth + namespace: {{ .Release.Namespace }} +type: kubernetes.io/basic-auth +stringData: + {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__DEFAULT_USERNAME) }} + username: admin + {{- else }} + username: {{ .Values.emqxConfig.EMQX_DASHBOARD__DEFAULT_USERNAME }} + {{- end }} + {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__DEFAULT_PASSWORD) }} + password: public + {{- else }} + password: {{ .Values.emqxConfig.EMQX_DASHBOARD__DEFAULT_PASSWORD}} + {{- end }} +{{- end }} diff --git a/deploy/charts/emqx-ee/templates/service-monitor.yaml b/deploy/charts/emqx-ee/templates/service-monitor.yaml new file mode 100644 index 000000000..3d97ab351 --- /dev/null +++ b/deploy/charts/emqx-ee/templates/service-monitor.yaml @@ -0,0 +1,39 @@ +{{- if and (.Values.metrics.enabled) (eq .Values.metrics.type "prometheus") }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "emqx.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.service.annotations }} + annotations: + {{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} +spec: + endpoints: + - interval: 10s + port: dashboard + scheme: http + path: /api/v5/prometheus/stats + params: + type: + - prometheus + basicAuth: + password: + name: {{ include "emqx.fullname" . }}-basic-auth + key: password + username: + name: {{ include "emqx.fullname" . }}-basic-auth + key: username + jobLabel: {{ .Release.Name }}-scraping + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "emqx.name" . }} +{{- end }} diff --git a/deploy/charts/emqx-ee/templates/service.yaml b/deploy/charts/emqx-ee/templates/service.yaml new file mode 100644 index 000000000..54efa6426 --- /dev/null +++ b/deploy/charts/emqx-ee/templates/service.yaml @@ -0,0 +1,149 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "emqx.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if eq .Values.service.type "LoadBalancer" }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml .Values.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- if .Values.service.externalIPs }} + externalIPs: {{- toYaml .Values.service.externalIPs | nindent 4 }} + {{- end }} + {{- end }} + ports: + - name: mqtt + port: {{ .Values.service.mqtt | default 1883 }} + protocol: TCP + targetPort: mqtt + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.mqtt)) }} + nodePort: {{ .Values.service.nodePorts.mqtt }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + - name: internalmqtt + port: {{ .Values.service.internalmqtt | default 11883 }} + protocol: TCP + targetPort: internalmqtt + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.internalmqtt)) }} + nodePort: {{ .Values.service.nodePorts.internalmqtt }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{ end }} + - name: mqttssl + port: {{ .Values.service.mqttssl | default 8883 }} + protocol: TCP + targetPort: mqttssl + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.mqttssl)) }} + nodePort: {{ .Values.service.nodePorts.mqttssl }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + - name: ws + port: {{ .Values.service.ws | default 8083 }} + protocol: TCP + targetPort: ws + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.ws)) }} + nodePort: {{ .Values.service.nodePorts.ws }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + - name: wss + port: {{ .Values.service.wss | default 8084 }} + protocol: TCP + targetPort: wss + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.wss)) }} + nodePort: {{ .Values.service.nodePorts.wss }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + - name: dashboard + port: {{ .Values.service.dashboard | default 18083 }} + protocol: TCP + targetPort: dashboard + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.dashboard)) }} + nodePort: {{ .Values.service.nodePorts.dashboard }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if not (empty .Values.service.dashboardtls) }} + - name: dashboardtls + port: {{ .Values.service.dashboardtls }} + protocol: TCP + targetPort: dashboardtls + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.dashboardtls)) }} + nodePort: {{ .Values.service.nodePorts.dashboardtls }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- end }} + selector: + app.kubernetes.io/name: {{ include "emqx.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "emqx.fullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + type: ClusterIP + sessionAffinity: None + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: mqtt + port: {{ .Values.service.mqtt | default 1883 }} + protocol: TCP + targetPort: mqtt + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + - name: internalmqtt + port: {{ .Values.service.internalmqtt | default 11883 }} + protocol: TCP + targetPort: internalmqtt + {{ end }} + - name: mqttssl + port: {{ .Values.service.mqttssl | default 8883 }} + protocol: TCP + targetPort: mqttssl + - name: ws + port: {{ .Values.service.ws | default 8083 }} + protocol: TCP + targetPort: ws + - name: wss + port: {{ .Values.service.wss | default 8084 }} + protocol: TCP + targetPort: wss + - name: dashboard + port: {{ .Values.service.dashboard | default 18083 }} + protocol: TCP + targetPort: dashboard + - name: ekka + port: 4370 + protocol: TCP + targetPort: ekka + selector: + app.kubernetes.io/name: {{ include "emqx.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/deploy/charts/emqx-ee/values.yaml b/deploy/charts/emqx-ee/values.yaml new file mode 100644 index 000000000..aa61a62ea --- /dev/null +++ b/deploy/charts/emqx-ee/values.yaml @@ -0,0 +1,214 @@ +## Default values for emqx. +## This is a YAML-formatted file. +## Declare variables to be passed into your templates. + +## It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. +replicaCount: 3 +image: + repository: emqx/emqx-enterprise + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistryKeySecretName + + +# The name of a secret in the same kubernetes namespace which contains values to +# be added to the environment (must be manually created) +# This can be useful for passwords and logins, etc. + +# envFromSecret: "emqx-secrets" + +## Forces the recreation of pods during helm upgrades. This can be useful to update configuration values even if the container image did not change. +recreatePods: false + +podAnnotations: {} + +# Pod deployment policy +# value: OrderedReady | Parallel +# To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock +podManagementPolicy: Parallel + +persistence: + enabled: false + size: 20Mi + storageClassName: "" + accessMode: ReadWriteOnce + ## Existing PersistentVolumeClaims + ## The value is evaluated as a template + ## So, for example, the name can depend on .Release or .Chart + # existingClaim: "" + +resources: {} + # limits: + # cpu: 500m + # memory: 512Mi + # requests: + # cpu: 500m + # memory: 512Mi + +# Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. +initContainers: {} + # - name: sysctl + # image: busybox + # securityContext: + # runAsUser: 0 + # runAsGroup: 0 + # capabilities: + # add: + # - SYS_ADMIN + # drop: + # - ALL + # command: + # - /bin/sh + # - -c + # - | + # mount -o remount rw /proc/sys + # sysctl -w net.core.somaxconn=65535 + # sysctl -w net.ipv4.ip_local_port_range="1024 65535" + # sysctl -w kernel.core_uses_pid=0 + # sysctl -w net.ipv4.tcp_tw_reuse=1 + # sysctl -w fs.nr_open=1000000000 + # sysctl -w fs.file-max=1000000000 + # sysctl -w net.ipv4.ip_local_port_range='1025 65534' + # sysctl -w net.ipv4.udp_mem='74583000 499445000 749166000' + # sysctl -w net.ipv4.tcp_max_sync_backlog=163840 + # sysctl -w net.core.netdev_max_backlog=163840 + # sysctl -w net.core.optmem_max=16777216 + # sysctl -w net.ipv4.tcp_rmem='1024 4096 16777216' + # sysctl -w net.ipv4.tcp_wmem='1024 4096 16777216' + # sysctl -w net.ipv4.tcp_max_tw_buckets=1048576 + # sysctl -w net.ipv4.tcp_fin_timeout=15 + # sysctl -w net.core.rmem_default=262144000 + # sysctl -w net.core.wmem_default=262144000 + # sysctl -w net.core.rmem_max=262144000 + # sysctl -w net.core.wmem_max=262144000 + # sysctl -w net.ipv4.tcp_mem='378150000 504200000 756300000' + # sysctl -w net.netfilter.nf_conntrack_max=1000000 + # sysctl -w net.netfilter.nf_conntrack_tcp_timeout_time_wait=30 + +## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) +emqxConfig: + EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns" + EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local" + EMQX_CLUSTER__DNS__RECORD_TYPE: "srv" + # EMQX_CLUSTER__DISCOVERY_STRATEGY: "k8s" + # EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" + # EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless" + # EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}" + ## The address type is used to extract host from k8s service. + ## Value: ip | dns | hostname + ## Noteļ¼šHostname is only supported after v4.0-rc.2 + EMQX_CLUSTER__K8S__ADDRESS_TYPE: "hostname" + EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local" + ## if EMQX_CLUSTER__K8S__ADDRESS_TYPE eq dns + # EMQX_CLUSTER__K8S__SUFFIX: "pod.cluster.local" + EMQX_DASHBOARD__DEFAULT_USERNAME: "admin" + EMQX_DASHBOARD__DEFAULT_PASSWORD: "public" + +## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicenseSecretName" +## Example: +## kubectl create secret generic emqx-license-secret-name --from-file=/path/to/emqx.lic +emqxLicenseSecretName: + +service: + ## Service type + ## + type: ClusterIP + ## Port for MQTT + ## + mqtt: 1883 + ## Port for MQTT(SSL) + ## + mqttssl: 8883 + ## Port for mgmt API + ## + mgmt: 8081 + ## Port for WebSocket/HTTP + ## + ws: 8083 + ## Port for WSS/HTTPS + ## + wss: 8084 + ## Port for dashboard + ## + dashboard: 18083 + ## Port for dashboard HTTPS + ## + # dashboardtls: 18084 + ## Specify the nodePort(s) value for the LoadBalancer and NodePort service types. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + ## + nodePorts: + mqtt: + mqttssl: + mgmt: + ws: + wss: + dashboard: + dashboardtls: + ## Set the LoadBalancer service type to internal only. + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer + ## + # loadBalancerIP: + ## Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## Example: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## Set the ExternalIPs + ## + externalIPs: [] + ## Provide any additional annotations which may be required. Evaluated as a template + ## + annotations: {} + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +ingress: + ## ingress for EMQX Dashboard + dashboard: + enabled: false + # ingressClassName: nginx + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + path: / + pathType: ImplementationSpecific + hosts: + - dashboard.emqx.local + tls: [] + +podSecurityContext: + enabled: true + fsGroup: 1000 + fsGroupChangePolicy: Always + runAsUser: 1000 + supplementalGroups: + - 1000 + +containerSecurityContext: + enabled: true + runAsNonRoot: true + runAsUser: 1000 + +metrics: + enabled: false + type: prometheus + +ssl: + enabled: false + useExisting: false + existingName: emqx-tls + dnsnames: {} + issuer: + name: letsencrypt-dns + kind: ClusterIssuer From cbe21735470ac12ce09ace1b3fd66d60be3e6917 Mon Sep 17 00:00:00 2001 From: Rory Z Date: Tue, 13 Sep 2022 15:27:51 +0800 Subject: [PATCH 3/4] chore(helm): add image.tag for helm chart --- .github/workflows/run_fvt_tests.yaml | 9 +++------ deploy/charts/emqx-ee/templates/StatefulSet.yaml | 2 +- deploy/charts/emqx/templates/StatefulSet.yaml | 2 +- deploy/charts/emqx/values.yaml | 2 ++ 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 214603665..b22254c8a 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -148,11 +148,7 @@ jobs: echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV - run: minikube start - - name: setup helm chart - working-directory: source - run: | - minikube image load $TARGET:$EMQX_TAG - sed -i -r "s/^appVersion: .*$/appVersion: \"$EMQX_TAG\"/g" deploy/charts/emqx/Chart.yaml + - run: minikube image load $TARGET:$EMQX_TAG - name: run emqx on chart working-directory: source if: matrix.discovery == 'k8s' @@ -170,8 +166,8 @@ jobs: --set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \ --set image.repository=$TARGET \ --set image.pullPolicy=Never \ + --set image.tag=$EMQX_TAG \ --set emqxAclConfig="" \ - --set image.pullPolicy=Never \ --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \ --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ @@ -194,6 +190,7 @@ jobs: --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \ --set image.repository=$TARGET \ --set image.pullPolicy=Never \ + --set image.tag=$EMQX_TAG \ --set emqxAclConfig="" \ --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \ --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ diff --git a/deploy/charts/emqx-ee/templates/StatefulSet.yaml b/deploy/charts/emqx-ee/templates/StatefulSet.yaml index 3af9fd62d..4b970e183 100644 --- a/deploy/charts/emqx-ee/templates/StatefulSet.yaml +++ b/deploy/charts/emqx-ee/templates/StatefulSet.yaml @@ -91,7 +91,7 @@ spec: {{- end }} containers: - name: emqx - image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.containerSecurityContext.enabled }} securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 3af9fd62d..4b970e183 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -91,7 +91,7 @@ spec: {{- end }} containers: - name: emqx - image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.containerSecurityContext.enabled }} securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 94e7eeb3c..5bf7377f4 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -7,6 +7,8 @@ replicaCount: 3 image: repository: emqx/emqx pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" ## Optionally specify an array of imagePullSecrets. ## Secrets must be manually created in the namespace. ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ From b3355ff0357d466d928a376cae8ae5c0b358e5ca Mon Sep 17 00:00:00 2001 From: Rory Z Date: Tue, 13 Sep 2022 16:33:09 +0800 Subject: [PATCH 4/4] chore(helm): rename emqx-ee helm chart to emqx-enterprise --- .github/workflows/release.yaml | 2 +- .github/workflows/run_fvt_tests.yaml | 52 +++++-------------- .../{emqx-ee => emqx-enterprise}/Chart.yaml | 2 +- .../{emqx-ee => emqx-enterprise}/README.md | 0 .../templates/StatefulSet.yaml | 0 .../templates/_helpers.tpl | 0 .../templates/certificate.yaml | 0 .../templates/configmap.yaml | 0 .../templates/ingress.yaml | 0 .../templates/rbac.yaml | 0 .../templates/secret.yaml | 0 .../templates/service-monitor.yaml | 0 .../templates/service.yaml | 0 .../{emqx-ee => emqx-enterprise}/values.yaml | 0 14 files changed, 16 insertions(+), 40 deletions(-) rename deploy/charts/{emqx-ee => emqx-enterprise}/Chart.yaml (97%) rename deploy/charts/{emqx-ee => emqx-enterprise}/README.md (100%) rename deploy/charts/{emqx-ee => emqx-enterprise}/templates/StatefulSet.yaml (100%) rename deploy/charts/{emqx-ee => emqx-enterprise}/templates/_helpers.tpl (100%) rename deploy/charts/{emqx-ee => emqx-enterprise}/templates/certificate.yaml (100%) rename deploy/charts/{emqx-ee => emqx-enterprise}/templates/configmap.yaml (100%) rename deploy/charts/{emqx-ee => emqx-enterprise}/templates/ingress.yaml (100%) rename deploy/charts/{emqx-ee => emqx-enterprise}/templates/rbac.yaml (100%) rename deploy/charts/{emqx-ee => emqx-enterprise}/templates/secret.yaml (100%) rename deploy/charts/{emqx-ee => emqx-enterprise}/templates/service-monitor.yaml (100%) rename deploy/charts/{emqx-ee => emqx-enterprise}/templates/service.yaml (100%) rename deploy/charts/{emqx-ee => emqx-enterprise}/values.yaml (100%) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1ffe9a2cb..a7abcb244 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -69,7 +69,7 @@ jobs: - uses: emqx/push-helm-action@v1 if: github.event_name == 'release' && startsWith(github.ref_name, 'e') with: - charts_dir: "${{ github.workspace }}/deploy/charts/emqx-ee" + charts_dir: "${{ github.workspace }}/deploy/charts/emqx-enterprise" version: ${{ github.ref_name }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index b22254c8a..fbd18f1ce 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -153,16 +153,10 @@ jobs: working-directory: source if: matrix.discovery == 'k8s' run: | - if [ ${{ matrix.profile }} = "emqx-enterprise" ]; then - chart_name="emqx-ee" - else - chart_name="emqx" - fi - - helm install emqx \ + helm install ${{ matrix.profile }} \ --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \ --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \ - --set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="emqx-headless" \ + --set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="${{ matrix.profile }}-headless" \ --set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \ --set image.repository=$TARGET \ --set image.pullPolicy=Never \ @@ -172,22 +166,16 @@ jobs: --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ - deploy/charts/${chart_name} \ + deploy/charts/${{ matrix.profile }} \ --debug - name: run emqx on chart working-directory: source if: matrix.discovery == 'dns' run: | - if [ ${{ matrix.profile }} = "emqx-enterprise" ]; then - chart_name="emqx-ee" - else - chart_name="emqx" - fi - - helm install emqx \ + helm install ${{ matrix.profile }} \ --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \ --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \ - --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \ + --set emqxConfig.EMQX_CLUSTER__DNS__NAME="${{ matrix.profile }}-headless.default.svc.cluster.local" \ --set image.repository=$TARGET \ --set image.pullPolicy=Never \ --set image.tag=$EMQX_TAG \ @@ -196,13 +184,13 @@ jobs: --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ - deploy/charts/${chart_name} \ + deploy/charts/${{ matrix.profile }} \ --debug - name: waiting emqx started timeout-minutes: 10 run: | - while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=emqx -o jsonpath='{.items[0].status.replicas}')" \ - != "$(kubectl get StatefulSet -l app.kubernetes.io/instance=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do + while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${{ matrix.profile }} -o jsonpath='{.items[0].status.replicas}')" \ + != "$(kubectl get StatefulSet -l app.kubernetes.io/instance=${{ matrix.profile }} -o jsonpath='{.items[0].status.readyReplicas}')" ]; do echo "=============================="; kubectl get pods; echo "=============================="; @@ -212,28 +200,13 @@ jobs: - name: Check cluster timeout-minutes: 10 run: | - kubectl port-forward svc/emqx 18083:18083 & + kubectl port-forward service/${{ matrix.profile }} 18083:18083 > /dev/null & while [ "$(curl --silent --basic -u admin:public -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ]; do - echo "waiting emqx cluster scale" + echo "waiting ${{ matrix.profile }} cluster scale" sleep 1 done - - name: get emqx-0 pods log - if: failure() - run: | - kubectl describe pods emqx-0 - kubectl logs emqx-0 - - name: get emqx-1 pods log - if: failure() - run: | - kubectl describe pods emqx-1 - kubectl logs emqx-1 - - name: get emqx-2 pods log - if: failure() - run: | - kubectl describe pods emqx-2 - kubectl logs emqx-2 - uses: actions/checkout@v2 with: repository: emqx/paho.mqtt.testing @@ -244,6 +217,7 @@ jobs: pip install pytest echo "$HOME/.local/bin" >> $GITHUB_PATH - name: run paho test + timeout-minutes: 10 run: | port_connected () { local server="$1" @@ -251,7 +225,7 @@ jobs: echo > /dev/tcp/${server}/${port} 2>/dev/null } - kubectl port-forward service/emqx 1883:1883 > /dev/null & + kubectl port-forward service/${{ matrix.profile }} 1883:1883 > /dev/null & while ! port_connected localhost 1883; do echo server not listening yet... @@ -259,3 +233,5 @@ jobs: done pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1" + - if: failure() + run: kubectl logs -l "app.kubernetes.io/instance=${{ matrix.profile }}" -c emqx --tail=1000 diff --git a/deploy/charts/emqx-ee/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml similarity index 97% rename from deploy/charts/emqx-ee/Chart.yaml rename to deploy/charts/emqx-enterprise/Chart.yaml index 24323b4ef..e407ab0d9 100644 --- a/deploy/charts/emqx-ee/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: emqx-ee +name: emqx-enterprise icon: https://github.com/emqx.png description: A Helm chart for EMQX # A chart can be either an 'application' or a 'library' chart. diff --git a/deploy/charts/emqx-ee/README.md b/deploy/charts/emqx-enterprise/README.md similarity index 100% rename from deploy/charts/emqx-ee/README.md rename to deploy/charts/emqx-enterprise/README.md diff --git a/deploy/charts/emqx-ee/templates/StatefulSet.yaml b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml similarity index 100% rename from deploy/charts/emqx-ee/templates/StatefulSet.yaml rename to deploy/charts/emqx-enterprise/templates/StatefulSet.yaml diff --git a/deploy/charts/emqx-ee/templates/_helpers.tpl b/deploy/charts/emqx-enterprise/templates/_helpers.tpl similarity index 100% rename from deploy/charts/emqx-ee/templates/_helpers.tpl rename to deploy/charts/emqx-enterprise/templates/_helpers.tpl diff --git a/deploy/charts/emqx-ee/templates/certificate.yaml b/deploy/charts/emqx-enterprise/templates/certificate.yaml similarity index 100% rename from deploy/charts/emqx-ee/templates/certificate.yaml rename to deploy/charts/emqx-enterprise/templates/certificate.yaml diff --git a/deploy/charts/emqx-ee/templates/configmap.yaml b/deploy/charts/emqx-enterprise/templates/configmap.yaml similarity index 100% rename from deploy/charts/emqx-ee/templates/configmap.yaml rename to deploy/charts/emqx-enterprise/templates/configmap.yaml diff --git a/deploy/charts/emqx-ee/templates/ingress.yaml b/deploy/charts/emqx-enterprise/templates/ingress.yaml similarity index 100% rename from deploy/charts/emqx-ee/templates/ingress.yaml rename to deploy/charts/emqx-enterprise/templates/ingress.yaml diff --git a/deploy/charts/emqx-ee/templates/rbac.yaml b/deploy/charts/emqx-enterprise/templates/rbac.yaml similarity index 100% rename from deploy/charts/emqx-ee/templates/rbac.yaml rename to deploy/charts/emqx-enterprise/templates/rbac.yaml diff --git a/deploy/charts/emqx-ee/templates/secret.yaml b/deploy/charts/emqx-enterprise/templates/secret.yaml similarity index 100% rename from deploy/charts/emqx-ee/templates/secret.yaml rename to deploy/charts/emqx-enterprise/templates/secret.yaml diff --git a/deploy/charts/emqx-ee/templates/service-monitor.yaml b/deploy/charts/emqx-enterprise/templates/service-monitor.yaml similarity index 100% rename from deploy/charts/emqx-ee/templates/service-monitor.yaml rename to deploy/charts/emqx-enterprise/templates/service-monitor.yaml diff --git a/deploy/charts/emqx-ee/templates/service.yaml b/deploy/charts/emqx-enterprise/templates/service.yaml similarity index 100% rename from deploy/charts/emqx-ee/templates/service.yaml rename to deploy/charts/emqx-enterprise/templates/service.yaml diff --git a/deploy/charts/emqx-ee/values.yaml b/deploy/charts/emqx-enterprise/values.yaml similarity index 100% rename from deploy/charts/emqx-ee/values.yaml rename to deploy/charts/emqx-enterprise/values.yaml