From ec902871cfcebd3cbb080a4b88ae8eb34c1f5671 Mon Sep 17 00:00:00 2001
From: Fernando Almeida <nanditu@gmail.com>
Date: Tue, 25 Jan 2022 11:57:20 +0000
Subject: [PATCH 01/13] feat(helm-chart): split ingress into separate files for
 readability feat(helm-chart): fix ingress to allow custom user path as need
 (eg. mgmt) feat(helm-chart): revised README to include missing contents

---
 deploy/charts/emqx/README.md                  | 116 ++++++++++--------
 .../{ingress.yaml => ingress.dashboard.yaml}  |  50 --------
 .../charts/emqx/templates/ingress.mgmt.yaml   |  49 ++++++++
 deploy/charts/emqx/templates/ingress.wss.yaml |  49 ++++++++
 deploy/charts/emqx/values.yaml                |  23 ++++
 5 files changed, 188 insertions(+), 99 deletions(-)
 rename deploy/charts/emqx/templates/{ingress.yaml => ingress.dashboard.yaml} (50%)
 create mode 100644 deploy/charts/emqx/templates/ingress.mgmt.yaml
 create mode 100644 deploy/charts/emqx/templates/ingress.wss.yaml

diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index cca0e88bc..1d350a896 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -1,5 +1,5 @@
 # Introduction
-This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager. 
+This chart bootstraps an emqx deployment on a Kubernetes (K8s) cluster using the Helm package manager.
 
 # Prerequisites
 + Kubernetes 1.6+
@@ -8,7 +8,7 @@ This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm
 # Installing the Chart
 To install the chart with the release name `my-emqx`:
 
-+   From github 
++   From github
     ```
     $ git clone https://github.com/emqx/emqx.git
     $ cd emqx/deploy/charts/emqx
@@ -29,51 +29,69 @@ $ helm del  my-emqx
 ```
 
 # Configuration
-The following table lists the configurable parameters of the emqx chart and their default values.
+The following sections describe the configurable parameters of the EMQx chart and their default values.
+## Kubernetes-specific
+The following table lists the configurable K8s parameters of the EMQx chart and their default values.
+| Parameter  | Description | Default Value
+| ---        |  ---        | ---
+| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | `3`
+| `image.repository` | EMQ X Image name | `emqx/emqx`
+| `image.pullPolicy`  | The image pull policy  | `IfNotPresent`
+| `image.pullSecrets `  | The image pull secrets (does not add image pull secrets to deployed pods)  |``[]``
+| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | `nil`
+| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | `false`
+| `persistence.enabled` | Enable EMQX persistence using PVC | `false`
+| `persistence.storageClass` | Storage class of backing PVC (uses alpha storage class annotation) | `nil`
+| `persistence.existingClaim` | EMQ X data Persistent Volume existing claim name, evaluated as a template | `""`
+| `persistence.accessMode` | PVC Access Mode for EMQX volume | `ReadWriteOnce`
+| `persistence.size` | PVC Storage Request for EMQX volume | `20Mi`
+| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`
+| `resources` | CPU/Memory resource requests/limits |`{}`
+| `nodeSelector` | Node labels for pod assignment |`{}`
+| `tolerations` | Toleration labels for pod assignment |``[]``
+| `affinity` | Map of node/pod affinities |`{}`
+| `service.type`  | Kubernetes Service type. | `ClusterIP`
+| `service.mqtt`  | Port for MQTT. | `1883`
+| `service.mqttssl` | Port for MQTT(SSL). | `8883`
+| `service.mgmt`  | Port for mgmt API. | `8081`
+| `service.ws`  | Port for WebSocket/HTTP. | `8083`
+| `service.wss`  | Port for WSS/HTTPS. | `8084`
+| `service.dashboard`  | Port for dashboard. | `18083`
+| `service.nodePorts.mqtt`  | Kubernetes node port for MQTT. | `nil`
+| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | `nil`
+| `service.nodePorts.mgmt`  | Kubernetes node port for mgmt API. | `nil`
+| `service.nodePorts.ws`  | Kubernetes node port for WebSocket/HTTP. | `nil`
+| `service.nodePorts.wss`  | Kubernetes node port for WSS/HTTPS. | `nil`
+| `service.nodePorts.dashboard`  | Kubernetes node port for dashboard. | `nil`
+| `service.loadBalancerIP`  | loadBalancerIP for Service |	`nil`
+| `service.loadBalancerSourceRanges` |	Address(es) that are allowed when service is LoadBalancer |	`[]`
+| `service.externalIPs` |	ExternalIPs for the service |	`[]`
+| `service.annotations` |	Service annotations (evaluated as a template) |	`{}`
+| `ingress.dashboard.enabled` |	Enable ingress for EMQX Dashboard |	false
+| `ingress.dashboard.ingressClassName` |	Set the ingress class for EMQX Dashboard
+| `ingress.dashboard.path` | Ingress path for EMQX Dashboard |	`/`
+| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API |	dashboard.emqx.local
+| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API |	`[]`
+| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API |	`{}`
+| `ingress.mgmt.enabled` |	Enable ingress for EMQX Mgmt API |	`false`
+| `ingress.mqtt.ingressClassName` |	Set the ingress class for EMQX Mgmt API | `nil`
+| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | `/`
+| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API |	`api.emqx.local`
+| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API |	`[]`
+| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API |	`{}`
+| `ingress.wss.enabled` |	Enable ingress for EMQX Mgmt API |	`false`
+| `ingress.wss.ingressClassName` |	Set the ingress class for EMQX Mgmt API | `nil`
+| `ingress.wss.path` | Ingress path for EMQX WSS |	`/`
+| `ingress.wss.hosts` | Ingress hosts for EMQX WSS |    `wss.emqx.local`
+| `ingress.wss.tls` | Ingress tls for EMQX WSS |	`[]`
+| `ingress.wss.annotations` | Ingress annotations for EMQX WSS |	`{}`
 
-| Parameter  | Description | Default Value |
-| ---        |  ---        | ---           |
-| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. |3|
-| `image.repository` | EMQ X Image name |emqx/emqx|
-| `image.pullPolicy`  | The image pull policy  |IfNotPresent|
-| `image.pullSecrets `  | The image pull secrets  |`[]` (does not add image pull secrets to deployed pods)|
-| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false |
-| `persistence.enabled` | Enable EMQX persistence using PVC |false|
-| `persistence.storageClass` | Storage class of backing PVC |`nil` (uses alpha storage class annotation)|
-| `persistence.existingClaim` | EMQ X data Persistent Volume existing claim name, evaluated as a template |""|
-| `persistence.accessMode` | PVC Access Mode for EMQX volume |ReadWriteOnce|
-| `persistence.size` | PVC Storage Request for EMQX volume |20Mi|
-| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`|
-| `resources` | CPU/Memory resource requests/limits |{}|
-| `nodeSelector` | Node labels for pod assignment |`{}`|
-| `tolerations` | Toleration labels for pod assignment |`[]`|
-| `affinity` | Map of node/pod affinities |`{}`|
-| `service.type`  | Kubernetes Service type. |ClusterIP|
-| `service.mqtt`  | Port for MQTT. |1883|
-| `service.mqttssl` | Port for MQTT(SSL). |8883|
-| `service.mgmt`  | Port for mgmt API. |8081|
-| `service.ws`  | Port for WebSocket/HTTP. |8083|
-| `service.wss`  | Port for WSS/HTTPS. |8084|
-| `service.dashboard`  | Port for dashboard. |18083|
-| `service.nodePorts.mqtt`  | Kubernetes node port for MQTT. |nil|
-| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). |nil|
-| `service.nodePorts.mgmt`  | Kubernetes node port for mgmt API. |nil|
-| `service.nodePorts.ws`  | Kubernetes node port for WebSocket/HTTP. |nil|
-| `service.nodePorts.wss`  | Kubernetes node port for WSS/HTTPS. |nil|
-| `service.nodePorts.dashboard`  | Kubernetes node port for dashboard. |nil|
-| `service.loadBalancerIP`  | loadBalancerIP for Service |	nil |
-| `service.loadBalancerSourceRanges` |	Address(es) that are allowed when service is LoadBalancer |	[] |
-| `service.externalIPs` |      ExternalIPs for the service |   [] |
-| `service.annotations` |	Service annotations |	{}(evaluated as a template)|
-| `ingress.dashboard.enabled` |	Enable ingress for EMQX Dashboard |	false |
-| `ingress.dashboard.path` | Ingress path for EMQX Dashboard |	/ |
-| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API |	dashboard.emqx.local |
-| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API |	[] |
-| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API |	{} |
-| `ingress.mgmt.enabled` |	Enable ingress for EMQX Mgmt API |	false |
-| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API |	/ |
-| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API |	api.emqx.local |
-| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API |	[] |
-| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API |	{} |
-| `emqxConfig` | Emqx configuration item, see the [documentation](https://hub.docker.com/r/emqx/emqx) | |
-| `emqxAclConfig` | Emqx acl configuration item, see the [documentation](https://docs.emqx.io/broker/latest/en/advanced/acl-file.html) | |
+## EMQx-specific
+The following table lists the configurable EMQx parameters of the EMQx chart and their default values.
+| Parameter  | Description | Default Value
+| ---        |  ---        | ---
+| `emqxConfig` | [Global configuration](https://hub.docker.com/r/emqx/emqx) items | `nil`
+| `emqxLicenseSecretName` | Name of the secret that holds the license information | `nil`
+| `emqxAclConfig` | [ACL]((https://docs.emqx.io/broker/latest/en/advanced/acl-file.html)) configuration | `{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. {allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. {allow, all}.`
+| `emqxLoadedModules` | Modules to load on start | `{emqx_mod_acl_internal, true}. {emqx_mod_presence, true}. {emqx_mod_delayed, false}. {emqx_mod_rewrite, false}. {emqx_mod_subscription, false}. {emqx_mod_topic_metrics, false}.`
+| `emqxLoadedPlugins` | Plugins to load on start | `{emqx_management, true}. {emqx_recon, true}. {emqx_retainer, true}. {emqx_dashboard, true}. {emqx_telemetry, true}. {emqx_rule_engine, true}. {emqx_bridge_mqtt, false}.`
\ No newline at end of file
diff --git a/deploy/charts/emqx/templates/ingress.yaml b/deploy/charts/emqx/templates/ingress.dashboard.yaml
similarity index 50%
rename from deploy/charts/emqx/templates/ingress.yaml
rename to deploy/charts/emqx/templates/ingress.dashboard.yaml
index c6d7f7fa1..cb0a53cf0 100644
--- a/deploy/charts/emqx/templates/ingress.yaml
+++ b/deploy/charts/emqx/templates/ingress.dashboard.yaml
@@ -48,53 +48,3 @@ spec:
   {{- end }}
 ---
 {{- end }}
-{{- if .Values.ingress.mgmt.enabled -}}
-{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1
-{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
-apiVersion: networking.k8s.io/v1beta1
-{{- else -}}
-apiVersion: extensions/v1beta1
-{{- end }}
-kind: Ingress
-metadata:
-  name: {{ printf "%s-%s" (include "emqx.fullname" .) "mgmt" }}
-  labels:
-    app.kubernetes.io/name: {{ include "emqx.name" . }}
-    helm.sh/chart: {{ include "emqx.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-  {{- if .Values.ingress.mgmt.annotations }}
-  annotations:
-    {{- toYaml .Values.ingress.mgmt.annotations | nindent 4 }}
-  {{- end }}
-spec:
-{{- if and .Values.ingress.mgmt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
-  ingressClassName: {{ .Values.ingress.mgmt.ingressClassName }}
-{{- end }}
-  rules:
-  {{- range $host := .Values.ingress.mgmt.hosts }}
-  - host: {{ $host }}
-    http:
-      paths:
-      - path: /
-        {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
-        pathType: ImplementationSpecific
-        {{- end }}
-        backend:
-          {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
-          service:
-            name: {{ include "emqx.fullname" $ }}
-            port:
-              number: {{ $.Values.service.mgmt }}
-          {{- else }}
-          serviceName: {{ include "emqx.fullname" $ }}
-          servicePort: {{ $.Values.service.mgmt }}
-          {{- end }}
-  {{- end -}}
-  {{- if .Values.ingress.mgmt.tls }}
-  tls:
-    {{- toYaml .Values.ingress.mgmt.tls | nindent 4 }}
-  {{- end }}
----
-{{- end }}
\ No newline at end of file
diff --git a/deploy/charts/emqx/templates/ingress.mgmt.yaml b/deploy/charts/emqx/templates/ingress.mgmt.yaml
new file mode 100644
index 000000000..bf4158955
--- /dev/null
+++ b/deploy/charts/emqx/templates/ingress.mgmt.yaml
@@ -0,0 +1,49 @@
+{{- if .Values.ingress.mgmt.enabled -}}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ printf "%s-%s" (include "emqx.fullname" .) "mgmt" }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- if .Values.ingress.mgmt.annotations }}
+  annotations:
+    {{- toYaml .Values.ingress.mgmt.annotations | nindent 4 }}
+  {{- end }}
+spec:
+{{- if and .Values.ingress.mgmt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingress.mgmt.ingressClassName }}
+{{- end }}
+  rules:
+  {{- range $host := .Values.ingress.mgmt.hosts }}
+  - host: {{ $host }}
+    http:
+      paths:
+      - path: {{ $.Values.ingress.mgmt.path | default "/" }}
+        {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+        pathType: ImplementationSpecific
+        {{- end }}
+        backend:
+          {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ include "emqx.fullname" $ }}
+            port:
+              number: {{ $.Values.service.mgmt }}
+          {{- else }}
+          serviceName: {{ include "emqx.fullname" $ }}
+          servicePort: {{ $.Values.service.mgmt }}
+          {{- end }}
+  {{- end -}}
+  {{- if .Values.ingress.mgmt.tls }}
+  tls:
+    {{- toYaml .Values.ingress.mgmt.tls | nindent 4 }}
+  {{- end }}
+{{- end }}
diff --git a/deploy/charts/emqx/templates/ingress.wss.yaml b/deploy/charts/emqx/templates/ingress.wss.yaml
new file mode 100644
index 000000000..88d91272b
--- /dev/null
+++ b/deploy/charts/emqx/templates/ingress.wss.yaml
@@ -0,0 +1,49 @@
+{{- if .Values.ingress.wss.enabled -}}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ printf "%s-%s" (include "emqx.fullname" .) "wss" }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- if .Values.ingress.wss.annotations }}
+  annotations:
+    {{- toYaml .Values.ingress.wss.annotations | nindent 4 }}
+  {{- end }}
+spec:
+{{- if and .Values.ingress.wss.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingress.wss.ingressClassName }}
+{{- end }}
+  rules:
+  {{- range $host := .Values.ingress.wss.hosts }}
+  - host: {{ $host }}
+    http:
+      paths:
+      - path: {{ $.Values.ingress.wss.path | default "/mqtt" }}
+        {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+        pathType: ImplementationSpecific
+        {{- end }}
+        backend:
+          {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ include "emqx.fullname" $ }}
+            port:
+              number: {{ $.Values.service.ws }}
+          {{- else }}
+          serviceName: {{ include "emqx.fullname" $ }}
+          servicePort: {{ $.Values.service.ws }}
+          {{- end }}
+  {{- end -}}
+  {{- if .Values.ingress.wss.tls }}
+  tls:
+    {{- toYaml .Values.ingress.wss.tls | nindent 4 }}
+  {{- end }}
+{{- end }}
diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index 4b1526cc9..39c158610 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -187,6 +187,29 @@ ingress:
     - api.emqx.local
     tls: []
 
+  ## ingress for EMQX Mgmt API
+  wss:
+    enabled: false
+    # ingressClassName: nginx
+    annotations: {}
+      # Sample annotations for nginx-ingress community controller
+      # nginx.ingress.kubernetes.io/rewrite-target: /mqtt$1 # Use to rewrite backend path if needed
+      # nginx.ingress.kubernetes.io/backend-protocol: "http"
+      # nginx.ingress.kubernetes.io/use-forwarded-headers: "true"
+      # nginx.ingress.kubernetes.io/enable-real-ip: "true"
+      # nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
+      # nginx.ingress.kubernetes.io/proxy-connect-timeout: "120"
+      # nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+      # nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+      # nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+      # nginx.ingress.kubernetes.io/use-proxy-protocol: "false"
+      # nginx.ingress.kubernetes.io/proxy-protocol-header-timeout: "5s"
+    path: /mqtt
+    # path: /wss(\/.*)?
+    hosts:
+    - wss.emqx.local
+    tls: []
+
 podSecurityContext:
   enabled: true
   fsGroup: 1000

From 6a5dc485e2c733f9ab89512058b6f5336edaf9a7 Mon Sep 17 00:00:00 2001
From: Fernando Almeida <nanditu@gmail.com>
Date: Tue, 25 Jan 2022 13:06:38 +0000
Subject: [PATCH 02/13] feat(helm-chart): split configs into multiple files for
 readability feat(helm-chart): added conditional volumes in StatefulSet
 feat(helm-chart): revised checksum logic to include all config

---
 deploy/charts/emqx/templates/StatefulSet.yaml | 21 ++++++-
 .../charts/emqx/templates/configmap.acl.yaml  | 15 +++++
 .../charts/emqx/templates/configmap.env.yaml  | 14 +++++
 .../templates/configmap.loadedModules.yaml    | 15 +++++
 .../templates/configmap.loadedPlugins.yaml    | 15 +++++
 deploy/charts/emqx/templates/configmap.yaml   | 59 -------------------
 6 files changed, 78 insertions(+), 61 deletions(-)
 create mode 100644 deploy/charts/emqx/templates/configmap.acl.yaml
 create mode 100644 deploy/charts/emqx/templates/configmap.env.yaml
 create mode 100644 deploy/charts/emqx/templates/configmap.loadedModules.yaml
 create mode 100644 deploy/charts/emqx/templates/configmap.loadedPlugins.yaml
 delete mode 100644 deploy/charts/emqx/templates/configmap.yaml

diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml
index 3c2ff54f2..8df226335 100644
--- a/deploy/charts/emqx/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx/templates/StatefulSet.yaml
@@ -1,3 +1,8 @@
+{{ $configData := printf "%s\n%s\n%s\n%s"
+  (include (print $.Template.BasePath "/configmap.env.yaml") .)
+  (include (print $.Template.BasePath "/configmap.acl.yaml") .)
+  (include (print $.Template.BasePath "/configmap.loadedPlugins.yaml") .)
+  (include (print $.Template.BasePath "/configmap.loadedModules.yaml") .) }}
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -50,28 +55,34 @@ spec:
         app.kubernetes.io/instance: {{ .Release.Name }}
       {{- if .Values.recreatePods }}
       annotations:
-        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }}
+        checksum/config: {{ $configData | sha256sum | quote }}
       {{- end }}
     spec:
       volumes:
+      {{- if .Values.emqxLoadedPlugins }}
       - name: emqx-loaded-plugins
         configMap:
           name: {{ include "emqx.fullname" . }}-loaded-plugins
           items:
           - key: loaded_plugins
             path: loaded_plugins
+      {{- end }}
+      {{- if .Values.emqxLoadedPlugins }}
       - name: emqx-loaded-modules
         configMap:
           name: {{ include "emqx.fullname" . }}-loaded-modules
           items:
           - key: loaded_modules
             path: loaded_modules
+      {{- end }}
+      {{- if .Values.emqxAclConfig }}
       - name: emqx-acl
         configMap:
           name: {{ include "emqx.fullname" . }}-acl
           items:
           - key: acl.conf
             path: acl.conf
+      {{- end }}
       {{- if not .Values.persistence.enabled }}
       - name: emqx-data
         emptyDir: {}
@@ -155,16 +166,22 @@ spec:
           volumeMounts:
           - name: emqx-data
             mountPath: "/opt/emqx/data"
+          {{- if .Values.emqxAclConfig }}
           - name: emqx-acl
             mountPath: "/opt/emqx/etc/acl.conf"
             subPath: "acl.conf"
+          {{- end }}
+          {{- if .Values.emqxLoadedPlugins }}
           - name: emqx-loaded-plugins
             mountPath: "/opt/emqx/data/loaded_plugins"
             subPath: "loaded_plugins"
+          {{- end }}
+          {{- if .Values.emqxLoadedModules }}
           - name: emqx-loaded-modules
             mountPath: "/opt/emqx/data/loaded_modules"
             subPath: "loaded_modules"
-          {{ if .Values.emqxLicenseSecretName  }}
+          {{- end }}
+          {{- if .Values.emqxLicenseSecretName  }}
           - name: emqx-license
             mountPath: "/opt/emqx/etc/emqx.lic"
             subPath: "emqx.lic"
diff --git a/deploy/charts/emqx/templates/configmap.acl.yaml b/deploy/charts/emqx/templates/configmap.acl.yaml
new file mode 100644
index 000000000..897ea0f48
--- /dev/null
+++ b/deploy/charts/emqx/templates/configmap.acl.yaml
@@ -0,0 +1,15 @@
+{{ if .Values.emqxAclConfig }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "emqx.fullname" . }}-acl
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  "acl.conf": |
+    {{ .Values.emqxAclConfig }}
+{{ end }}
\ No newline at end of file
diff --git a/deploy/charts/emqx/templates/configmap.env.yaml b/deploy/charts/emqx/templates/configmap.env.yaml
new file mode 100644
index 000000000..ffd1b66dc
--- /dev/null
+++ b/deploy/charts/emqx/templates/configmap.env.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "emqx.fullname" . }}-env
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  {{- range $index, $value := .Values.emqxConfig}}
+    {{$index}}: "{{ $value }}" 
+  {{- end}}
diff --git a/deploy/charts/emqx/templates/configmap.loadedModules.yaml b/deploy/charts/emqx/templates/configmap.loadedModules.yaml
new file mode 100644
index 000000000..099db396a
--- /dev/null
+++ b/deploy/charts/emqx/templates/configmap.loadedModules.yaml
@@ -0,0 +1,15 @@
+{{ if .Values.emqxLoadedModules }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "emqx.fullname" . }}-loaded-modules
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  "loaded_modules": |
+    {{ .Values.emqxLoadedModules }}
+{{ end }}
\ No newline at end of file
diff --git a/deploy/charts/emqx/templates/configmap.loadedPlugins.yaml b/deploy/charts/emqx/templates/configmap.loadedPlugins.yaml
new file mode 100644
index 000000000..df77368f9
--- /dev/null
+++ b/deploy/charts/emqx/templates/configmap.loadedPlugins.yaml
@@ -0,0 +1,15 @@
+{{ if .Values.emqxLoadedPlugins }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "emqx.fullname" . }}-loaded-plugins
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  "loaded_plugins": |
+    {{ .Values.emqxLoadedPlugins }}
+{{ end }}
\ No newline at end of file
diff --git a/deploy/charts/emqx/templates/configmap.yaml b/deploy/charts/emqx/templates/configmap.yaml
deleted file mode 100644
index c9c4b4770..000000000
--- a/deploy/charts/emqx/templates/configmap.yaml
+++ /dev/null
@@ -1,59 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "emqx.fullname" . }}-env
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ include "emqx.name" . }}
-    helm.sh/chart: {{ include "emqx.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-data:
-  {{- range $index, $value := .Values.emqxConfig}}
-    {{$index}}: "{{ $value }}" 
-  {{- end}}
-
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "emqx.fullname" . }}-acl
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ include "emqx.name" . }}
-    helm.sh/chart: {{ include "emqx.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-data:
-  "acl.conf": |
-    {{ .Values.emqxAclConfig }}
-
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "emqx.fullname" . }}-loaded-plugins
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ include "emqx.name" . }}
-    helm.sh/chart: {{ include "emqx.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-data:
-  "loaded_plugins": |
-    {{ .Values.emqxLoadedPlugins }}
-
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "emqx.fullname" . }}-loaded-modules
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ include "emqx.name" . }}
-    helm.sh/chart: {{ include "emqx.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-data:
-  "loaded_modules": |
-    {{ .Values.emqxLoadedModules }}

From 6bb2fa666d2d622b18a966ba739187df4a881b40 Mon Sep 17 00:00:00 2001
From: Fernando Almeida <nanditu@gmail.com>
Date: Tue, 25 Jan 2022 13:14:43 +0000
Subject: [PATCH 03/13] feat(helm-chart): added explicit example for enable
 Websocket SSL.

---
 deploy/charts/emqx/README.md | 159 +++++++++++++++++++++--------------
 1 file changed, 94 insertions(+), 65 deletions(-)

diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index 1d350a896..a233b04e4 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -1,9 +1,9 @@
 # Introduction
-This chart bootstraps an emqx deployment on a Kubernetes (K8s) cluster using the Helm package manager.
+This chart bootstraps an [EMQx](https://www.emqx.io/) deployment on a [Kubernetes](https://kubernetes.io/) (K8s) cluster using the [Helm](https://helm.sh/) package manager.
 
 # Prerequisites
-+ Kubernetes 1.6+
-+ Helm
++ [Kubernetes](https://kubernetes.io/) 1.6+
++ [Helm](https://helm.sh/)
 
 # Installing the Chart
 To install the chart with the release name `my-emqx`:
@@ -25,73 +25,102 @@ To install the chart with the release name `my-emqx`:
 # Uninstalling the Chart
 To uninstall/delete the `my-emqx` deployment:
 ```
-$ helm del  my-emqx
+$ helm del my-emqx
 ```
 
 # Configuration
 The following sections describe the configurable parameters of the EMQx chart and their default values.
-## Kubernetes-specific
+## K8s-specific
 The following table lists the configurable K8s parameters of the EMQx chart and their default values.
-| Parameter  | Description | Default Value
-| ---        |  ---        | ---
-| `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | `3`
-| `image.repository` | EMQ X Image name | `emqx/emqx`
-| `image.pullPolicy`  | The image pull policy  | `IfNotPresent`
-| `image.pullSecrets `  | The image pull secrets (does not add image pull secrets to deployed pods)  |``[]``
-| `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | `nil`
-| `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | `false`
-| `persistence.enabled` | Enable EMQX persistence using PVC | `false`
-| `persistence.storageClass` | Storage class of backing PVC (uses alpha storage class annotation) | `nil`
-| `persistence.existingClaim` | EMQ X data Persistent Volume existing claim name, evaluated as a template | `""`
-| `persistence.accessMode` | PVC Access Mode for EMQX volume | `ReadWriteOnce`
-| `persistence.size` | PVC Storage Request for EMQX volume | `20Mi`
-| `initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`
-| `resources` | CPU/Memory resource requests/limits |`{}`
-| `nodeSelector` | Node labels for pod assignment |`{}`
-| `tolerations` | Toleration labels for pod assignment |``[]``
-| `affinity` | Map of node/pod affinities |`{}`
-| `service.type`  | Kubernetes Service type. | `ClusterIP`
-| `service.mqtt`  | Port for MQTT. | `1883`
-| `service.mqttssl` | Port for MQTT(SSL). | `8883`
-| `service.mgmt`  | Port for mgmt API. | `8081`
-| `service.ws`  | Port for WebSocket/HTTP. | `8083`
-| `service.wss`  | Port for WSS/HTTPS. | `8084`
-| `service.dashboard`  | Port for dashboard. | `18083`
-| `service.nodePorts.mqtt`  | Kubernetes node port for MQTT. | `nil`
-| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | `nil`
-| `service.nodePorts.mgmt`  | Kubernetes node port for mgmt API. | `nil`
-| `service.nodePorts.ws`  | Kubernetes node port for WebSocket/HTTP. | `nil`
-| `service.nodePorts.wss`  | Kubernetes node port for WSS/HTTPS. | `nil`
-| `service.nodePorts.dashboard`  | Kubernetes node port for dashboard. | `nil`
-| `service.loadBalancerIP`  | loadBalancerIP for Service |	`nil`
-| `service.loadBalancerSourceRanges` |	Address(es) that are allowed when service is LoadBalancer |	`[]`
-| `service.externalIPs` |	ExternalIPs for the service |	`[]`
-| `service.annotations` |	Service annotations (evaluated as a template) |	`{}`
-| `ingress.dashboard.enabled` |	Enable ingress for EMQX Dashboard |	false
-| `ingress.dashboard.ingressClassName` |	Set the ingress class for EMQX Dashboard
-| `ingress.dashboard.path` | Ingress path for EMQX Dashboard |	`/`
-| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API |	dashboard.emqx.local
-| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API |	`[]`
-| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API |	`{}`
-| `ingress.mgmt.enabled` |	Enable ingress for EMQX Mgmt API |	`false`
-| `ingress.mqtt.ingressClassName` |	Set the ingress class for EMQX Mgmt API | `nil`
-| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | `/`
-| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API |	`api.emqx.local`
-| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API |	`[]`
-| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API |	`{}`
-| `ingress.wss.enabled` |	Enable ingress for EMQX Mgmt API |	`false`
-| `ingress.wss.ingressClassName` |	Set the ingress class for EMQX Mgmt API | `nil`
-| `ingress.wss.path` | Ingress path for EMQX WSS |	`/`
-| `ingress.wss.hosts` | Ingress hosts for EMQX WSS |    `wss.emqx.local`
-| `ingress.wss.tls` | Ingress tls for EMQX WSS |	`[]`
-| `ingress.wss.annotations` | Ingress annotations for EMQX WSS |	`{}`
+Parameter  | Description | Default Value
+---        |  ---        | ---
+`replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | `3`
+`image.repository` | EMQ X Image name | `emqx/emqx`
+`image.pullPolicy`  | The image pull policy  | `IfNotPresent`
+`image.pullSecrets `  | The image pull secrets (does not add image pull secrets to deployed pods)  |``[]``
+`envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | `nil`
+`recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | `false`
+`persistence.enabled` | Enable EMQX persistence using PVC | `false`
+`persistence.storageClass` | Storage class of backing PVC (uses alpha storage class annotation) | `nil`
+`persistence.existingClaim` | EMQ X data Persistent Volume existing claim name, evaluated as a template | `""`
+`persistence.accessMode` | PVC Access Mode for EMQX volume | `ReadWriteOnce`
+`persistence.size` | PVC Storage Request for EMQX volume | `20Mi`
+`initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`
+`resources` | CPU/Memory resource requests/limits |`{}`
+`nodeSelector` | Node labels for pod assignment |`{}`
+`tolerations` | Toleration labels for pod assignment |``[]``
+`affinity` | Map of node/pod affinities |`{}`
+`service.type`  | Kubernetes Service type. | `ClusterIP`
+`service.mqtt`  | Port for MQTT. | `1883`
+`service.mqttssl` | Port for MQTT(SSL). | `8883`
+`service.mgmt`  | Port for mgmt API. | `8081`
+`service.ws`  | Port for WebSocket/HTTP. | `8083`
+`service.wss`  | Port for WSS/HTTPS. | `8084`
+`service.dashboard`  | Port for dashboard. | `18083`
+`service.nodePorts.mqtt`  | Kubernetes node port for MQTT. | `nil`
+`service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | `nil`
+`service.nodePorts.mgmt`  | Kubernetes node port for mgmt API. | `nil`
+`service.nodePorts.ws`  | Kubernetes node port for WebSocket/HTTP. | `nil`
+`service.nodePorts.wss`  | Kubernetes node port for WSS/HTTPS. | `nil`
+`service.nodePorts.dashboard`  | Kubernetes node port for dashboard. | `nil`
+`service.loadBalancerIP`  | loadBalancerIP for Service |	`nil`
+`service.loadBalancerSourceRanges` |	Address(es) that are allowed when service is LoadBalancer |	`[]`
+`service.externalIPs` |	ExternalIPs for the service |	`[]`
+`service.annotations` |	Service annotations (evaluated as a template) |	`{}`
+`ingress.dashboard.enabled` |	Enable ingress for EMQX Dashboard |	false
+`ingress.dashboard.ingressClassName` |	Set the ingress class for EMQX Dashboard
+`ingress.dashboard.path` | Ingress path for EMQX Dashboard |	`/`
+`ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API |	dashboard.emqx.local
+`ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API |	`[]`
+`ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API |	`{}`
+`ingress.mgmt.enabled` |	Enable ingress for EMQX Mgmt API |	`false`
+`ingress.mqtt.ingressClassName` |	Set the ingress class for EMQX Mgmt API | `nil`
+`ingress.mgmt.path` | Ingress path for EMQX Mgmt API | `/`
+`ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API |	`api.emqx.local`
+`ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API |	`[]`
+`ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API |	`{}`
+`ingress.wss.enabled` |	Enable ingress for EMQX Mgmt API |	`false`
+`ingress.wss.ingressClassName` |	Set the ingress class for EMQX Mgmt API | `nil`
+`ingress.wss.path` | Ingress path for EMQX WSS |	`/`
+`ingress.wss.hosts` | Ingress hosts for EMQX WSS |    `wss.emqx.local`
+`ingress.wss.tls` | Ingress tls for EMQX WSS |	`[]`
+`ingress.wss.annotations` | Ingress annotations for EMQX WSS |	`{}`
 
 ## EMQx-specific
 The following table lists the configurable EMQx parameters of the EMQx chart and their default values.
-| Parameter  | Description | Default Value
-| ---        |  ---        | ---
-| `emqxConfig` | [Global configuration](https://hub.docker.com/r/emqx/emqx) items | `nil`
-| `emqxLicenseSecretName` | Name of the secret that holds the license information | `nil`
-| `emqxAclConfig` | [ACL]((https://docs.emqx.io/broker/latest/en/advanced/acl-file.html)) configuration | `{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. {allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. {allow, all}.`
-| `emqxLoadedModules` | Modules to load on start | `{emqx_mod_acl_internal, true}. {emqx_mod_presence, true}. {emqx_mod_delayed, false}. {emqx_mod_rewrite, false}. {emqx_mod_subscription, false}. {emqx_mod_topic_metrics, false}.`
-| `emqxLoadedPlugins` | Plugins to load on start | `{emqx_management, true}. {emqx_recon, true}. {emqx_retainer, true}. {emqx_dashboard, true}. {emqx_telemetry, true}. {emqx_rule_engine, true}. {emqx_bridge_mqtt, false}.`
\ No newline at end of file
+Parameter  | Description | Default Value
+---        |  ---        | ---
+`emqxConfig` | Emqx configuration item, see the [documentation](https://hub.docker.com/r/emqx/emqx) | `nil`
+`emqxLicenseSecretName` | Name of the secret that holds the license information | `nil`
+`emqxAclConfig` | EMQx ACL configuration | `{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. {allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. {allow, all}.`
+`emqxLoadedModules` | Modules to load on start | `{emqx_mod_acl_internal, true}. {emqx_mod_presence, true}. {emqx_mod_delayed, false}. {emqx_mod_rewrite, false}. {emqx_mod_subscription, false}. {emqx_mod_topic_metrics, false}.`
+`emqxLoadedPlugins` | Plugins to load on start | `{emqx_management, true}. {emqx_recon, true}. {emqx_retainer, true}. {emqx_dashboard, true}. {emqx_telemetry, true}. {emqx_rule_engine, true}. {emqx_bridge_mqtt, false}.`
+
+# Examples
+This section provides some examples for the configuration of common scenarios.
+## Enable Websockets SSL via [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/)
+The following settings describe a working scenario for acessing EMQx Websockets with SSL termination at the [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/).
+```yaml
+ingress:
+  wss:
+    enabled: false
+    # ingressClassName: nginx
+    annotations:
+      nginx.ingress.kubernetes.io/backend-protocol: "http"
+      nginx.ingress.kubernetes.io/use-forwarded-headers: "true"
+      nginx.ingress.kubernetes.io/enable-real-ip: "true"
+      nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
+      nginx.ingress.kubernetes.io/proxy-connect-timeout: "120"
+      nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+      nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+      nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+      nginx.ingress.kubernetes.io/use-proxy-protocol: "false"
+      nginx.ingress.kubernetes.io/proxy-protocol-header-timeout: "5s"
+    path: /mqtt
+    hosts:
+    - myhost.example.com
+    tls:
+    - hosts:
+        - myhost.example.com
+      secretName: myhost-example-com-tls # Name of the secret that holds the certificates for the domain
+```

From a24e76535182b09317ff30c6ae005e61c59e8018 Mon Sep 17 00:00:00 2001
From: Fernando Almeida <nanditu@gmail.com>
Date: Tue, 25 Jan 2022 13:57:23 +0000
Subject: [PATCH 04/13] feat(helm-chart): added support for shared ingress
 annotations feat(helm-chart): revised README to include image.tag setting

---
 deploy/charts/emqx/README.md                        | 3 ++-
 deploy/charts/emqx/templates/StatefulSet.yaml       | 4 ++++
 deploy/charts/emqx/templates/ingress.dashboard.yaml | 7 ++++++-
 deploy/charts/emqx/templates/ingress.mgmt.yaml      | 7 ++++++-
 deploy/charts/emqx/templates/ingress.wss.yaml       | 7 ++++++-
 deploy/charts/emqx/values.yaml                      | 3 +++
 6 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index a233b04e4..e066a9328 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -35,7 +35,8 @@ The following table lists the configurable K8s parameters of the EMQx chart and
 Parameter  | Description | Default Value
 ---        |  ---        | ---
 `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | `3`
-`image.repository` | EMQ X Image name | `emqx/emqx`
+`image.tag` | EMQ X Image tag (defaults to `.Chart.AppVersion`) | `nil`
+`image.repository` | EMQ X Image repository | `emqx/emqx`
 `image.pullPolicy`  | The image pull policy  | `IfNotPresent`
 `image.pullSecrets `  | The image pull secrets (does not add image pull secrets to deployed pods)  |``[]``
 `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | `nil`
diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml
index 8df226335..160a7fce4 100644
--- a/deploy/charts/emqx/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx/templates/StatefulSet.yaml
@@ -119,7 +119,11 @@ spec:
       {{- end }}
       containers:
         - name: emqx
+          {{- if .Values.image.tag }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          {{- else -}}}}
           image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}"
+          {{- end }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           {{- if .Values.containerSecurityContext.enabled }}
           securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }}
diff --git a/deploy/charts/emqx/templates/ingress.dashboard.yaml b/deploy/charts/emqx/templates/ingress.dashboard.yaml
index cb0a53cf0..9ea06cdc4 100644
--- a/deploy/charts/emqx/templates/ingress.dashboard.yaml
+++ b/deploy/charts/emqx/templates/ingress.dashboard.yaml
@@ -14,10 +14,15 @@ metadata:
     helm.sh/chart: {{ include "emqx.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
-  {{- if .Values.ingress.dashboard.annotations }}
+  {{- if or .Values.ingress.annotations .Values.ingress.dashboard.annotations }}
   annotations:
+  {{- if .Values.ingress.annotations }}
+    {{- toYaml .Values.ingress.annotations | nindent 4 }}
+  {{- end }}
+  {{- if .Values.ingress.dashboard.annotations }}
     {{- toYaml .Values.ingress.dashboard.annotations | nindent 4 }}
   {{- end }}
+  {{- end }}
 spec:
 {{- if and .Values.ingress.dashboard.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
   ingressClassName: {{ .Values.ingress.dashboard.ingressClassName }}
diff --git a/deploy/charts/emqx/templates/ingress.mgmt.yaml b/deploy/charts/emqx/templates/ingress.mgmt.yaml
index bf4158955..c9abe79a9 100644
--- a/deploy/charts/emqx/templates/ingress.mgmt.yaml
+++ b/deploy/charts/emqx/templates/ingress.mgmt.yaml
@@ -14,10 +14,15 @@ metadata:
     helm.sh/chart: {{ include "emqx.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
-  {{- if .Values.ingress.mgmt.annotations }}
+  {{- if or .Values.ingress.annotations .Values.ingress.mgmt.annotations }}
   annotations:
+  {{- if .Values.ingress.annotations }}
+    {{- toYaml .Values.ingress.annotations | nindent 4 }}
+  {{- end }}
+  {{- if .Values.ingress.mgmt.annotations }}
     {{- toYaml .Values.ingress.mgmt.annotations | nindent 4 }}
   {{- end }}
+  {{- end }}
 spec:
 {{- if and .Values.ingress.mgmt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
   ingressClassName: {{ .Values.ingress.mgmt.ingressClassName }}
diff --git a/deploy/charts/emqx/templates/ingress.wss.yaml b/deploy/charts/emqx/templates/ingress.wss.yaml
index 88d91272b..7c7b395a5 100644
--- a/deploy/charts/emqx/templates/ingress.wss.yaml
+++ b/deploy/charts/emqx/templates/ingress.wss.yaml
@@ -14,10 +14,15 @@ metadata:
     helm.sh/chart: {{ include "emqx.chart" . }}
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
-  {{- if .Values.ingress.wss.annotations }}
+  {{- if or .Values.ingress.annotations .Values.ingress.wss.annotations }}
   annotations:
+  {{- if .Values.ingress.annotations }}
+    {{- toYaml .Values.ingress.annotations | nindent 4 }}
+  {{- end }}
+  {{- if .Values.ingress.wss.annotations }}
     {{- toYaml .Values.ingress.wss.annotations | nindent 4 }}
   {{- end }}
+  {{- end }}
 spec:
 {{- if and .Values.ingress.wss.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
   ingressClassName: {{ .Values.ingress.wss.ingressClassName }}
diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index 39c158610..90b5ede98 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -165,6 +165,9 @@ tolerations: []
 affinity: {}
 
 ingress:
+  ## Ingress shared annotations
+  annotations: {}
+
   ## ingress for EMQX Dashboard
   dashboard:
     enabled: false

From 43624d6407ea04c1dca12588d68eb1b3c9bd5114 Mon Sep 17 00:00:00 2001
From: Fernando Almeida <nanditu@gmail.com>
Date: Tue, 25 Jan 2022 16:02:56 +0000
Subject: [PATCH 05/13] feat(helm-chart): revised emqx env ConfigMap to allow
 specifying settings also using the namespaced dot notation feat(helm-chart):
 updated README to include description of previous change and fix value in WSS
 example

---
 deploy/charts/emqx/README.md                    | 16 ++++++++--------
 deploy/charts/emqx/templates/configmap.env.yaml | 10 ++++++++--
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index e066a9328..3b5d66a7f 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -29,9 +29,9 @@ $ helm del my-emqx
 ```
 
 # Configuration
-The following sections describe the configurable parameters of the EMQx chart and their default values.
+The following sections describe the configurable parameters of the chart and their default values.
 ## K8s-specific
-The following table lists the configurable K8s parameters of the EMQx chart and their default values.
+The following table lists the configurable K8s parameters of the [EMQx](https://www.emqx.io/) chart and their default values.
 Parameter  | Description | Default Value
 ---        |  ---        | ---
 `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | `3`
@@ -88,19 +88,19 @@ Parameter  | Description | Default Value
 `ingress.wss.annotations` | Ingress annotations for EMQX WSS |	`{}`
 
 ## EMQx-specific
-The following table lists the configurable EMQx parameters of the EMQx chart and their default values.
+The following table lists the configurable [EMQx](https://www.emqx.io/)-specific parameters of the chart and their default values.
 Parameter  | Description | Default Value
 ---        |  ---        | ---
-`emqxConfig` | Emqx configuration item, see the [documentation](https://hub.docker.com/r/emqx/emqx) | `nil`
+`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix can be omitted) or using the configuration files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil`
 `emqxLicenseSecretName` | Name of the secret that holds the license information | `nil`
-`emqxAclConfig` | EMQx ACL configuration | `{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. {allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. {allow, all}.`
-`emqxLoadedModules` | Modules to load on start | `{emqx_mod_acl_internal, true}. {emqx_mod_presence, true}. {emqx_mod_delayed, false}. {emqx_mod_rewrite, false}. {emqx_mod_subscription, false}. {emqx_mod_topic_metrics, false}.`
-`emqxLoadedPlugins` | Plugins to load on start | `{emqx_management, true}. {emqx_recon, true}. {emqx_retainer, true}. {emqx_dashboard, true}. {emqx_telemetry, true}. {emqx_rule_engine, true}. {emqx_bridge_mqtt, false}.`
+`emqxAclConfig` | [ACL](https://docs.emqx.io/broker/latest/en/advanced/acl-file.html) configuration | `{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. {allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. {allow, all}.`
+`emqxLoadedModules` | Modules to load on startup | `{emqx_mod_acl_internal, true}. {emqx_mod_presence, true}. {emqx_mod_delayed, false}. {emqx_mod_rewrite, false}. {emqx_mod_subscription, false}. {emqx_mod_topic_metrics, false}.`
+`emqxLoadedPlugins` | Plugins to load on startup | `{emqx_management, true}. {emqx_recon, true}. {emqx_retainer, true}. {emqx_dashboard, true}. {emqx_telemetry, true}. {emqx_rule_engine, true}. {emqx_bridge_mqtt, false}.`
 
 # Examples
 This section provides some examples for the configuration of common scenarios.
 ## Enable Websockets SSL via [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/)
-The following settings describe a working scenario for acessing EMQx Websockets with SSL termination at the [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/).
+The following settings describe a working scenario for acessing [EMQx](https://www.emqx.io/) Websockets with SSL termination at the [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/).
 ```yaml
 ingress:
   wss:
diff --git a/deploy/charts/emqx/templates/configmap.env.yaml b/deploy/charts/emqx/templates/configmap.env.yaml
index ffd1b66dc..61674e853 100644
--- a/deploy/charts/emqx/templates/configmap.env.yaml
+++ b/deploy/charts/emqx/templates/configmap.env.yaml
@@ -1,3 +1,4 @@
+{{- if .Values.emqxConfig }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -9,6 +10,11 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 data:
-  {{- range $index, $value := .Values.emqxConfig}}
-    {{$index}}: "{{ $value }}" 
+  {{- range $index, $value := .Values.emqxConfig }}
+    {{- if ne $value nil }}
+      {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}
+      {{ print "EMQX_" $key }}: {{ $value | quote }}
+    {{- end }}
   {{- end}}
+
+{{- end }}
\ No newline at end of file

From 36b5982630ebd441d74b591b4f5307dcd86faf43 Mon Sep 17 00:00:00 2001
From: Fernando Almeida <nanditu@gmail.com>
Date: Tue, 25 Jan 2022 16:10:29 +0000
Subject: [PATCH 06/13] feat(helm-chart): fix value in WSS example

---
 deploy/charts/emqx/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index 3b5d66a7f..8b7a82cd9 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -104,7 +104,7 @@ The following settings describe a working scenario for acessing [EMQx](https://w
 ```yaml
 ingress:
   wss:
-    enabled: false
+    enabled: true
     # ingressClassName: nginx
     annotations:
       nginx.ingress.kubernetes.io/backend-protocol: "http"

From 86887e855981a08abf3524974956e087b6ec8d29 Mon Sep 17 00:00:00 2001
From: Fernando Almeida <nanditu@gmail.com>
Date: Tue, 25 Jan 2022 16:46:11 +0000
Subject: [PATCH 07/13] fix(helm-chart): fix indentation bug in env configmap

---
 deploy/charts/emqx/templates/configmap.env.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/deploy/charts/emqx/templates/configmap.env.yaml b/deploy/charts/emqx/templates/configmap.env.yaml
index 61674e853..c27b65343 100644
--- a/deploy/charts/emqx/templates/configmap.env.yaml
+++ b/deploy/charts/emqx/templates/configmap.env.yaml
@@ -11,10 +11,10 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 data:
   {{- range $index, $value := .Values.emqxConfig }}
-    {{- if ne $value nil }}
-      {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}
-      {{ print "EMQX_" $key }}: {{ $value | quote }}
-    {{- end }}
+  {{- if ne $value nil }}
+  {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}
+  {{ print "EMQX_" $key }}: {{ $value | quote }}
+  {{- end }}
   {{- end}}
 
 {{- end }}
\ No newline at end of file

From 73c452c64d10dae698e3c609447fab140bbce232 Mon Sep 17 00:00:00 2001
From: Fernando Almeida <nanditu@gmail.com>
Date: Tue, 25 Jan 2022 23:41:06 +0000
Subject: [PATCH 08/13] fix(helm-chart): replace envFromSecret with more broad
 extraEnvFrom fix(helm-chart): refactored Stateful set for readability.
 fix(helm-chart): added missing extra* variables to StatefulSet and README.
 fix(helm-chart): added sensible defaults for extra* settings.

---
 deploy/charts/emqx/README.md                  |  6 ++-
 deploy/charts/emqx/templates/StatefulSet.yaml | 48 ++++++++++---------
 deploy/charts/emqx/values.yaml                | 20 ++++++++
 3 files changed, 51 insertions(+), 23 deletions(-)

diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index 8b7a82cd9..7a4358be6 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -39,7 +39,6 @@ Parameter  | Description | Default Value
 `image.repository` | EMQ X Image repository | `emqx/emqx`
 `image.pullPolicy`  | The image pull policy  | `IfNotPresent`
 `image.pullSecrets `  | The image pull secrets (does not add image pull secrets to deployed pods)  |``[]``
-`envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | `nil`
 `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | `false`
 `persistence.enabled` | Enable EMQX persistence using PVC | `false`
 `persistence.storageClass` | Storage class of backing PVC (uses alpha storage class annotation) | `nil`
@@ -86,6 +85,11 @@ Parameter  | Description | Default Value
 `ingress.wss.hosts` | Ingress hosts for EMQX WSS |    `wss.emqx.local`
 `ingress.wss.tls` | Ingress tls for EMQX WSS |	`[]`
 `ingress.wss.annotations` | Ingress annotations for EMQX WSS |	`{}`
+`extraEnv` | Aditional container env vars | `[]`
+`extraEnvFrom` | Aditional container env from vars (eg. [config map](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/), [secrets](https://kubernetes.io/docs/concepts/configuration/secret/) | `[]`
+`extraArgs` | Additional container executable arguments | `[]`
+`extraVolumes` | Additional container volumes (eg. for mounting certs from secrets) | `[]`
+`extraVolumeMounts` | Additional container volume mounts (eg. for mounting certs from secrets) | `[]`
 
 ## EMQx-specific
 The following table lists the configurable [EMQx](https://www.emqx.io/)-specific parameters of the chart and their default values.
diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml
index 160a7fce4..73ce5ef77 100644
--- a/deploy/charts/emqx/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx/templates/StatefulSet.yaml
@@ -3,6 +3,10 @@
   (include (print $.Template.BasePath "/configmap.acl.yaml") .)
   (include (print $.Template.BasePath "/configmap.loadedPlugins.yaml") .)
   (include (print $.Template.BasePath "/configmap.loadedModules.yaml") .) }}
+  ## Compatible with previous misspellings
+{{ $licenseSecretName := coalesce .Values.emqxLicenseSecretName .Values.emqxLicneseSecretName }}
+{{ $image := printf "%s:%s" .Values.image.repository (default .Values.image.tag .Chart.AppVersion) }}
+
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -67,7 +71,7 @@ spec:
           - key: loaded_plugins
             path: loaded_plugins
       {{- end }}
-      {{- if .Values.emqxLoadedPlugins }}
+      {{- if .Values.emqxLoadedModules }}
       - name: emqx-loaded-modules
         configMap:
           name: {{ include "emqx.fullname" . }}-loaded-modules
@@ -93,15 +97,13 @@ spec:
           claimName: {{ tpl . $ }}
         {{- end }}
       {{- end }}
-      {{- if .Values.emqxLicenseSecretName  }}
+      {{- if .Values.extraVolumes }}
+{{ toYaml .Values.extraVolumes | indent 6 }}
+      {{- end }}
+      {{- if $licenseSecretName }}
       - name: emqx-license
         secret:
-          secretName: {{ .Values.emqxLicenseSecretName }}
-      ## Compatible with previous misspellings
-      {{- else if .Values.emqxLicneseSecretName }}
-      - name: emqx-license
-        secret:
-          secretName: {{ .Values.emqxLicneseSecretName }}
+          secretName: {{ $licenseSecretName }}
       {{- end }}
       serviceAccountName:  {{ include "emqx.fullname" . }}
       {{- if .Values.podSecurityContext.enabled }}
@@ -119,11 +121,7 @@ spec:
       {{- end }}
       containers:
         - name: emqx
-          {{- if .Values.image.tag }}
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-          {{- else -}}}}
-          image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}"
-          {{- end }}
+          image: {{ $image }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           {{- if .Values.containerSecurityContext.enabled }}
           securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }}
@@ -152,8 +150,11 @@ spec:
           - name: ekka
             containerPort: 4370
           envFrom:
-            - configMapRef:
-                name: {{ include "emqx.fullname" . }}-env
+          - configMapRef:
+              name: {{ include "emqx.fullname" . }}-env
+          {{- if .Values.extraEnvFrom }}
+{{ toYaml .Values.extraEnvFrom | indent 10 }}
+          {{- end }}
           env:
           - name: EMQX_NAME
             value: {{ .Release.Name }}
@@ -165,6 +166,9 @@ spec:
             value: {{ include "emqx.fullname" . }}-headless
           - name: EMQX_CLUSTER__K8S__NAMESPACE
             value: {{ .Release.Namespace }}
+          {{- if .Values.extraEnv }}
+{{ toYaml .Values.extraEnv | indent 10 }}
+          {{- end }}
           resources:
 {{ toYaml .Values.resources | indent 12 }}
           volumeMounts:
@@ -185,18 +189,18 @@ spec:
             mountPath: "/opt/emqx/data/loaded_modules"
             subPath: "loaded_modules"
           {{- end }}
-          {{- if .Values.emqxLicenseSecretName  }}
-          - name: emqx-license
-            mountPath: "/opt/emqx/etc/emqx.lic"
-            subPath: "emqx.lic"
-            readOnly: true
-          ## Compatible with previous misspellings
-          {{ else if .Values.emqxLicneseSecretName  }}
+          {{- if $licenseSecretName }}
           - name: emqx-license
             mountPath: "/opt/emqx/etc/emqx.lic"
             subPath: "emqx.lic"
             readOnly: true
           {{ end }}
+          {{- if and .Values.extraVolumes .Values.extraVolumeMounts }}
+{{ toYaml .Values.extraVolumeMounts | indent 10 }}
+          {{- end }}
+          {{- if .Values.extraArgs }}
+          args: {{ toYaml .Values.extraArgs | nindent 10 }}
+          {{- end }}
           readinessProbe:
             httpGet:
               path: /status
diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index 90b5ede98..d78f19a8e 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -22,6 +22,26 @@ recreatePods: false
 # To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock
 podManagementPolicy: Parallel
 
+## Aditional container env vars
+##
+extraEnv: []
+
+## Aditional container env from vars
+##
+extraEnvFrom: []
+
+## Additional container executable args
+##
+extraArgs: []
+
+## Additional container volumes (eg. for mounting certs from secrets)
+##
+extraVolumes: []
+
+## Additional container volume mounts (eg. for mounting certs from secrets)
+##
+extraVolumeMounts: []
+
 persistence:
   enabled: false
   size: 20Mi

From b91dc1c13d448b3ca465da5cbeafa2a36bb58642 Mon Sep 17 00:00:00 2001
From: Fernando Almeida <nanditu@gmail.com>
Date: Wed, 26 Jan 2022 09:01:25 +0000
Subject: [PATCH 09/13] fix(helm-chart): revised software name to match
 conventioned name fix(helm-chart): added missing new-line from resource
 templates

---
 deploy/charts/emqx/README.md                  | 56 +++++++++----------
 .../charts/emqx/templates/configmap.acl.yaml  |  2 +-
 .../charts/emqx/templates/configmap.env.yaml  |  2 +-
 .../templates/configmap.loadedModules.yaml    |  2 +-
 .../templates/configmap.loadedPlugins.yaml    |  2 +-
 deploy/charts/emqx/templates/rbac.yaml        |  2 +-
 deploy/charts/emqx/values.yaml                | 12 ++--
 7 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index 7a4358be6..4837ffced 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -1,5 +1,5 @@
 # Introduction
-This chart bootstraps an [EMQx](https://www.emqx.io/) deployment on a [Kubernetes](https://kubernetes.io/) (K8s) cluster using the [Helm](https://helm.sh/) package manager.
+This chart bootstraps an [EMQ X](https://www.emqx.io/) deployment on a [Kubernetes](https://kubernetes.io/) (K8s) cluster using the [Helm](https://helm.sh/) package manager.
 
 # Prerequisites
 + [Kubernetes](https://kubernetes.io/) 1.6+
@@ -30,8 +30,8 @@ $ helm del my-emqx
 
 # Configuration
 The following sections describe the configurable parameters of the chart and their default values.
-## K8s-specific
-The following table lists the configurable K8s parameters of the [EMQx](https://www.emqx.io/) chart and their default values.
+## [K8s]((https://kubernetes.io/)) specific settings
+The following table lists the configurable K8s parameters of the [EMQ X](https://www.emqx.io/) chart and their default values.
 Parameter  | Description | Default Value
 ---        |  ---        | ---
 `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | `3`
@@ -40,12 +40,12 @@ Parameter  | Description | Default Value
 `image.pullPolicy`  | The image pull policy  | `IfNotPresent`
 `image.pullSecrets `  | The image pull secrets (does not add image pull secrets to deployed pods)  |``[]``
 `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | `false`
-`persistence.enabled` | Enable EMQX persistence using PVC | `false`
+`persistence.enabled` | Enable EMQ X persistence using PVC | `false`
 `persistence.storageClass` | Storage class of backing PVC (uses alpha storage class annotation) | `nil`
 `persistence.existingClaim` | EMQ X data Persistent Volume existing claim name, evaluated as a template | `""`
-`persistence.accessMode` | PVC Access Mode for EMQX volume | `ReadWriteOnce`
-`persistence.size` | PVC Storage Request for EMQX volume | `20Mi`
-`initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}`
+`persistence.accessMode` | PVC Access Mode for EMQ X volume | `ReadWriteOnce`
+`persistence.size` | PVC Storage Request for EMQ X volume | `20Mi`
+`initContainers` | Containers that run before the creation of EMQ X containers. They can contain utilities or setup scripts. |`{}`
 `resources` | CPU/Memory resource requests/limits |`{}`
 `nodeSelector` | Node labels for pod assignment |`{}`
 `tolerations` | Toleration labels for pod assignment |``[]``
@@ -67,32 +67,32 @@ Parameter  | Description | Default Value
 `service.loadBalancerSourceRanges` |	Address(es) that are allowed when service is LoadBalancer |	`[]`
 `service.externalIPs` |	ExternalIPs for the service |	`[]`
 `service.annotations` |	Service annotations (evaluated as a template) |	`{}`
-`ingress.dashboard.enabled` |	Enable ingress for EMQX Dashboard |	false
-`ingress.dashboard.ingressClassName` |	Set the ingress class for EMQX Dashboard
-`ingress.dashboard.path` | Ingress path for EMQX Dashboard |	`/`
-`ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API |	dashboard.emqx.local
-`ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API |	`[]`
-`ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API |	`{}`
-`ingress.mgmt.enabled` |	Enable ingress for EMQX Mgmt API |	`false`
-`ingress.mqtt.ingressClassName` |	Set the ingress class for EMQX Mgmt API | `nil`
-`ingress.mgmt.path` | Ingress path for EMQX Mgmt API | `/`
-`ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API |	`api.emqx.local`
-`ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API |	`[]`
-`ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API |	`{}`
-`ingress.wss.enabled` |	Enable ingress for EMQX Mgmt API |	`false`
-`ingress.wss.ingressClassName` |	Set the ingress class for EMQX Mgmt API | `nil`
-`ingress.wss.path` | Ingress path for EMQX WSS |	`/`
-`ingress.wss.hosts` | Ingress hosts for EMQX WSS |    `wss.emqx.local`
-`ingress.wss.tls` | Ingress tls for EMQX WSS |	`[]`
-`ingress.wss.annotations` | Ingress annotations for EMQX WSS |	`{}`
+`ingress.dashboard.enabled` |	Enable ingress for EMQ X Dashboard |	false
+`ingress.dashboard.ingressClassName` |	Set the ingress class for EMQ X Dashboard
+`ingress.dashboard.path` | Ingress path for EMQ X Dashboard |	`/`
+`ingress.dashboard.hosts` | Ingress hosts for EMQ X Mgmt API |	dashboard.emqx.local
+`ingress.dashboard.tls` | Ingress tls for EMQ X Mgmt API |	`[]`
+`ingress.dashboard.annotations` | Ingress annotations for EMQ X Mgmt API |	`{}`
+`ingress.mgmt.enabled` |	Enable ingress for EMQ X Mgmt API |	`false`
+`ingress.mqtt.ingressClassName` |	Set the ingress class for EMQ X Mgmt API | `nil`
+`ingress.mgmt.path` | Ingress path for EMQ X Mgmt API | `/`
+`ingress.mgmt.hosts` | Ingress hosts for EMQ X Mgmt API |	`api.emqx.local`
+`ingress.mgmt.tls` | Ingress tls for EMQ X Mgmt API |	`[]`
+`ingress.mgmt.annotations` | Ingress annotations for EMQ X Mgmt API |	`{}`
+`ingress.wss.enabled` |	Enable ingress for EMQ X Mgmt API |	`false`
+`ingress.wss.ingressClassName` |	Set the ingress class for EMQ X Mgmt API | `nil`
+`ingress.wss.path` | Ingress path for EMQ X WSS |	`/`
+`ingress.wss.hosts` | Ingress hosts for EMQ X WSS |    `wss.emqx.local`
+`ingress.wss.tls` | Ingress tls for EMQ X WSS |	`[]`
+`ingress.wss.annotations` | Ingress annotations for EMQ X WSS |	`{}`
 `extraEnv` | Aditional container env vars | `[]`
 `extraEnvFrom` | Aditional container env from vars (eg. [config map](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/), [secrets](https://kubernetes.io/docs/concepts/configuration/secret/) | `[]`
 `extraArgs` | Additional container executable arguments | `[]`
 `extraVolumes` | Additional container volumes (eg. for mounting certs from secrets) | `[]`
 `extraVolumeMounts` | Additional container volume mounts (eg. for mounting certs from secrets) | `[]`
 
-## EMQx-specific
-The following table lists the configurable [EMQx](https://www.emqx.io/)-specific parameters of the chart and their default values.
+## EMQ X specific settings
+The following table lists the configurable [EMQ X](https://www.emqx.io/)-specific parameters of the chart and their default values.
 Parameter  | Description | Default Value
 ---        |  ---        | ---
 `emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix can be omitted) or using the configuration files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil`
@@ -104,7 +104,7 @@ Parameter  | Description | Default Value
 # Examples
 This section provides some examples for the configuration of common scenarios.
 ## Enable Websockets SSL via [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/)
-The following settings describe a working scenario for acessing [EMQx](https://www.emqx.io/) Websockets with SSL termination at the [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/).
+The following settings describe a working scenario for acessing [EMQ X](https://www.emqx.io/) Websockets with SSL termination at the [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/).
 ```yaml
 ingress:
   wss:
diff --git a/deploy/charts/emqx/templates/configmap.acl.yaml b/deploy/charts/emqx/templates/configmap.acl.yaml
index 897ea0f48..25f67b42b 100644
--- a/deploy/charts/emqx/templates/configmap.acl.yaml
+++ b/deploy/charts/emqx/templates/configmap.acl.yaml
@@ -12,4 +12,4 @@ metadata:
 data:
   "acl.conf": |
     {{ .Values.emqxAclConfig }}
-{{ end }}
\ No newline at end of file
+{{ end }}
diff --git a/deploy/charts/emqx/templates/configmap.env.yaml b/deploy/charts/emqx/templates/configmap.env.yaml
index c27b65343..851a7496b 100644
--- a/deploy/charts/emqx/templates/configmap.env.yaml
+++ b/deploy/charts/emqx/templates/configmap.env.yaml
@@ -17,4 +17,4 @@ data:
   {{- end }}
   {{- end}}
 
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/deploy/charts/emqx/templates/configmap.loadedModules.yaml b/deploy/charts/emqx/templates/configmap.loadedModules.yaml
index 099db396a..45c97d4d0 100644
--- a/deploy/charts/emqx/templates/configmap.loadedModules.yaml
+++ b/deploy/charts/emqx/templates/configmap.loadedModules.yaml
@@ -12,4 +12,4 @@ metadata:
 data:
   "loaded_modules": |
     {{ .Values.emqxLoadedModules }}
-{{ end }}
\ No newline at end of file
+{{ end }}
diff --git a/deploy/charts/emqx/templates/configmap.loadedPlugins.yaml b/deploy/charts/emqx/templates/configmap.loadedPlugins.yaml
index df77368f9..242c1ab01 100644
--- a/deploy/charts/emqx/templates/configmap.loadedPlugins.yaml
+++ b/deploy/charts/emqx/templates/configmap.loadedPlugins.yaml
@@ -12,4 +12,4 @@ metadata:
 data:
   "loaded_plugins": |
     {{ .Values.emqxLoadedPlugins }}
-{{ end }}
\ No newline at end of file
+{{ end }}
diff --git a/deploy/charts/emqx/templates/rbac.yaml b/deploy/charts/emqx/templates/rbac.yaml
index 87cd18178..45806d698 100644
--- a/deploy/charts/emqx/templates/rbac.yaml
+++ b/deploy/charts/emqx/templates/rbac.yaml
@@ -39,4 +39,4 @@ subjects:
 roleRef:
   kind: Role
   name: {{ include "emqx.fullname" . }}
-  apiGroup: rbac.authorization.k8s.io
\ No newline at end of file
+  apiGroup: rbac.authorization.k8s.io
diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index d78f19a8e..f630f3d25 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -62,13 +62,13 @@ resources: {}
   #   cpu: 500m
   #   memory: 512Mi
 
-# Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts.
+# Containers that run before the creation of EMQ X containers. They can contain utilities or setup scripts.
 initContainers: {}
   # - name: mysql-probe
   #   image: alpine
   #   command: ["sh", "-c", "for i in $(seq 1 300); do nc -zvw1 mysql 3306 && exit 0 || sleep 3; done; exit 1"]
 
-## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
+## EMQ X configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
 emqxConfig:
   EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
   ## The address type is used to extract host from k8s service.
@@ -119,7 +119,7 @@ emqxLoadedModules: >
   {emqx_mod_subscription, false}.
   {emqx_mod_topic_metrics, false}.
 
-## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicenseSecretName"
+## EMQ X Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicenseSecretName"
 ## Example:
 ##   kubectl create secret generic emqx-license-secret-name --from-file=/path/to/emqx.lic
 emqxLicenseSecretName:
@@ -188,7 +188,7 @@ ingress:
   ## Ingress shared annotations
   annotations: {}
 
-  ## ingress for EMQX Dashboard
+  ## ingress for EMQ X Dashboard
   dashboard:
     enabled: false
     annotations: {}
@@ -199,7 +199,7 @@ ingress:
     - dashboard.emqx.local
     tls: []
 
-  ## ingress for EMQX Mgmt API
+  ## ingress for EMQ X Mgmt API
   mgmt:
     enabled: false
     annotations: {}
@@ -210,7 +210,7 @@ ingress:
     - api.emqx.local
     tls: []
 
-  ## ingress for EMQX Mgmt API
+  ## ingress for EMQ X Mgmt API
   wss:
     enabled: false
     # ingressClassName: nginx

From 579905b92a71cc974ad1bbb0883e6f4000877b23 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Fri, 28 Jan 2022 17:56:51 +0800
Subject: [PATCH 10/13] fix(watermark): hight watermark should greater than low
 watermark.

---
 priv/emqx.schema | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/priv/emqx.schema b/priv/emqx.schema
index 5f21c36d4..9623f9b86 100644
--- a/priv/emqx.schema
+++ b/priv/emqx.schema
@@ -2388,12 +2388,20 @@ end}.
 ]}.
 
 {translation, "emqx.os_mon", fun(Conf) ->
-    [{cpu_check_interval, cuttlefish:conf_get("os_mon.cpu_check_interval", Conf)},
-     {cpu_high_watermark, cuttlefish:conf_get("os_mon.cpu_high_watermark", Conf) * 100},
-     {cpu_low_watermark, cuttlefish:conf_get("os_mon.cpu_low_watermark", Conf) * 100},
-     {mem_check_interval, cuttlefish:conf_get("os_mon.mem_check_interval", Conf)},
-     {sysmem_high_watermark, cuttlefish:conf_get("os_mon.sysmem_high_watermark", Conf) * 100},
-     {procmem_high_watermark, cuttlefish:conf_get("os_mon.procmem_high_watermark", Conf) * 100}]
+   CpuHw = cuttlefish:conf_get("os_mon.cpu_high_watermark", Conf) * 100,
+   CpuLw = cuttlefish:conf_get("os_mon.cpu_low_watermark", Conf) * 100,
+   case CpuHw > CpuLw of
+     true ->
+      [{cpu_check_interval, cuttlefish:conf_get("os_mon.cpu_check_interval", Conf)},
+       {cpu_high_watermark, CpuHw},
+       {cpu_low_watermark, CpuLw},
+       {mem_check_interval, cuttlefish:conf_get("os_mon.mem_check_interval", Conf)},
+       {sysmem_high_watermark, cuttlefish:conf_get("os_mon.sysmem_high_watermark", Conf) * 100},
+       {procmem_high_watermark, cuttlefish:conf_get("os_mon.procmem_high_watermark", Conf) * 100}];
+      false ->
+         Msg = io_lib:format("high(~w)_must_greater_than_low(~w)", [CpuHw, CpuLw]),
+         error(lists:flatten(Msg))
+      end
 end}.
 
 %%--------------------------------------------------------------------
@@ -2415,9 +2423,17 @@ end}.
 ]}.
 
 {translation, "emqx.vm_mon", fun(Conf) ->
-    [{check_interval, cuttlefish:conf_get("vm_mon.check_interval", Conf)},
-     {process_high_watermark, cuttlefish:conf_get("vm_mon.process_high_watermark", Conf) * 100},
-     {process_low_watermark, cuttlefish:conf_get("vm_mon.process_low_watermark", Conf) * 100}]
+   Hw = cuttlefish:conf_get("vm_mon.process_high_watermark", Conf) * 100,
+   Lw = cuttlefish:conf_get("vm_mon.process_low_watermark", Conf) * 100,
+   case Hw > Lw of
+     true ->
+       [{check_interval, cuttlefish:conf_get("vm_mon.check_interval", Conf)},
+        {process_high_watermark, Hw},
+        {process_low_watermark, Lw}];
+     false ->
+        Msg = io_lib:format("high(~w)_must_greater_than_low(~w)", [Hw, Lw]),
+        error(lists:flatten(Msg))
+      end
 end}.
 
 %%--------------------------------------------------------------------

From 51ed6abcd60ee5874947794d4c985d4c3e840749 Mon Sep 17 00:00:00 2001
From: zhongwencool <zhongwencool@gmail.com>
Date: Mon, 7 Feb 2022 17:23:13 +0800
Subject: [PATCH 11/13] ci(pytest): pin pytest version at 6.2.5.

---
 .ci/docker-compose-file/python/pytest.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/docker-compose-file/python/pytest.sh b/.ci/docker-compose-file/python/pytest.sh
index eacbecc3b..0189c86ba 100755
--- a/.ci/docker-compose-file/python/pytest.sh
+++ b/.ci/docker-compose-file/python/pytest.sh
@@ -10,7 +10,7 @@ LB="haproxy"
 
 apk update && apk add git curl
 git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing
-pip install pytest
+pip install pytest==6.2.5
 
 pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB"
 RESULT=$?

From 352635f22741fc588f3a985a412ac85ca499ac32 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 8 Feb 2022 10:40:16 +0800
Subject: [PATCH 12/13] fix(helm): fix deploy error

---
 .../charts/emqx/templates/configmap.env.yaml  |  2 +-
 deploy/charts/emqx/templates/configmap.yaml   | 59 -------------------
 2 files changed, 1 insertion(+), 60 deletions(-)
 delete mode 100644 deploy/charts/emqx/templates/configmap.yaml

diff --git a/deploy/charts/emqx/templates/configmap.env.yaml b/deploy/charts/emqx/templates/configmap.env.yaml
index 851a7496b..04a72033c 100644
--- a/deploy/charts/emqx/templates/configmap.env.yaml
+++ b/deploy/charts/emqx/templates/configmap.env.yaml
@@ -13,7 +13,7 @@ data:
   {{- range $index, $value := .Values.emqxConfig }}
   {{- if ne $value nil }}
   {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}
-  {{ print "EMQX_" $key }}: {{ $value | quote }}
+  {{ print "EMQX_" $key }}: "{{ tpl (printf "%v" $value) $ }}"
   {{- end }}
   {{- end}}
 
diff --git a/deploy/charts/emqx/templates/configmap.yaml b/deploy/charts/emqx/templates/configmap.yaml
deleted file mode 100644
index 328df2000..000000000
--- a/deploy/charts/emqx/templates/configmap.yaml
+++ /dev/null
@@ -1,59 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "emqx.fullname" . }}-env
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ include "emqx.name" . }}
-    helm.sh/chart: {{ include "emqx.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-data:
-  {{- range $index, $value := .Values.emqxConfig}}
-    {{$index}}: "{{ tpl (printf "%v" $value) $ }}"
-  {{- end}}
-
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "emqx.fullname" . }}-acl
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ include "emqx.name" . }}
-    helm.sh/chart: {{ include "emqx.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-data:
-  "acl.conf": |
-    {{ .Values.emqxAclConfig }}
-
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "emqx.fullname" . }}-loaded-plugins
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ include "emqx.name" . }}
-    helm.sh/chart: {{ include "emqx.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-data:
-  "loaded_plugins": |
-    {{ .Values.emqxLoadedPlugins }}
-
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "emqx.fullname" . }}-loaded-modules
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app.kubernetes.io/name: {{ include "emqx.name" . }}
-    helm.sh/chart: {{ include "emqx.chart" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-data:
-  "loaded_modules": |
-    {{ .Values.emqxLoadedModules }}

From cc56ad272fd0842fb3c4782abb09c8f65ab3b957 Mon Sep 17 00:00:00 2001
From: zhanghongtong <rory-z@outlook.com>
Date: Tue, 8 Feb 2022 14:06:39 +0800
Subject: [PATCH 13/13] fix(helm): remove the default environment variables
 from the template

---
 deploy/charts/emqx/templates/StatefulSet.yaml | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml
index 6a333bbe7..e155d71e2 100644
--- a/deploy/charts/emqx/templates/StatefulSet.yaml
+++ b/deploy/charts/emqx/templates/StatefulSet.yaml
@@ -157,18 +157,8 @@ spec:
           {{- if .Values.extraEnvFrom }}
 {{ toYaml .Values.extraEnvFrom | indent 10 }}
           {{- end }}
-          env:
-          - name: EMQX_NAME
-            value: {{ .Release.Name }}
-          - name: EMQX_CLUSTER__K8S__APP_NAME
-            value: {{ .Release.Name }}
-          - name: EMQX_CLUSTER__DISCOVERY
-            value: k8s
-          - name: EMQX_CLUSTER__K8S__SERVICE_NAME
-            value: {{ include "emqx.fullname" . }}-headless
-          - name: EMQX_CLUSTER__K8S__NAMESPACE
-            value: {{ .Release.Namespace }}
           {{- if .Values.extraEnv }}
+          env:
 {{ toYaml .Values.extraEnv | indent 10 }}
           {{- end }}
           resources: