From 4cdfa088ae9eb74e2d278bfef97c3eb6a960417f Mon Sep 17 00:00:00 2001 From: Rory Z Date: Thu, 19 Jan 2023 11:22:09 +0800 Subject: [PATCH 01/11] feat(helm): user can define service account by userself --- deploy/charts/emqx-enterprise/README.md | 14 ++++++----- .../templates/StatefulSet.yaml | 4 +-- .../emqx-enterprise/templates/_helpers.tpl | 11 ++++++++ .../emqx-enterprise/templates/rbac.yaml | 25 ++++++++++++++++--- deploy/charts/emqx-enterprise/values.yaml | 9 +++++++ deploy/charts/emqx/README.md | 3 +++ deploy/charts/emqx/templates/StatefulSet.yaml | 4 +-- deploy/charts/emqx/templates/_helpers.tpl | 11 ++++++++ deploy/charts/emqx/templates/rbac.yaml | 25 ++++++++++++++++--- deploy/charts/emqx/values.yaml | 9 +++++++ 10 files changed, 97 insertions(+), 18 deletions(-) diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md index 33a3fa22f..2899dc7e0 100644 --- a/deploy/charts/emqx-enterprise/README.md +++ b/deploy/charts/emqx-enterprise/README.md @@ -14,14 +14,14 @@ To install the chart with the release name `my-emqx`: + From github ``` $ git clone https://github.com/emqx/emqx.git - $ cd emqx/deploy/charts/emqx + $ cd emqx/deploy/charts/emqx-enterprise $ helm install my-emqx . ``` + From chart repos ``` helm repo add emqx https://repos.emqx.io/charts - helm install my-emqx emqx/emqx + helm install my-emqx emqx/emqx-enterprise ``` > If you want to install an unstable version, you need to add `--devel` when you execute the `helm install` command. @@ -43,6 +43,9 @@ The following table lists the configurable parameters of the emqx chart and thei | `image.repository` | EMQX Image name | `emqx/emqx-enterprise` | | `image.pullPolicy` | The image pull policy | IfNotPresent | | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | +| `serviceAccount.create` | If `true`, create a new service account | `true` | +| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the fullname template | | +| `serviceAccount.annotations` | Annotations to add to the service account | | | `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | | `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | | `podAnnotations ` | Annotations for pod | `{}` | @@ -102,10 +105,9 @@ The following table lists the configurable [EMQX](https://www.emqx.io/)-specific default values. Parameter | Description | Default Value --- | --- | --- -`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items -expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix -can be omitted) or using the configuration -files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil` +`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/v5.0/admin/cfg.html) items +expressed as [environment variables](https://www.emqx.io/docs/en/v5.0/admin/cfg.html#environment-variables) (prefix `EMQX_` can be omitted) or using the configuration +files [namespaced dotted notation](https://www.emqx.io/docs/en/v5.0/admin/cfg.html#syntax) | `nil` `emqxLicenseSecretName` | Name of the secret that holds the license information | `nil` ## SSL settings diff --git a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml index 0917cadac..3932dc727 100644 --- a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml +++ b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml @@ -52,6 +52,7 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }} {{- end }} spec: + serviceAccountName: {{ include "emqx.serviceAccountName" . }} volumes: {{- if .Values.ssl.enabled }} - name: ssl-cert @@ -73,9 +74,6 @@ spec: secret: secretName: {{ .Values.emqxLicenseSecretName }} {{- end }} - {{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} - serviceAccountName: {{ include "emqx.fullname" . }} - {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} diff --git a/deploy/charts/emqx-enterprise/templates/_helpers.tpl b/deploy/charts/emqx-enterprise/templates/_helpers.tpl index e55740b01..64acf7961 100644 --- a/deploy/charts/emqx-enterprise/templates/_helpers.tpl +++ b/deploy/charts/emqx-enterprise/templates/_helpers.tpl @@ -42,3 +42,14 @@ Get ssl secret name . {{ include "emqx.fullname" . }}-tls {{- end -}} {{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "emqx.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "emqx.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/charts/emqx-enterprise/templates/rbac.yaml b/deploy/charts/emqx-enterprise/templates/rbac.yaml index f2cdd3601..9faf4e147 100644 --- a/deploy/charts/emqx-enterprise/templates/rbac.yaml +++ b/deploy/charts/emqx-enterprise/templates/rbac.yaml @@ -1,10 +1,23 @@ -{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} +{{- if .Values.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: + name: {{ include "emqx.serviceAccountName" . }} namespace: {{ .Release.Namespace }} - name: {{ include "emqx.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + --- +{{- if .Values.serviceAccount.create }} +{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }} kind: Role {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} apiVersion: rbac.authorization.k8s.io/v1 @@ -23,7 +36,12 @@ rules: - get - watch - list +{{- end }} +{{- end }} + --- +{{- if .Values.serviceAccount.create }} +{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }} kind: RoleBinding {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} apiVersion: rbac.authorization.k8s.io/v1 @@ -35,10 +53,11 @@ metadata: name: {{ include "emqx.fullname" . }} subjects: - kind: ServiceAccount - name: {{ include "emqx.fullname" . }} + name: {{ include "emqx.serviceAccountName" . }} namespace: {{ .Release.Namespace }} roleRef: kind: Role name: {{ include "emqx.fullname" . }} apiGroup: rbac.authorization.k8s.io {{- end }} +{{- end }} diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml index 0396b2b20..b9507c5a0 100644 --- a/deploy/charts/emqx-enterprise/values.yaml +++ b/deploy/charts/emqx-enterprise/values.yaml @@ -16,6 +16,15 @@ image: # pullSecrets: # - myRegistryKeySecretName +serviceAccount: + # Specifies whether a service account should be created + # If set false, means you need create service account by yourself + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + # Annotations to add to the service account + annotations: {} # The name of a secret in the same kubernetes namespace which contains values to # be added to the environment (must be manually created) diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 1a2090320..6ee3617ce 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -43,6 +43,9 @@ The following table lists the configurable parameters of the emqx chart and thei | `image.repository` | EMQX Image name | emqx/emqx | | `image.pullPolicy` | The image pull policy | IfNotPresent | | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | +| `serviceAccount.create` | If `true`, create a new service account | `true` | +| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the fullname template | | +| `serviceAccount.annotations` | Annotations to add to the service account | | | `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | | `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | | `podAnnotations ` | Annotations for pod | `{}` | diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 0917cadac..3932dc727 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -52,6 +52,7 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }} {{- end }} spec: + serviceAccountName: {{ include "emqx.serviceAccountName" . }} volumes: {{- if .Values.ssl.enabled }} - name: ssl-cert @@ -73,9 +74,6 @@ spec: secret: secretName: {{ .Values.emqxLicenseSecretName }} {{- end }} - {{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} - serviceAccountName: {{ include "emqx.fullname" . }} - {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} diff --git a/deploy/charts/emqx/templates/_helpers.tpl b/deploy/charts/emqx/templates/_helpers.tpl index e55740b01..64acf7961 100644 --- a/deploy/charts/emqx/templates/_helpers.tpl +++ b/deploy/charts/emqx/templates/_helpers.tpl @@ -42,3 +42,14 @@ Get ssl secret name . {{ include "emqx.fullname" . }}-tls {{- end -}} {{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "emqx.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "emqx.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/charts/emqx/templates/rbac.yaml b/deploy/charts/emqx/templates/rbac.yaml index f2cdd3601..9faf4e147 100644 --- a/deploy/charts/emqx/templates/rbac.yaml +++ b/deploy/charts/emqx/templates/rbac.yaml @@ -1,10 +1,23 @@ -{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} +{{- if .Values.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: + name: {{ include "emqx.serviceAccountName" . }} namespace: {{ .Release.Namespace }} - name: {{ include "emqx.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + --- +{{- if .Values.serviceAccount.create }} +{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }} kind: Role {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} apiVersion: rbac.authorization.k8s.io/v1 @@ -23,7 +36,12 @@ rules: - get - watch - list +{{- end }} +{{- end }} + --- +{{- if .Values.serviceAccount.create }} +{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }} kind: RoleBinding {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} apiVersion: rbac.authorization.k8s.io/v1 @@ -35,10 +53,11 @@ metadata: name: {{ include "emqx.fullname" . }} subjects: - kind: ServiceAccount - name: {{ include "emqx.fullname" . }} + name: {{ include "emqx.serviceAccountName" . }} namespace: {{ .Release.Namespace }} roleRef: kind: Role name: {{ include "emqx.fullname" . }} apiGroup: rbac.authorization.k8s.io {{- end }} +{{- end }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 4fb263c7a..0423c8cdf 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -16,6 +16,15 @@ image: # pullSecrets: # - myRegistryKeySecretName +serviceAccount: + # Specifies whether a service account should be created + # If set false, means you need create service account by yourself + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + # Annotations to add to the service account + annotations: {} # The name of a secret in the same kubernetes namespace which contains values to # be added to the environment (must be manually created) From a4583d2afa1fa9d404b0a8fea41653c76ebb914e Mon Sep 17 00:00:00 2001 From: Parham Alvani Date: Thu, 19 Jan 2023 13:46:44 +0330 Subject: [PATCH 02/11] fix: Correct chart variable for internal MQTT port and HTTPS dashboard. These variables are coming from v4, and they were incorrect. This commit will fix them. --- deploy/charts/emqx-enterprise/templates/StatefulSet.yaml | 8 ++++---- deploy/charts/emqx-enterprise/templates/service.yaml | 2 +- deploy/charts/emqx/templates/StatefulSet.yaml | 8 ++++---- deploy/charts/emqx/templates/service.yaml | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml index 0917cadac..022c4ee62 100644 --- a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml +++ b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml @@ -107,13 +107,13 @@ spec: containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT__BIND | default 8084 }} - name: dashboard containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt - containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT }} + containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND }} {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS) }} + {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND) }} - name: dashboardtls - containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS }} + containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND }} {{- end }} - name: ekka containerPort: 4370 diff --git a/deploy/charts/emqx-enterprise/templates/service.yaml b/deploy/charts/emqx-enterprise/templates/service.yaml index 301213150..0fe3dc411 100644 --- a/deploy/charts/emqx-enterprise/templates/service.yaml +++ b/deploy/charts/emqx-enterprise/templates/service.yaml @@ -38,7 +38,7 @@ spec: {{- else if eq .Values.service.type "ClusterIP" }} nodePort: null {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt port: {{ .Values.service.internalmqtt | default 11883 }} protocol: TCP diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 0917cadac..022c4ee62 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -107,13 +107,13 @@ spec: containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT__BIND | default 8084 }} - name: dashboard containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt - containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT }} + containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND }} {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS) }} + {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND) }} - name: dashboardtls - containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS }} + containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND }} {{- end }} - name: ekka containerPort: 4370 diff --git a/deploy/charts/emqx/templates/service.yaml b/deploy/charts/emqx/templates/service.yaml index 301213150..5b6376a85 100644 --- a/deploy/charts/emqx/templates/service.yaml +++ b/deploy/charts/emqx/templates/service.yaml @@ -121,7 +121,7 @@ spec: port: {{ .Values.service.mqtt | default 1883 }} protocol: TCP targetPort: mqtt - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt port: {{ .Values.service.internalmqtt | default 11883 }} protocol: TCP From 622e42a8caf7066ec8365dd200655a40ec13e305 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 19 Jan 2023 11:45:09 +0100 Subject: [PATCH 03/11] chore: pin enterprise dashboard at version e1.0.1 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fc87f1d95..0e2aacf31 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v1.1.5 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.13 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) From d755b43c774bf623f58f6db82059cbe69dcbbbd9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 19 Jan 2023 09:24:45 -0300 Subject: [PATCH 04/11] fix(jwt_worker): handle exceptions when decoding jwk from pem Returns a more controlled error if users attempt to use the Service Account JSON from the GCP PubSub example from swagger, which is redacted. --- .../src/emqx_connector_jwt_worker.erl | 13 +++++++++++- .../test/emqx_connector_jwt_worker_SUITE.erl | 20 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl index e51b9bbee..b13e74a4d 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl @@ -120,7 +120,7 @@ init(#{private_key := PrivateKeyPEM} = Config) -> handle_continue({make_key, PrivateKeyPEM}, State0) -> ?tp(connector_jwt_worker_make_key, #{state => State0}), - case jose_jwk:from_pem(PrivateKeyPEM) of + try jose_jwk:from_pem(PrivateKeyPEM) of JWK = #jose_jwk{} -> State = State0#{jwk := JWK}, {noreply, State, {continue, create_token}}; @@ -135,6 +135,17 @@ handle_continue({make_key, PrivateKeyPEM}, State0) -> Error = {invalid_private_key, Error0}, ?tp(connector_jwt_worker_startup_error, #{error => Error}), {stop, {shutdown, {error, Error}}, State0} + catch + Kind:Error -> + ?tp( + error, + connector_jwt_worker_startup_error, + #{ + kind => Kind, + error => Error + } + ), + {stop, {shutdown, {error, Error}}, State0} end; handle_continue(create_token, State0) -> State = generate_and_store_jwt(State0), diff --git a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl b/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl index eb104801c..a079d632f 100644 --- a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl @@ -364,3 +364,23 @@ t_unknown_requests(_Config) -> gen_server:cast(Worker, unknown_cast), ?assertEqual({error, bad_call}, gen_server:call(Worker, unknown_call)), ok. + +t_truncated_private_key(_Config) -> + Config0 = generate_config(), + Config = Config0#{private_key := <<"-----BEGIN PRIVATE KEY-----\nMIIEvQI...">>}, + process_flag(trap_exit, true), + ?check_trace( + ?wait_async_action( + ?assertMatch({ok, _}, emqx_connector_jwt_worker:start_link(Config)), + #{?snk_kind := connector_jwt_worker_startup_error}, + 1_000 + ), + fun(Trace) -> + ?assertMatch( + [#{error := function_clause}], + ?of_kind(connector_jwt_worker_startup_error, Trace) + ), + ok + end + ), + ok. From 16f45a60fdc52fa275eeaa53ef5938db0c6b729a Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 20 Jan 2023 11:46:11 +0800 Subject: [PATCH 05/11] chore: i18n typo fix --- apps/emqx_connector/i18n/emqx_connector_redis.conf | 2 +- apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/i18n/emqx_connector_redis.conf b/apps/emqx_connector/i18n/emqx_connector_redis.conf index f42f38f30..e8e05d08f 100644 --- a/apps/emqx_connector/i18n/emqx_connector_redis.conf +++ b/apps/emqx_connector/i18n/emqx_connector_redis.conf @@ -54,7 +54,7 @@ The Redis default port 6379 is used if `[:Port]` is not specified. zh: """ 将要连接的 IPv4 或 IPv6 地址,或者主机名。
主机名具有以下形式:`Host[:Port]`。
-如果未指定 `[:Port]`,则使用 MongoDB 默认端口 27017。 +如果未指定 `[:Port]`,则使用 Redis 默认端口 6379。 """ } label: { diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index de76967ab..7e9fcd9e7 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -26,7 +26,7 @@ emqx_resource_schema { desc { en: """The number of buffer workers. Only applicable for egress type bridges. For bridges only have ingress direction data flow, it can be set to 0 otherwise must be greater than 0.""" - zh: """缓存队列 worker 数量。仅对 egress 类型的桥接有意义。当桥接仅有 ingress 方向时,可设置为 0,否则必须大于 0)。""" + zh: """缓存队列 worker 数量。仅对 egress 类型的桥接有意义。当桥接仅有 ingress 方向时,可设置为 0,否则必须大于 0。""" } label { en: """Buffer Pool Size""" From 34c1ebbe483daf19e7aba2c53116f28698075322 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 20 Jan 2023 11:09:57 +0100 Subject: [PATCH 06/11] chore: upgrade to dashboard v1.1.6 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0e2aacf31..4be33b567 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.1.5 +export EMQX_DASHBOARD_VERSION ?= v1.1.6 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 From 57607ca0ce6dd7729bd1d34beced74de803ea5f7 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 20 Jan 2023 11:10:46 +0100 Subject: [PATCH 07/11] chore: prepare for v5.0.15 release --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index a4963f31a..e9eab302c 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.14"). +-define(EMQX_RELEASE_CE, "5.0.15-rc.1"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.0-rc.1"). diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index ae48f9de2..fb689839c 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.14 +version: 5.0.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.14 +appVersion: 5.0.15 From 8c52264c41064c2fa8c96016afa70e6ba82b4be5 Mon Sep 17 00:00:00 2001 From: Kjell Winblad Date: Mon, 16 Jan 2023 13:01:51 +0100 Subject: [PATCH 08/11] fix: atom leak when doing Kafka bridge dry-run A new atom was created every time one did a dry run of a Kafka bridge (that is, clicking the Test button in the settings dialog for the bridge). After this fix, we will only create a new atom when a bridge with a new name is created. This should be acceptable as bridges with new names are created relatively rarely and it seems to be useful to have an unique atom for every Wolff producer. Fixes: https://emqx.atlassian.net/browse/EMQX-8739 --- .../kafka/emqx_bridge_impl_kafka_producer.erl | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl index bec3c49fa..25741b6cd 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl @@ -75,7 +75,16 @@ on_start(InstId, Config) -> }), throw(failed_to_start_kafka_client) end, - WolffProducerConfig = producers_config(BridgeName, ClientId, ProducerConfig), + %% Check if this is a dry run + TestIdStart = string:find(InstId, ?TEST_ID_PREFIX), + IsDryRun = + case TestIdStart of + nomatch -> + false; + _ -> + string:equal(TestIdStart, InstId) + end, + WolffProducerConfig = producers_config(BridgeName, ClientId, ProducerConfig, IsDryRun), case wolff:ensure_supervised_producers(ClientId, KafkaTopic, WolffProducerConfig) of {ok, Producers} -> {ok, #{ @@ -241,7 +250,7 @@ ssl(#{enable := true} = SSL) -> ssl(_) -> []. -producers_config(BridgeName, ClientId, Input) -> +producers_config(BridgeName, ClientId, Input, IsDryRun) -> #{ max_batch_bytes := MaxBatchBytes, compression := Compression, @@ -271,7 +280,7 @@ producers_config(BridgeName, ClientId, Input) -> BridgeType = kafka, ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName), #{ - name => make_producer_name(BridgeName), + name => make_producer_name(BridgeName, IsDryRun), partitioner => partitioner(PartitionStrategy), partition_count_refresh_interval_seconds => PCntRefreshInterval, replayq_dir => ReplayqDir, @@ -302,12 +311,20 @@ make_client_id(BridgeName) -> %% Producer name must be an atom which will be used as a ETS table name for %% partition worker lookup. -make_producer_name(BridgeName) when is_atom(BridgeName) -> - make_producer_name(atom_to_list(BridgeName)); -make_producer_name(BridgeName) -> - %% Woff needs atom for ets table name registration - %% The assumption here is bridge is not often re-created - binary_to_atom(iolist_to_binary(["kafka_producer_", BridgeName])). +make_producer_name(BridgeName, IsDryRun) when is_atom(BridgeName) -> + make_producer_name(atom_to_list(BridgeName), IsDryRun); +make_producer_name(BridgeName, IsDryRun) -> + %% Woff needs an atom for ets table name registration. The assumption here is + %% that bridges with new names are not often created. + case IsDryRun of + true -> + %% It is a dry run and we don't want to leak too many atoms + %% so we use the default producer name instead of creating + %% an unique name. + probing_wolff_producers; + false -> + binary_to_atom(iolist_to_binary(["kafka_producer_", BridgeName])) + end. with_log_at_error(Fun, Log) -> try From 727100e094e162d7d65371dda430d96adcd5f5e2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 20 Jan 2023 16:42:01 +0100 Subject: [PATCH 09/11] chore: prepare for v5.0.15 release --- apps/emqx/include/emqx_release.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index e9eab302c..7437bc299 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.15-rc.1"). +-define(EMQX_RELEASE_CE, "5.0.15"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.0-rc.1"). From 92797d7260c0688af7baa137e9e1e1c1593227ab Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 20 Jan 2023 17:00:55 +0100 Subject: [PATCH 10/11] chore: Generate changelog for v5.0.15 --- changes/v5.0.15-en.md | 79 +++++++++++++++++++++++++++++++++++++++++++ changes/v5.0.15-zh.md | 76 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 changes/v5.0.15-en.md create mode 100644 changes/v5.0.15-zh.md diff --git a/changes/v5.0.15-en.md b/changes/v5.0.15-en.md new file mode 100644 index 000000000..f77753c25 --- /dev/null +++ b/changes/v5.0.15-en.md @@ -0,0 +1,79 @@ +# v5.0.15 + +## Enhancements + +- [#9569](https://github.com/emqx/emqx/pull/9569) Refactor `/authorization/sources/built_in_database/` by adding `rules/` to the path. + +- [#9585](https://github.com/emqx/emqx/pull/9585) `/bridges_probe` API endpoint to test params for creating a new data bridge. + +- [#9586](https://github.com/emqx/emqx/pull/9586) Basic auth is no longer allowed for API calls, must use API key instead. + +- [#9628](https://github.com/emqx/emqx/pull/9628) Expose additional resource configuration parameters: `start_after_created` and `start_timeout`. + +- [#9722](https://github.com/emqx/emqx/pull/9722) Add the following configuration options for Pushing metrics to Prometheus Push Gateway: + - `headers`: Allows custom HTTP request headers. + - `job_name`: allows to customize the name of the Job pushed to Push Gateway. + +- [#9725](https://github.com/emqx/emqx/pull/9725) Remove the config `auto_reconnect` from the emqx_authz, emqx_authn and data-bridge componets. + This is because we have another config with similar functions: `resource_opts.auto_restart_interval`。 + + The functions of these two config are difficult to distinguish, which will lead to confusion. + After this change, `auto_reconnect` will not be configurable (always be true), and the underlying + drivers that support this config will automatically reconnect the abnormally disconnected + connection every `2s`. + + And the config `resource_opts.auto_restart_interval` is still available for user. + It is the time interval that emqx restarts the resource when the connection cannot be + established for some reason. + +- [#9736](https://github.com/emqx/emqx/pull/9736) Refactor of /bridges API to make it more consistent with other APIs: + - bridge enable/disable is now done via the endpoint `/bridges/{id}/enable/[true,false]` + - `/bridges/{id}/operation/{operation}` endpoints are now `/bridges/{id}/{operation}` + - metrics are moved out from the GET `/bridges/{id}` response and can now be fetched via `/bridges/{id}/metrics` + - the `bridges/{id}/reset_metrics` endpoint is now `/bridges/{id}/metrics/reset` + +- [#9774](https://github.com/emqx/emqx/pull/9774) Add a password complexity requirement when adding or modifying Dashboard users via the API. + Now password must contain at least 2 of alphabetic, numeric and special characters, + and must be 8 to 64 characters long. + +## Bug fixes + +- [#9626](https://github.com/emqx/emqx/pull/9626) Return authorization settings with default values. + The authorization cache is enabled by default, but due to the missing default value in `GET` response of `/authorization/settings`, it seemed to be disabled from the dashboard. + +- [#9680](https://github.com/emqx/emqx/pull/9680) Fix the problem that username and password authentication is mandatory in Influxdb v1 write API. + +- [#9726](https://github.com/emqx/emqx/pull/9726) Client fuzzy search API results were missing information which could tell if more results are available in the next pages, this is now fixed by providing `hasnext` flag in the response. + +- [#9735](https://github.com/emqx/emqx/pull/9735) Password information has been removed from information log messages for http, ldap, mongo, mqtt, mysql, pgsql and redis. + +- [#9748](https://github.com/emqx/emqx/pull/9748) Listeners not configured with `max_connections` will cause the cluster `/listeners` API to return 500 error. + +- [#9749](https://github.com/emqx/emqx/pull/9749) In some cases search APIs could respond with an incorrect `count` value in the metadata, that is usually much bigger than expected, this is now fixed. + +- [#9750](https://github.com/emqx/emqx/pull/9750) Reload overriding configs after boot. + Prior to this change, two configs were allow to change from dashboard, but will not take effect after reboot: + * Logging (such as level) + * Prometheus configs + + +- [#9751](https://github.com/emqx/emqx/pull/9751) Fix that obsoleted cert file will not be deleted after the listener is updated/deleted + +- [#9763](https://github.com/emqx/emqx/pull/9763) Fix an authentication exception when password is not provided + +- [#9765](https://github.com/emqx/emqx/pull/9765) Parse decimals as password from environment variable overrides correctly. + Prior to this change, config values for passwords are not allowed to be decimals. + e.g. `EMQX_FOOBAR__PASSWORD=12344` or `emqx.foobar.password=1234` + would result in a type check error, unless quoted as: + `EMQX_FOOBAR__PASSWORD='"12344"'` or `emqx.foobar.password="1234"`. + After this fix, the value does not have to be quoted. + +- [#9769](https://github.com/emqx/emqx/pull/9769) Fix Erlang shell prompt version prefix. e5.0.15 -> v5.0.15 + +- [#9780](https://github.com/emqx/emqx/pull/9780) When creating disk queue directory for resource worker, substitute ':' with '-' in worker id. + +- [#9781](https://github.com/emqx/emqx/pull/9781) Trace files were left on a node when creating a zip file for download. They are now removed when the file is sent. Also, concurrent downloads will no longer interfere with each other. + +- [#9785](https://github.com/emqx/emqx/pull/9785) Stop authentication hook chain if `emqx_authentication` provides a definitive result. + +- [#9787](https://github.com/emqx/emqx/pull/9787) Fix a compatible problem for the `webhook` bridge configuration which was created before the v5.0.12. diff --git a/changes/v5.0.15-zh.md b/changes/v5.0.15-zh.md new file mode 100644 index 000000000..679298613 --- /dev/null +++ b/changes/v5.0.15-zh.md @@ -0,0 +1,76 @@ +# v5.0.15 + +## 增强 + +- [#9569](https://github.com/emqx/emqx/pull/9569) 重构 `/authorization/sources/built_in_database/` 接口,将 `rules/` 添加到了其路径中。 + +- [#9585](https://github.com/emqx/emqx/pull/9585) 添加新 API 接口 `/bridges_probe` 用于测试创建桥接的参数是否可用。 + +- [#9586](https://github.com/emqx/emqx/pull/9586) API 调用不再支持基于 `username:password` 的 `baisc` 认证, 现在 API 必须通过 API Key 才能进行调用。 + +- [#9628](https://github.com/emqx/emqx/pull/9628) 为桥接资源增加了配置参数:`start_after_created` 和 `start_timeout`。 + +- [#9722](https://github.com/emqx/emqx/pull/9722) 为 Prometheus 推送到 Push Gateway 新增以下配置项: + - `headers`:允许自定义 HTTP 请求头。 + - `job_name`:允许自定义推送到 Push Gateway 的 Job 名称。 + +- [#9725](https://github.com/emqx/emqx/pull/9725) 从认证、鉴权和数据桥接功能中,删除 `auto_reconnect` 配置项,因为我们还有另一个功能类似的配置项: + `resource_opts.auto_restart_interval`。 + + 这两个配置项的功能难以区分,会导致困惑。此修改之后,`auto_reconnect` 将不可配置(永远为 true), + 支持此配置的底层驱动将以 `2s` 为周期自动重连异常断开的连接。 + + 而 `resource_opts.auto_restart_interval` 配置项仍然开放给用户配置,它是资源因为某些原因 + 无法建立连接的时候,emqx 重新启动该资源的时间间隔。 + +- [#9736](https://github.com/emqx/emqx/pull/9736) 重构部分 /bridges 的API 使得其和其他 API 能够更加一致: + - 桥接的启用和禁用现在是通过 `/bridges/{id}/enable/[true,false]` API 来实现的 + - 使用 `/bridges/{id}/{operation}` 替换了旧的 `/bridges/{id}/operation/{operation}` API + - 指标数据从 `/bridges/{id}` 的响应消息中移除,现在可以使用新的 API `/bridges/{id}/metrics` 进行访问 + - 使用 `/bridges/{id}/metrics/reset` 替换了旧的 `bridges/{id}/reset_metrics` API + +- [#9774](https://github.com/emqx/emqx/pull/9774) 通过 API 添加、修改 Dashboard 用户时,增加对密码复杂度的要求。 + 现在密码必须包含字母、数字以及特殊字符中的至少 2 种,并且长度范围必须是 8~64 个字符。 + +## 修复 + +- [#9626](https://github.com/emqx/emqx/pull/9626) 为授权设置 API 返回默认值。 + 授权缓存默认为开启,但是在此修复前,因为默认值在 `/authorization/settings` 这个 API 的返回值中缺失, + 使得在仪表盘配置页面中看起来是关闭了。 + +- [#9680](https://github.com/emqx/emqx/pull/9680) 修复 InfluxDB v1 桥接写入 API 配置中强制需要用户名密码认证的问题。 + +- [#9726](https://github.com/emqx/emqx/pull/9726) 在此修复前,客户端模糊搜索 API 缺少一些可以用于判断是否可以继续翻页的信息,现在通过在响应中提供 `hasnext` 标志来解决这个问题。 + +- [#9735](https://github.com/emqx/emqx/pull/9735) 密码信息已从http、ldap、mongo、mqtt、mysql、pgsql和redis的信息日志消息中删除。 + +- [#9748](https://github.com/emqx/emqx/pull/9748) 监听器不配置 `max_connections` 时会导致集群 `/listeners` 接口返回 500 错误。 + +- [#9749](https://github.com/emqx/emqx/pull/9749) 在某些情况下,搜索 API 可能会在元数据中响应不正确的 `count` 值,这通常比预期的要大得多,现在已经修复了。 + +- [#9750](https://github.com/emqx/emqx/pull/9750) 启动后重新加载一些重载配置项。 + 在此修复前,下面两个配置项允许从 Dashboard 控制台修改,但是在重启后无法生效: + * 日志 (例如日志级别) + * Prometheus 配置 + +- [#9751](https://github.com/emqx/emqx/pull/9751) 修复在更新或者删除监听器后,过时的证书文件没有被删除的问题。 + +- [#9763](https://github.com/emqx/emqx/pull/9763) 修复客户端没有提供密码时的一个异常 + +- [#9765](https://github.com/emqx/emqx/pull/9765) 允许使用纯数字作为密码配置。 + 在此修复前,密码的配置必须是字符串,使用纯数字时,会报类型检查错误。 + 例如,`EMQX_FOOBAR__PASSWORD=12344` 或 `emqx.foobar.password=1234` 会出错, + 必须用引把值括起来才行: + `EMQX_FOOBAR__PASSWORD='"12344"'` 或 `emqx.foobar.password="1234"`。 + 修复后可以不使用引号。在环境变量重载中使用更加方便。 + + +- [#9769](https://github.com/emqx/emqx/pull/9769) 修复 Eralng 控制台版本号前缀的打印错误 e5.0.15 -> v5.0.15 + +- [#9780](https://github.com/emqx/emqx/pull/9780) 在为资源缓存进程创建磁盘队列目录时,在ID中用 '-' 代替 ':'。 + +- [#9781](https://github.com/emqx/emqx/pull/9781) 当下载 日志追踪 的日志时,一些中间文件将存留在处理节点上,现在这个问题得到了修复。同时,并发下载日志将不再相互干扰。 + +- [#9785](https://github.com/emqx/emqx/pull/9785) 如果 `emqx_authentication` 提供了确定的结果,则停止认证钩子链。 + +- [#9787](https://github.com/emqx/emqx/pull/9787) 修复对在 v5.0.12 之前创建的 `webhook` 桥接配置的兼容问题。 From a0100c0a4e6dd849b5f8ec32dddc2304dfd65d80 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 23 Jan 2023 16:18:07 +0100 Subject: [PATCH 11/11] chore: bump app versions --- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_resource/src/emqx_resource.app.src | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 39cb1b18b..ca0001319 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.9"}, + {vsn, "0.1.10"}, {registered, []}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index e3a37fd10..6bd9fd213 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 11831ab06..b2a3c80c6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel,