diff --git a/Makefile b/Makefile index fc87f1d95..4be33b567 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,8 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.1.5 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.13 +export EMQX_DASHBOARD_VERSION ?= v1.1.6 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index a4963f31a..7437bc299 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Community edition --define(EMQX_RELEASE_CE, "5.0.14"). +-define(EMQX_RELEASE_CE, "5.0.15"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.0.0-rc.1"). diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 39cb1b18b..ca0001319 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.9"}, + {vsn, "0.1.10"}, {registered, []}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_connector/i18n/emqx_connector_redis.conf b/apps/emqx_connector/i18n/emqx_connector_redis.conf index f42f38f30..e8e05d08f 100644 --- a/apps/emqx_connector/i18n/emqx_connector_redis.conf +++ b/apps/emqx_connector/i18n/emqx_connector_redis.conf @@ -54,7 +54,7 @@ The Redis default port 6379 is used if `[:Port]` is not specified. zh: """ 将要连接的 IPv4 或 IPv6 地址，或者主机名。
主机名具有以下形式：`Host[:Port]`。
-如果未指定 `[:Port]`，则使用 MongoDB 默认端口 27017。 +如果未指定 `[:Port]`，则使用 Redis 默认端口 6379。 """ } label: { diff --git a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl index e51b9bbee..b13e74a4d 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl @@ -120,7 +120,7 @@ init(#{private_key := PrivateKeyPEM} = Config) -> handle_continue({make_key, PrivateKeyPEM}, State0) -> ?tp(connector_jwt_worker_make_key, #{state => State0}), - case jose_jwk:from_pem(PrivateKeyPEM) of + try jose_jwk:from_pem(PrivateKeyPEM) of JWK = #jose_jwk{} -> State = State0#{jwk := JWK}, {noreply, State, {continue, create_token}}; @@ -135,6 +135,17 @@ handle_continue({make_key, PrivateKeyPEM}, State0) -> Error = {invalid_private_key, Error0}, ?tp(connector_jwt_worker_startup_error, #{error => Error}), {stop, {shutdown, {error, Error}}, State0} + catch + Kind:Error -> + ?tp( + error, + connector_jwt_worker_startup_error, + #{ + kind => Kind, + error => Error + } + ), + {stop, {shutdown, {error, Error}}, State0} end; handle_continue(create_token, State0) -> State = generate_and_store_jwt(State0), diff --git a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl b/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl index eb104801c..a079d632f 100644 --- a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl @@ -364,3 +364,23 @@ t_unknown_requests(_Config) -> gen_server:cast(Worker, unknown_cast), ?assertEqual({error, bad_call}, gen_server:call(Worker, unknown_call)), ok. + +t_truncated_private_key(_Config) -> + Config0 = generate_config(), + Config = Config0#{private_key := <<"-----BEGIN PRIVATE KEY-----\nMIIEvQI...">>}, + process_flag(trap_exit, true), + ?check_trace( + ?wait_async_action( + ?assertMatch({ok, _}, emqx_connector_jwt_worker:start_link(Config)), + #{?snk_kind := connector_jwt_worker_startup_error}, + 1_000 + ), + fun(Trace) -> + ?assertMatch( + [#{error := function_clause}], + ?of_kind(connector_jwt_worker_startup_error, Trace) + ), + ok + end + ), + ok. diff --git a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf index de76967ab..7e9fcd9e7 100644 --- a/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf +++ b/apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf @@ -26,7 +26,7 @@ emqx_resource_schema { desc { en: """The number of buffer workers. Only applicable for egress type bridges. For bridges only have ingress direction data flow, it can be set to 0 otherwise must be greater than 0.""" - zh: """缓存队列 worker 数量。仅对 egress 类型的桥接有意义。当桥接仅有 ingress 方向时，可设置为 0，否则必须大于 0）。""" + zh: """缓存队列 worker 数量。仅对 egress 类型的桥接有意义。当桥接仅有 ingress 方向时，可设置为 0，否则必须大于 0。""" } label { en: """Buffer Pool Size""" diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index e3a37fd10..6bd9fd213 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/changes/v5.0.15-en.md b/changes/v5.0.15-en.md new file mode 100644 index 000000000..f77753c25 --- /dev/null +++ b/changes/v5.0.15-en.md @@ -0,0 +1,79 @@ +# v5.0.15 + +## Enhancements + +- [#9569](https://github.com/emqx/emqx/pull/9569) Refactor `/authorization/sources/built_in_database/` by adding `rules/` to the path. + +- [#9585](https://github.com/emqx/emqx/pull/9585) `/bridges_probe` API endpoint to test params for creating a new data bridge. + +- [#9586](https://github.com/emqx/emqx/pull/9586) Basic auth is no longer allowed for API calls, must use API key instead. + +- [#9628](https://github.com/emqx/emqx/pull/9628) Expose additional resource configuration parameters: `start_after_created` and `start_timeout`. + +- [#9722](https://github.com/emqx/emqx/pull/9722) Add the following configuration options for Pushing metrics to Prometheus Push Gateway: + - `headers`: Allows custom HTTP request headers. + - `job_name`: allows to customize the name of the Job pushed to Push Gateway. + +- [#9725](https://github.com/emqx/emqx/pull/9725) Remove the config `auto_reconnect` from the emqx_authz, emqx_authn and data-bridge componets. + This is because we have another config with similar functions: `resource_opts.auto_restart_interval`。 + + The functions of these two config are difficult to distinguish, which will lead to confusion. + After this change, `auto_reconnect` will not be configurable (always be true), and the underlying + drivers that support this config will automatically reconnect the abnormally disconnected + connection every `2s`. + + And the config `resource_opts.auto_restart_interval` is still available for user. + It is the time interval that emqx restarts the resource when the connection cannot be + established for some reason. + +- [#9736](https://github.com/emqx/emqx/pull/9736) Refactor of /bridges API to make it more consistent with other APIs: + - bridge enable/disable is now done via the endpoint `/bridges/{id}/enable/[true,false]` + - `/bridges/{id}/operation/{operation}` endpoints are now `/bridges/{id}/{operation}` + - metrics are moved out from the GET `/bridges/{id}` response and can now be fetched via `/bridges/{id}/metrics` + - the `bridges/{id}/reset_metrics` endpoint is now `/bridges/{id}/metrics/reset` + +- [#9774](https://github.com/emqx/emqx/pull/9774) Add a password complexity requirement when adding or modifying Dashboard users via the API. + Now password must contain at least 2 of alphabetic, numeric and special characters, + and must be 8 to 64 characters long. + +## Bug fixes + +- [#9626](https://github.com/emqx/emqx/pull/9626) Return authorization settings with default values. + The authorization cache is enabled by default, but due to the missing default value in `GET` response of `/authorization/settings`, it seemed to be disabled from the dashboard. + +- [#9680](https://github.com/emqx/emqx/pull/9680) Fix the problem that username and password authentication is mandatory in Influxdb v1 write API. + +- [#9726](https://github.com/emqx/emqx/pull/9726) Client fuzzy search API results were missing information which could tell if more results are available in the next pages, this is now fixed by providing `hasnext` flag in the response. + +- [#9735](https://github.com/emqx/emqx/pull/9735) Password information has been removed from information log messages for http, ldap, mongo, mqtt, mysql, pgsql and redis. + +- [#9748](https://github.com/emqx/emqx/pull/9748) Listeners not configured with `max_connections` will cause the cluster `/listeners` API to return 500 error. + +- [#9749](https://github.com/emqx/emqx/pull/9749) In some cases search APIs could respond with an incorrect `count` value in the metadata, that is usually much bigger than expected, this is now fixed. + +- [#9750](https://github.com/emqx/emqx/pull/9750) Reload overriding configs after boot. + Prior to this change, two configs were allow to change from dashboard, but will not take effect after reboot: + * Logging (such as level) + * Prometheus configs + + +- [#9751](https://github.com/emqx/emqx/pull/9751) Fix that obsoleted cert file will not be deleted after the listener is updated/deleted + +- [#9763](https://github.com/emqx/emqx/pull/9763) Fix an authentication exception when password is not provided + +- [#9765](https://github.com/emqx/emqx/pull/9765) Parse decimals as password from environment variable overrides correctly. + Prior to this change, config values for passwords are not allowed to be decimals. + e.g. `EMQX_FOOBAR__PASSWORD=12344` or `emqx.foobar.password=1234` + would result in a type check error, unless quoted as: + `EMQX_FOOBAR__PASSWORD='"12344"'` or `emqx.foobar.password="1234"`. + After this fix, the value does not have to be quoted. + +- [#9769](https://github.com/emqx/emqx/pull/9769) Fix Erlang shell prompt version prefix. e5.0.15 -> v5.0.15 + +- [#9780](https://github.com/emqx/emqx/pull/9780) When creating disk queue directory for resource worker, substitute ':' with '-' in worker id. + +- [#9781](https://github.com/emqx/emqx/pull/9781) Trace files were left on a node when creating a zip file for download. They are now removed when the file is sent. Also, concurrent downloads will no longer interfere with each other. + +- [#9785](https://github.com/emqx/emqx/pull/9785) Stop authentication hook chain if `emqx_authentication` provides a definitive result. + +- [#9787](https://github.com/emqx/emqx/pull/9787) Fix a compatible problem for the `webhook` bridge configuration which was created before the v5.0.12. diff --git a/changes/v5.0.15-zh.md b/changes/v5.0.15-zh.md new file mode 100644 index 000000000..679298613 --- /dev/null +++ b/changes/v5.0.15-zh.md @@ -0,0 +1,76 @@ +# v5.0.15 + +## 增强 + +- [#9569](https://github.com/emqx/emqx/pull/9569) 重构 `/authorization/sources/built_in_database/` 接口，将 `rules/` 添加到了其路径中。 + +- [#9585](https://github.com/emqx/emqx/pull/9585) 添加新 API 接口 `/bridges_probe` 用于测试创建桥接的参数是否可用。 + +- [#9586](https://github.com/emqx/emqx/pull/9586) API 调用不再支持基于 `username:password` 的 `baisc` 认证, 现在 API 必须通过 API Key 才能进行调用。 + +- [#9628](https://github.com/emqx/emqx/pull/9628) 为桥接资源增加了配置参数：`start_after_created` 和 `start_timeout`。 + +- [#9722](https://github.com/emqx/emqx/pull/9722) 为 Prometheus 推送到 Push Gateway 新增以下配置项： + - `headers`：允许自定义 HTTP 请求头。 + - `job_name`：允许自定义推送到 Push Gateway 的 Job 名称。 + +- [#9725](https://github.com/emqx/emqx/pull/9725) 从认证、鉴权和数据桥接功能中，删除 `auto_reconnect` 配置项，因为我们还有另一个功能类似的配置项： + `resource_opts.auto_restart_interval`。 + + 这两个配置项的功能难以区分，会导致困惑。此修改之后，`auto_reconnect` 将不可配置(永远为 true)， + 支持此配置的底层驱动将以 `2s` 为周期自动重连异常断开的连接。 + + 而 `resource_opts.auto_restart_interval` 配置项仍然开放给用户配置，它是资源因为某些原因 + 无法建立连接的时候，emqx 重新启动该资源的时间间隔。 + +- [#9736](https://github.com/emqx/emqx/pull/9736) 重构部分 /bridges 的API 使得其和其他 API 能够更加一致： + - 桥接的启用和禁用现在是通过 `/bridges/{id}/enable/[true,false]` API 来实现的 + - 使用 `/bridges/{id}/{operation}` 替换了旧的 `/bridges/{id}/operation/{operation}` API + - 指标数据从 `/bridges/{id}` 的响应消息中移除，现在可以使用新的 API `/bridges/{id}/metrics` 进行访问 + - 使用 `/bridges/{id}/metrics/reset` 替换了旧的 `bridges/{id}/reset_metrics` API + +- [#9774](https://github.com/emqx/emqx/pull/9774) 通过 API 添加、修改 Dashboard 用户时，增加对密码复杂度的要求。 + 现在密码必须包含字母、数字以及特殊字符中的至少 2 种，并且长度范围必须是 8~64 个字符。 + +## 修复 + +- [#9626](https://github.com/emqx/emqx/pull/9626) 为授权设置 API 返回默认值。 + 授权缓存默认为开启，但是在此修复前，因为默认值在 `/authorization/settings` 这个 API 的返回值中缺失， + 使得在仪表盘配置页面中看起来是关闭了。 + +- [#9680](https://github.com/emqx/emqx/pull/9680) 修复 InfluxDB v1 桥接写入 API 配置中强制需要用户名密码认证的问题。 + +- [#9726](https://github.com/emqx/emqx/pull/9726) 在此修复前，客户端模糊搜索 API 缺少一些可以用于判断是否可以继续翻页的信息，现在通过在响应中提供 `hasnext` 标志来解决这个问题。 + +- [#9735](https://github.com/emqx/emqx/pull/9735) 密码信息已从http、ldap、mongo、mqtt、mysql、pgsql和redis的信息日志消息中删除。 + +- [#9748](https://github.com/emqx/emqx/pull/9748) 监听器不配置 `max_connections` 时会导致集群 `/listeners` 接口返回 500 错误。 + +- [#9749](https://github.com/emqx/emqx/pull/9749) 在某些情况下，搜索 API 可能会在元数据中响应不正确的 `count` 值，这通常比预期的要大得多，现在已经修复了。 + +- [#9750](https://github.com/emqx/emqx/pull/9750) 启动后重新加载一些重载配置项。 + 在此修复前，下面两个配置项允许从 Dashboard 控制台修改，但是在重启后无法生效： + * 日志 （例如日志级别） + * Prometheus 配置 + +- [#9751](https://github.com/emqx/emqx/pull/9751) 修复在更新或者删除监听器后，过时的证书文件没有被删除的问题。 + +- [#9763](https://github.com/emqx/emqx/pull/9763) 修复客户端没有提供密码时的一个异常 + +- [#9765](https://github.com/emqx/emqx/pull/9765) 允许使用纯数字作为密码配置。 + 在此修复前，密码的配置必须是字符串，使用纯数字时，会报类型检查错误。 + 例如，`EMQX_FOOBAR__PASSWORD=12344` 或 `emqx.foobar.password=1234` 会出错， + 必须用引把值括起来才行： + `EMQX_FOOBAR__PASSWORD='"12344"'` 或 `emqx.foobar.password="1234"`。 + 修复后可以不使用引号。在环境变量重载中使用更加方便。 + + +- [#9769](https://github.com/emqx/emqx/pull/9769) 修复 Eralng 控制台版本号前缀的打印错误 e5.0.15 -> v5.0.15 + +- [#9780](https://github.com/emqx/emqx/pull/9780) 在为资源缓存进程创建磁盘队列目录时，在ID中用 '-' 代替 ':'。 + +- [#9781](https://github.com/emqx/emqx/pull/9781) 当下载 日志追踪 的日志时，一些中间文件将存留在处理节点上，现在这个问题得到了修复。同时，并发下载日志将不再相互干扰。 + +- [#9785](https://github.com/emqx/emqx/pull/9785) 如果 `emqx_authentication` 提供了确定的结果，则停止认证钩子链。 + +- [#9787](https://github.com/emqx/emqx/pull/9787) 修复对在 v5.0.12 之前创建的 `webhook` 桥接配置的兼容问题。 diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md index 33a3fa22f..2899dc7e0 100644 --- a/deploy/charts/emqx-enterprise/README.md +++ b/deploy/charts/emqx-enterprise/README.md @@ -14,14 +14,14 @@ To install the chart with the release name `my-emqx`: + From github ``` $ git clone https://github.com/emqx/emqx.git - $ cd emqx/deploy/charts/emqx + $ cd emqx/deploy/charts/emqx-enterprise $ helm install my-emqx . ``` + From chart repos ``` helm repo add emqx https://repos.emqx.io/charts - helm install my-emqx emqx/emqx + helm install my-emqx emqx/emqx-enterprise ``` > If you want to install an unstable version, you need to add `--devel` when you execute the `helm install` command. @@ -43,6 +43,9 @@ The following table lists the configurable parameters of the emqx chart and thei | `image.repository` | EMQX Image name | `emqx/emqx-enterprise` | | `image.pullPolicy` | The image pull policy | IfNotPresent | | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | +| `serviceAccount.create` | If `true`, create a new service account | `true` | +| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the fullname template | | +| `serviceAccount.annotations` | Annotations to add to the service account | | | `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | | `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | | `podAnnotations ` | Annotations for pod | `{}` | @@ -102,10 +105,9 @@ The following table lists the configurable [EMQX](https://www.emqx.io/)-specific default values. Parameter | Description | Default Value --- | --- | --- -`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items -expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix -can be omitted) or using the configuration -files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil` +`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/v5.0/admin/cfg.html) items +expressed as [environment variables](https://www.emqx.io/docs/en/v5.0/admin/cfg.html#environment-variables) (prefix `EMQX_` can be omitted) or using the configuration +files [namespaced dotted notation](https://www.emqx.io/docs/en/v5.0/admin/cfg.html#syntax) | `nil` `emqxLicenseSecretName` | Name of the secret that holds the license information | `nil` ## SSL settings diff --git a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml index 0917cadac..91b6b1cb2 100644 --- a/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml +++ b/deploy/charts/emqx-enterprise/templates/StatefulSet.yaml @@ -52,6 +52,7 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }} {{- end }} spec: + serviceAccountName: {{ include "emqx.serviceAccountName" . }} volumes: {{- if .Values.ssl.enabled }} - name: ssl-cert @@ -73,9 +74,6 @@ spec: secret: secretName: {{ .Values.emqxLicenseSecretName }} {{- end }} - {{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} - serviceAccountName: {{ include "emqx.fullname" . }} - {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} @@ -107,13 +105,13 @@ spec: containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT__BIND | default 8084 }} - name: dashboard containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt - containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT }} + containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND }} {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS) }} + {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND) }} - name: dashboardtls - containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS }} + containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND }} {{- end }} - name: ekka containerPort: 4370 diff --git a/deploy/charts/emqx-enterprise/templates/_helpers.tpl b/deploy/charts/emqx-enterprise/templates/_helpers.tpl index e55740b01..64acf7961 100644 --- a/deploy/charts/emqx-enterprise/templates/_helpers.tpl +++ b/deploy/charts/emqx-enterprise/templates/_helpers.tpl @@ -42,3 +42,14 @@ Get ssl secret name . {{ include "emqx.fullname" . }}-tls {{- end -}} {{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "emqx.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "emqx.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/charts/emqx-enterprise/templates/rbac.yaml b/deploy/charts/emqx-enterprise/templates/rbac.yaml index f2cdd3601..9faf4e147 100644 --- a/deploy/charts/emqx-enterprise/templates/rbac.yaml +++ b/deploy/charts/emqx-enterprise/templates/rbac.yaml @@ -1,10 +1,23 @@ -{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} +{{- if .Values.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: + name: {{ include "emqx.serviceAccountName" . }} namespace: {{ .Release.Namespace }} - name: {{ include "emqx.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + --- +{{- if .Values.serviceAccount.create }} +{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }} kind: Role {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} apiVersion: rbac.authorization.k8s.io/v1 @@ -23,7 +36,12 @@ rules: - get - watch - list +{{- end }} +{{- end }} + --- +{{- if .Values.serviceAccount.create }} +{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }} kind: RoleBinding {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} apiVersion: rbac.authorization.k8s.io/v1 @@ -35,10 +53,11 @@ metadata: name: {{ include "emqx.fullname" . }} subjects: - kind: ServiceAccount - name: {{ include "emqx.fullname" . }} + name: {{ include "emqx.serviceAccountName" . }} namespace: {{ .Release.Namespace }} roleRef: kind: Role name: {{ include "emqx.fullname" . }} apiGroup: rbac.authorization.k8s.io {{- end }} +{{- end }} diff --git a/deploy/charts/emqx-enterprise/templates/service.yaml b/deploy/charts/emqx-enterprise/templates/service.yaml index 301213150..0fe3dc411 100644 --- a/deploy/charts/emqx-enterprise/templates/service.yaml +++ b/deploy/charts/emqx-enterprise/templates/service.yaml @@ -38,7 +38,7 @@ spec: {{- else if eq .Values.service.type "ClusterIP" }} nodePort: null {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt port: {{ .Values.service.internalmqtt | default 11883 }} protocol: TCP diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml index 0396b2b20..b9507c5a0 100644 --- a/deploy/charts/emqx-enterprise/values.yaml +++ b/deploy/charts/emqx-enterprise/values.yaml @@ -16,6 +16,15 @@ image: # pullSecrets: # - myRegistryKeySecretName +serviceAccount: + # Specifies whether a service account should be created + # If set false, means you need create service account by yourself + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + # Annotations to add to the service account + annotations: {} # The name of a secret in the same kubernetes namespace which contains values to # be added to the environment (must be manually created) diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index ae48f9de2..fb689839c 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.0.14 +version: 5.0.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.0.14 +appVersion: 5.0.15 diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 1a2090320..6ee3617ce 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -43,6 +43,9 @@ The following table lists the configurable parameters of the emqx chart and thei | `image.repository` | EMQX Image name | emqx/emqx | | `image.pullPolicy` | The image pull policy | IfNotPresent | | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | +| `serviceAccount.create` | If `true`, create a new service account | `true` | +| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the fullname template | | +| `serviceAccount.annotations` | Annotations to add to the service account | | | `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil | | `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false | | `podAnnotations ` | Annotations for pod | `{}` | diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 0917cadac..91b6b1cb2 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -52,6 +52,7 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }} {{- end }} spec: + serviceAccountName: {{ include "emqx.serviceAccountName" . }} volumes: {{- if .Values.ssl.enabled }} - name: ssl-cert @@ -73,9 +74,6 @@ spec: secret: secretName: {{ .Values.emqxLicenseSecretName }} {{- end }} - {{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} - serviceAccountName: {{ include "emqx.fullname" . }} - {{- end }} {{- if .Values.podSecurityContext.enabled }} securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }} {{- end }} @@ -107,13 +105,13 @@ spec: containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT__BIND | default 8084 }} - name: dashboard containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt - containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT }} + containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND }} {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS) }} + {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND) }} - name: dashboardtls - containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS }} + containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND }} {{- end }} - name: ekka containerPort: 4370 diff --git a/deploy/charts/emqx/templates/_helpers.tpl b/deploy/charts/emqx/templates/_helpers.tpl index e55740b01..64acf7961 100644 --- a/deploy/charts/emqx/templates/_helpers.tpl +++ b/deploy/charts/emqx/templates/_helpers.tpl @@ -42,3 +42,14 @@ Get ssl secret name . {{ include "emqx.fullname" . }}-tls {{- end -}} {{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "emqx.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "emqx.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/charts/emqx/templates/rbac.yaml b/deploy/charts/emqx/templates/rbac.yaml index f2cdd3601..9faf4e147 100644 --- a/deploy/charts/emqx/templates/rbac.yaml +++ b/deploy/charts/emqx/templates/rbac.yaml @@ -1,10 +1,23 @@ -{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}} +{{- if .Values.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: + name: {{ include "emqx.serviceAccountName" . }} namespace: {{ .Release.Namespace }} - name: {{ include "emqx.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + --- +{{- if .Values.serviceAccount.create }} +{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }} kind: Role {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} apiVersion: rbac.authorization.k8s.io/v1 @@ -23,7 +36,12 @@ rules: - get - watch - list +{{- end }} +{{- end }} + --- +{{- if .Values.serviceAccount.create }} +{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }} kind: RoleBinding {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }} apiVersion: rbac.authorization.k8s.io/v1 @@ -35,10 +53,11 @@ metadata: name: {{ include "emqx.fullname" . }} subjects: - kind: ServiceAccount - name: {{ include "emqx.fullname" . }} + name: {{ include "emqx.serviceAccountName" . }} namespace: {{ .Release.Namespace }} roleRef: kind: Role name: {{ include "emqx.fullname" . }} apiGroup: rbac.authorization.k8s.io {{- end }} +{{- end }} diff --git a/deploy/charts/emqx/templates/service.yaml b/deploy/charts/emqx/templates/service.yaml index 301213150..5b6376a85 100644 --- a/deploy/charts/emqx/templates/service.yaml +++ b/deploy/charts/emqx/templates/service.yaml @@ -121,7 +121,7 @@ spec: port: {{ .Values.service.mqtt | default 1883 }} protocol: TCP targetPort: mqtt - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt port: {{ .Values.service.internalmqtt | default 11883 }} protocol: TCP diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index 4fb263c7a..0423c8cdf 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -16,6 +16,15 @@ image: # pullSecrets: # - myRegistryKeySecretName +serviceAccount: + # Specifies whether a service account should be created + # If set false, means you need create service account by yourself + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + # Annotations to add to the service account + annotations: {} # The name of a secret in the same kubernetes namespace which contains values to # be added to the environment (must be manually created) diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index 11831ab06..b2a3c80c6 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl index bec3c49fa..25741b6cd 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl @@ -75,7 +75,16 @@ on_start(InstId, Config) -> }), throw(failed_to_start_kafka_client) end, - WolffProducerConfig = producers_config(BridgeName, ClientId, ProducerConfig), + %% Check if this is a dry run + TestIdStart = string:find(InstId, ?TEST_ID_PREFIX), + IsDryRun = + case TestIdStart of + nomatch -> + false; + _ -> + string:equal(TestIdStart, InstId) + end, + WolffProducerConfig = producers_config(BridgeName, ClientId, ProducerConfig, IsDryRun), case wolff:ensure_supervised_producers(ClientId, KafkaTopic, WolffProducerConfig) of {ok, Producers} -> {ok, #{ @@ -241,7 +250,7 @@ ssl(#{enable := true} = SSL) -> ssl(_) -> []. -producers_config(BridgeName, ClientId, Input) -> +producers_config(BridgeName, ClientId, Input, IsDryRun) -> #{ max_batch_bytes := MaxBatchBytes, compression := Compression, @@ -271,7 +280,7 @@ producers_config(BridgeName, ClientId, Input) -> BridgeType = kafka, ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName), #{ - name => make_producer_name(BridgeName), + name => make_producer_name(BridgeName, IsDryRun), partitioner => partitioner(PartitionStrategy), partition_count_refresh_interval_seconds => PCntRefreshInterval, replayq_dir => ReplayqDir, @@ -302,12 +311,20 @@ make_client_id(BridgeName) -> %% Producer name must be an atom which will be used as a ETS table name for %% partition worker lookup. -make_producer_name(BridgeName) when is_atom(BridgeName) -> - make_producer_name(atom_to_list(BridgeName)); -make_producer_name(BridgeName) -> - %% Woff needs atom for ets table name registration - %% The assumption here is bridge is not often re-created - binary_to_atom(iolist_to_binary(["kafka_producer_", BridgeName])). +make_producer_name(BridgeName, IsDryRun) when is_atom(BridgeName) -> + make_producer_name(atom_to_list(BridgeName), IsDryRun); +make_producer_name(BridgeName, IsDryRun) -> + %% Woff needs an atom for ets table name registration. The assumption here is + %% that bridges with new names are not often created. + case IsDryRun of + true -> + %% It is a dry run and we don't want to leak too many atoms + %% so we use the default producer name instead of creating + %% an unique name. + probing_wolff_producers; + false -> + binary_to_atom(iolist_to_binary(["kafka_producer_", BridgeName])) + end. with_log_at_error(Fun, Log) -> try