chore: merge release-52 into master

This commit is contained in:
Ivan Dyachkov 2023-09-21 10:22:47 +02:00
commit 105bebc250
440 changed files with 349 additions and 905 deletions

View File

@ -15,8 +15,8 @@ endif
# Dashboard version
# from https://github.com/emqx/emqx-dashboard5
export EMQX_DASHBOARD_VERSION ?= v1.3.2
export EMQX_EE_DASHBOARD_VERSION ?= e1.2.0-beta.9
export EMQX_DASHBOARD_VERSION ?= v1.4.1
export EMQX_EE_DASHBOARD_VERSION ?= e1.2.1
# `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used
# In make 4.4+, for backward-compatibility the value from the original environment is used.

View File

@ -32,10 +32,10 @@
%% `apps/emqx/src/bpapi/README.md'
%% Opensource edition
-define(EMQX_RELEASE_CE, "5.1.6").
-define(EMQX_RELEASE_CE, "5.2.1").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.2.0-alpha.4").
-define(EMQX_RELEASE_EE, "5.2.1").
%% The HTTP API version
-define(EMQX_API_VERSION, "5.0").

View File

@ -28,7 +28,7 @@
{gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
{cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.13"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.14"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.0"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.16"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},

View File

@ -2,7 +2,7 @@
{application, emqx, [
{id, "emqx"},
{description, "EMQX Core"},
{vsn, "5.1.9"},
{vsn, "5.1.11"},
{modules, []},
{registered, []},
{applications, [

View File

@ -1,6 +1,6 @@
{application, emqx_conf, [
{description, "EMQX configuration management"},
{vsn, "0.1.27"},
{vsn, "0.1.28"},
{registered, []},
{mod, {emqx_conf_app, []}},
{applications, [kernel, stdlib, emqx_ctl]},

View File

@ -221,6 +221,16 @@ fields("cluster") ->
sc(
?R_REF(cluster_k8s),
#{}
)},
{"prevent_overlapping_partitions",
sc(
boolean(),
#{
mapping => "vm_args.-kernel prevent_overlapping_partitions",
desc => ?DESC(prevent_overlapping_partitions),
default => false,
importance => ?IMPORTANCE_HIDDEN
}
)}
];
fields(cluster_static) ->

View File

@ -2,7 +2,7 @@
{application, emqx_durable_storage, [
{description, "Message persistence and subscription replays for EMQX"},
% strict semver, bump manually!
{vsn, "0.1.3"},
{vsn, "0.1.4"},
{modules, []},
{registered, []},
{applications, [kernel, stdlib, rocksdb, gproc, mria]},

View File

@ -1,6 +1,6 @@
{application, emqx_eviction_agent, [
{description, "EMQX Eviction Agent"},
{vsn, "5.1.1"},
{vsn, "5.1.2"},
{registered, [
emqx_eviction_agent_sup,
emqx_eviction_agent,

View File

@ -2,7 +2,7 @@
{application, emqx_management, [
{description, "EMQX Management API and CLI"},
% strict semver, bump manually!
{vsn, "5.0.29"},
{vsn, "5.0.30"},
{modules, []},
{registered, [emqx_management_sup]},
{applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl, emqx_bridge_http]},

View File

@ -647,7 +647,7 @@ set_keepalive(put, #{bindings := #{clientid := ClientID}, body := Body}) ->
error ->
{400, 'BAD_REQUEST', "Interval Not Found"};
{ok, Interval} ->
case emqx_mgmt:set_keepalive(emqx_mgmt_util:urldecode(ClientID), Interval) of
case emqx_mgmt:set_keepalive(ClientID, Interval) of
ok -> lookup(#{clientid => ClientID});
{error, not_found} -> {404, ?CLIENTID_NOT_FOUND};
{error, Reason} -> {400, #{code => 'PARAMS_ERROR', message => Reason}}

View File

@ -45,11 +45,15 @@
%%--------------------------------------------------------------------
parse_action(#{function := ActionFunc} = Action) ->
{Mod, Func} = parse_action_func(ActionFunc),
#{
mod => Mod,
func => Func,
args => pre_process_args(Mod, Func, maps:get(args, Action, #{}))
}.
Res = #{mod => Mod, func => Func},
%% builtin_action_console don't have args field.
%% Attempting to save args to the console action config could cause validation issues
case Action of
#{args := Args} ->
Res#{args => pre_process_args(Mod, Func, Args)};
_ ->
Res
end.
%%--------------------------------------------------------------------
%% callbacks of emqx_rule_action

View File

@ -2,7 +2,7 @@
{application, emqx_rule_engine, [
{description, "EMQX Rule Engine"},
% strict semver, bump manually!
{vsn, "5.0.24"},
{vsn, "5.0.26"},
{modules, []},
{registered, [emqx_rule_engine_sup, emqx_rule_engine]},
{applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]},

View File

@ -525,6 +525,10 @@ do_format_action(#{mod := Mod, func := Func, args := Args}) ->
#{
function => printable_function_name(Mod, Func),
args => maps:remove(preprocessed_tmpl, Args)
};
do_format_action(#{mod := Mod, func := Func}) ->
#{
function => printable_function_name(Mod, Func)
}.
printable_function_name(emqx_rule_actions, Func) ->

View File

@ -98,10 +98,26 @@ fields("builtin_action_republish") ->
];
fields("builtin_action_console") ->
[
{function, ?HOCON(console, #{desc => ?DESC("console_function")})}
{function, ?HOCON(console, #{desc => ?DESC("console_function")})},
%% we may support some args for the console action in the future
%, {args, sc(map(), #{desc => "The arguments of the built-in 'console' action",
% default => #{}})}
%% "args" needs to be a reserved/ignored field in the schema
%% to maintain compatibility with rule data that may contain
%% it due to a validation bug in previous versions.
%% The "args" field was not validated by the HOCON schema before 5.2.0,
%% which allowed rules to be created with invalid "args" data.
%% In 5.2.1 the validation was added,
%% so existing rules saved with invalid "args" would now fail validation
%% To maintain backward compatibility for existing rule data that may contain invalid "args",
%% the field needs to be included in the schema even though it is not a valid field.
{args,
?HOCON(map(), #{
deprecated => true,
importance => ?IMPORTANCE_HIDDEN,
desc => "The arguments of the built-in 'console' action",
default => #{}
})}
];
fields("user_provided_function") ->
[

View File

@ -30,7 +30,7 @@ rule_engine.rules.my_rule {
metadata = {created_at = 1693918992079}
sql = \"select * from \\\"t/topic\\\" \"
actions = [
{function = console}
{function = console, args = {test = 1}}
{ function = republish
args = {
payload = \"${.}\"

View File

@ -1 +0,0 @@
开始发布Rocky Linux 9与Enterprise Linux 9兼容和 MacOS 12 软件包。

View File

@ -1,2 +0,0 @@
Add support for QUIC TLS password protected certificate file.

View File

@ -1 +0,0 @@
增加对 QUIC TLS 密码保护证书文件的支持。

View File

@ -1 +0,0 @@
Add support for OCSP stapling for SSL MQTT listeners.

View File

@ -1,2 +0,0 @@
Add `extraVolumeMounts` to EMQX Helm Chart, it will have the ability to mount the user-own files into the EMQX instance, for example, ACL rule files as mentioned in [#9052](https://github.com/emqx/emqx/issues/9052)
Done of [#10116](https://github.com/emqx/emqx/issues/10116)

View File

@ -1,2 +0,0 @@
`extraVolumeMounts` 添加到 EMQX Helm Chart 中,它将能够挂载用户自己的文件到 EMQX 实例中,例如在 [#9052](https://github.com/emqx/emqx/issues/9052) 中提到的 ACL 规则文件。
修复了 issue [#10116](https://github.com/emqx/emqx/issues/10116)

View File

@ -1,7 +0,0 @@
Change the priority of the configuration:
1. If it is a new installation of EMQX, the priority of configuration is `ENV > emqx.conf > HTTP API`.
2. If EMQX is upgraded from an old version (i.e., the cluster-override.conf file still exists in EMQX's data directory), then the configuration priority remains the same as before. That is, `HTTP API > ENV > emqx.conf`.
Deprecated data/configs/local-override.conf.
Stabilizing the HTTP API for hot updates.

View File

@ -1 +0,0 @@
Add CRL check support for TLS MQTT listeners.

View File

@ -1,7 +0,0 @@
Decouple the query mode from the underlying call mode for buffer
workers.
Prior to this change, setting the query mode of a resource
such as a bridge to `sync` would force the buffer to call the
underlying connector in a synchronous way, even if it supports async
calls.

View File

@ -1 +0,0 @@
Use 'label' from i18n file as 'summary' in OpenAPI spec.

View File

@ -1,4 +0,0 @@
Unregister Mnesia post commit hook when Mria is being stopped.
This fixes hook failures occasionally occurring on stopping/restarting Mria.
[Mria PR](https://github.com/emqx/mria/pull/133)

View File

@ -1 +0,0 @@
Add the option to customize `clusterIP` in Helm chart, so that a user may set it to a fixed IP.

View File

@ -1 +0,0 @@
Add command 'eval-ex' for Elixir expression evaluation.

View File

@ -1 +0,0 @@
Refactor the directory structure of all gateways.

View File

@ -1 +0,0 @@
重构所有网关的源码目录结构。

View File

@ -1,3 +0,0 @@
Add support for `async` query mode for most bridges.
Before this change, some bridges (Cassandra, MongoDB, MySQL, Postgres, Redis, RocketMQ, TDengine) were only allowed to be created with a `sync` query mode.

View File

@ -1 +0,0 @@
Now, the rule engine language's FROM clause supports both strings enclosed in double quotes (") and single quotes (').

View File

@ -1 +0,0 @@
现在,规则引擎语言的 FROM 子句支持使用双引号(")和单引号(')括起来的字符串。

View File

@ -1 +0,0 @@
Add `/rule_engine` API endpoint to manage configuration of rule engine.

View File

@ -1,2 +0,0 @@
More specific error messages when configure with bad max_heap_size value.
Log current value and the max value when the `message_queue_too_long` error is thrown.

View File

@ -1 +0,0 @@
Metrics now are not implicitly collected in places where API handlers don't make any use of them. Instead, a separate backplane RPC gathers cluster-wide metrics.

View File

@ -1,2 +0,0 @@
Deprecate the trace.payload_encode configuration.
Add payload_encode=[text,hidden,hex] option when creating a trace via HTTP API.

View File

@ -1,2 +0,0 @@
Unify the config formats for `cluster.core_nodes` and `cluster.statics.seeds`.
Now they both support formats in array `["emqx1@127.0.0.1", "emqx2@127.0.0.1"]` or semicolon-separated string `"emqx1@127.0.0.1,emqx2@127.0.0.1"`.

View File

@ -1,7 +0,0 @@
Hide a large number of advanced options to simplify the configuration file.
That includes `rewrite`, `topic_metric`, `persistent_session_store`, `overload_protection`,
`flapping_detect`, `conn_congestion`, `stats,auto_subscribe`, `broker_perf`,
`shared_subscription_group`, `slow_subs`, `ssl_options.user_lookup_fun` and some advance items
in `node` and `dashboard` section, [#10358](https://github.com/emqx/emqx/pull/10358),
[#10381](https://github.com/emqx/emqx/pull/10381), [#10385](https://github.com/emqx/emqx/pull/10385).

View File

@ -1 +0,0 @@
A new function to convert a formatted date to an integer timestamp has been added: date_to_unix_ts/3

View File

@ -1,2 +0,0 @@
Change the default queue mode for buffer workers to `memory_only`.
Before this change, the default queue mode was `volatile_offload`. When under high message rate pressure and when the resource is not keeping up with such rate, the buffer performance degraded a lot due to the constant disk operations.

View File

@ -1,4 +0,0 @@
Optimize the configuration priority mechanism to fix the issue where the configuration
changes made to `etc/emqx.conf` do not take effect after restarting EMQX.
More introduction about the new mechanism: [Configure Override Rules](https://www.emqx.io/docs/en/v5.0/configuration/configuration.html#configure-override-rules)

View File

@ -1,4 +0,0 @@
Deprecates the integration with StatsD.
There seemd to be no user using StatsD integration, so we have decided to hide this feature
for now. We will either remove or revive it based on requirements in the future.

View File

@ -1,3 +0,0 @@
Set the level of plugin configuration options to low level,
in most cases, users only need to manage plugins on the dashboard
without the need for manual modification, so we lowered the level.

View File

@ -1 +0,0 @@
Rename `etcd.ssl` to `etcd.ssl_options` to keep all of SSL options consistent in the configuration file.

View File

@ -1,3 +0,0 @@
Improved the storage format of Unicode characters in data files,
Now we can store Unicode characters normally.
For example: "SELECT * FROM \"t/1\" WHERE clientid = \"-测试专用-\""

View File

@ -1 +0,0 @@
Add shutdown counter information to `emqx ctl listeners` command

View File

@ -1,2 +0,0 @@
Do not emit useless crash report when EMQX stops.
Previously, when EMQX (and `emqx_topic_metrics` in particular) stopped and removed underlying tables, some messages were still being handled and crashed.

View File

@ -1 +0,0 @@
Add log level configuration to SSL communication

View File

@ -1,2 +0,0 @@
Increase the time precision of trace logs from second to microsecond.
For example, change from `2023-05-02T08:43:50+00:00` to `2023-05-02T08:43:50.237945+00:00`.

View File

@ -1 +0,0 @@
Provide a callback method of Unary type in ExProto to avoid possible message disorder issues.

View File

@ -1 +0,0 @@
Renamed `max_message_queue_len` to `max_mailbox_size` in the `force_shutdown` configuration. Old name is kept as an alias, so this change is backward compatible.

View File

@ -1 +0,0 @@
The MongoDB connector and bridge have been refactored to a separate app to improve code structure.

View File

@ -1,4 +0,0 @@
Implement configuration and user data import/export CLI.
The `emqx ctl export` and `emqx ctl import` commands allow to export configuration and built-in database
data from a running EMQX cluster and later import it to the same or another running EMQX cluster.

View File

@ -1 +0,0 @@
This change allows to set the minReadySeconds for the StatefulSet. This allows to add a gap between the restarts of each pod by upgrade or restart command.

View File

@ -1,4 +0,0 @@
Introduce a more straightforward configuration option `keepalive_multiplier` and
deprecate the old `keepalive_backoff` configuration.
After this enhancement, EMQX checks the client's keepalive timeout status
period by multiplying the "Client Requested Keepalive Interval" with `keepalive_multiplier`.

View File

@ -1,3 +0,0 @@
We hide the request_timeout in resource_option of the webhook to keep it consistent with the http request_timeout of the webhook.
From now on, when configuring a webhook through API or configuration files,
it is no longer necessary to configure the request_timeout of the resource. Only configuring the http request_timeout is sufficient, and the request_timeout in the resource will automatically be consistent with the http request_timeout.

View File

@ -1,2 +0,0 @@
Added a new `deliver_rate` option to the retainer configuration, which can limit the maximum delivery rate per session in the retainer.

View File

@ -1 +0,0 @@
A new utility function timezone_to_offset_seconds/1 has been added to the rule engine SQL language. This function converts a timezone string (for example, "+02:00", "Z" and "local") to the corresponding offset in seconds.

View File

@ -1 +0,0 @@
Remove the deprecated HTTP APIs for gateways

View File

@ -1,3 +0,0 @@
The data bridge resource option `auto_restart_interval` was deprecated in favor of `health_check_interval`, and `request_timeout` was renamed to `request_ttl`. Also, the default `request_ttl` value went from 15 seconds to 45 seconds.
The previous existence of both `auto_restart_interval` and `health_check_interval` was a source of confusion, as both parameters influenced the recovery of data bridges under failures. An inconsistent configuration of those two parameters could lead to messages being expired without a chance to retry. Now, `health_check_interval` is used both to control the periodicity of health checks that may transition the data bridge into `disconnected` or `connecting` states, as well as recovering from `disconnected`.

View File

@ -1,5 +0,0 @@
Allow 'enable' as well as 'enabled' as the state flag for listeners.
Prior to this change, listener can be enable/disabled by setting the 'true' or 'false' on the 'enabled' config.
This is slightly different naming comparing to other state flags in the system.
No the 'enable' flag is added as an aliase on listeners.

View File

@ -1 +0,0 @@
Upgrade Erlang/OTP to 25.3.2-1

View File

@ -1 +0,0 @@
Add support for configuring TCP keep-alive in MQTT/TCP and MQTT/SSL listeners

View File

@ -1,4 +0,0 @@
Add `live_connections` field for some HTTP APIs, i.e:
- `/monitor_current`, `/monitor_current/nodes/{node}`
- `/monitor/nodes/{node}`, `/monitor`
- `/node/{node}`, `/nodes`

View File

@ -1,3 +0,0 @@
Adds support for unlimited max connections for gateway listeners by allowing
infinity as a valid value for the `max_connections` field in the configuration
and HTTP API

View File

@ -1,2 +0,0 @@
Renamed emqx ctl command 'cluster_call' to 'conf cluster_sync'.
The old command 'cluster_call' is still a valid command, but not included in usage info.

View File

@ -1 +0,0 @@
Hide the broker and move the `broker.shared_subscription_strategy` to `mqtt.shared_subscription_strategy` as it belongs to `mqtt`.

View File

@ -1 +0,0 @@
The listener's authentication and zone related apis have been officially removed in version `5.1.0`.

View File

@ -1 +0,0 @@
Rename `log.file.to` to `log.file.path`.

View File

@ -1 +0,0 @@
Added info logs to indicate when buffered messages are dropped due to time-to-live (TTL) expiration.

View File

@ -1 +0,0 @@
Release packages for Amazon Linux 2023

View File

@ -1 +0,0 @@
Rename `deliver_rate` to `delivery_rate` in the configuration of `retainer`.

View File

@ -1 +0,0 @@
Refactors the dashboard listener configuration to use a nested `ssl_options` field for ssl settings.

View File

@ -1,4 +0,0 @@
- Change k8s `api_server` default value from `http://127.0.0.1:9091` to `https://kubernetes.default.svc:443`
- `emqx_ctl conf show cluster` no longer displays irrelevant configuration items, such as when `discovery_strategy=static`,
it will not display configuration information related to `etcd/k8s/dns`.
- Remove `zones`(deprecated config key) from `emqx_ctl conf show_keys`

View File

@ -1,2 +0,0 @@
Remove `/configs/limiter` api from `swagger.json`, only the api documentation was removed,
and the `/configs/limiter` api functionalities have not been changed.

View File

@ -1,4 +0,0 @@
Added 3 random SQL functions to the rule engine.
- random(): Generates a random number between 0 and 1 (0.0 =< X < 1.0).
- uuid_v4(): Generates a random UUID (version 4) string.
- uuid_v4_no_hyphen(): Generates a random UUID (version 4) string without hyphens.

View File

@ -1 +0,0 @@
Adding a new configuration API `/configs`(GET/PUT) that supports to reload the hocon format configuration file.

View File

@ -1 +0,0 @@
Unify the listener switch to `enable`, while being compatible with the previous `enabled`.

View File

@ -1 +0,0 @@
Support HTTP API for setting alarm watermark of license.

View File

@ -1,3 +0,0 @@
Add `/cluster/topology` HTTP API endpoint
`GET` request to the endpoint returns the cluster topology: connections between RLOG core and replicant nodes.

View File

@ -1 +0,0 @@
The Webhook/HTTP bridge has been refactored to its own Erlang application. This allows for more flexibility in the future, and also allows for the bridge to be run as a standalone application.

View File

@ -1 +0,0 @@
Release packages for Debian 12.

View File

@ -1 +0,0 @@
Updated `jq` dependency to version 0.3.10 which includes `oniguruma` library update to version 6.9.8 with few minor security fixes.

View File

@ -1 +0,0 @@
Updated RocksDB version to 1.8.0-emqx-1 via ekka update to 0.15.6.

View File

@ -1,3 +0,0 @@
Add `node.broker_pool_size`, `node.generic_pool_size`, `node.channel_cleanup_batch_size` options to EMQX configuration.
Tuning these options can significantly improve performance if cluster interconnect network latency is high.

View File

@ -1 +0,0 @@
Added option to configure detection of legacy protocol in MondoDB connectors and bridges.

View File

@ -1 +0,0 @@
Add a new API endpoint `DELETE /banned` to clear all `banned` data.

View File

@ -1,2 +0,0 @@
Changed the type of the `mqtt.mqx_packet_size` from string to byteSize to better represent the valid numeric range.
Strings will still be accepted for backwards compatibility.

View File

@ -1 +0,0 @@
Refactored datetime-related modules and functions to simplify the code.

View File

@ -1 +0,0 @@
Added support for specifying username in Redis authentication.

View File

@ -1,2 +0,0 @@
The bcrypt work factor is limited to the range 5-10, because higher values consume too much CPU resources.
Bcrypt library is updated to allow parallel hash evaluation.

View File

@ -1 +0,0 @@
Disabled the Erlang VM Prometheus exporter by default to improve performance and security.

View File

@ -1,2 +0,0 @@
Enhanced broker metrics collection and export by adding new metrics for messages, overload protection, authorization, authentication,
and improving naming consistency for OpenTelemetry.

View File

@ -1 +0,0 @@
Add MQTT ingress to helm charts and update helm charts documentation

View File

@ -1 +0,0 @@
Ensure Monitor API `/monitor(_current)/nodes/:node` returns `404` instead of `400` if node does not exist.

View File

@ -1 +0,0 @@
如果 API 查询的节点不存在,将会返回 `404` 而不再是 `400`

View File

@ -1 +0,0 @@
Metrics are now only exposed via the /bridges/:id/metrics endpoint. Metrics are no longer returned in other API operations such as getting the list of all bridges, or in the response when a bridge has been created.

View File

@ -1 +0,0 @@
现在只有显式调用 `/bridges/:id/metrics` 接口时才可以获得指标数据,而其他 API 接口将不再返回相关数据。

View File

@ -1 +0,0 @@
The configuration parameter `mqtt.max_awaiting_rel` was not functional and has now been corrected.

Some files were not shown because too many files have changed in this diff Show More