Merge pull request #12795 from id/0327-cut-5.6.0

chore: 5.6.0

apps/emqx/include/emqx_release.hrl

@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Opensource edition
--define(EMQX_RELEASE_CE, "5.6.0-rc.6").
+-define(EMQX_RELEASE_CE, "5.6.0").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.6.0-rc.6").
+-define(EMQX_RELEASE_EE, "5.6.0").

apps/emqx/test/emqx_bpapi_static_checks.erl

@@ -59,7 +59,8 @@
 -define(FORCE_DELETED_APIS, [
     {emqx_statsd, 1},
     {emqx_plugin_libs, 1},
-    {emqx_persistent_session, 1}
+    {emqx_persistent_session, 1},
+    {emqx_ds, 3}
 ]).
 %% List of known RPC backend modules:
 -define(RPC_MODULES, "gen_rpc, erpc, rpc, emqx_rpc").
@@ -119,17 +120,16 @@ check_compat(DumpFilenames) ->
 
 %% Note: sets nok flag
 -spec check_compat(fulldump(), fulldump()) -> ok.
-check_compat(Dump1 = #{release := Rel1}, Dump2 = #{release := Rel2}) ->
+check_compat(Dump1 = #{release := Rel1}, Dump2 = #{release := Rel2}) when Rel2 >= Rel1 ->
     check_api_immutability(Dump1, Dump2),
-    Rel2 >= Rel1 andalso
+    typecheck_apis(Dump1, Dump2);
-        typecheck_apis(Dump1, Dump2).
+check_compat(_, _) ->
+    ok.
 
 %% It's not allowed to change BPAPI modules. Check that no changes
 %% have been made. (sets nok flag)
 -spec check_api_immutability(fulldump(), fulldump()) -> ok.
-check_api_immutability(#{release := Rel1, api := APIs1}, #{release := Rel2, api := APIs2}) when
+check_api_immutability(#{release := Rel1, api := APIs1}, #{release := Rel2, api := APIs2}) ->
-    Rel2 >= Rel1
-->
     %% TODO: Handle API deprecation
     _ = maps:map(
         fun(Key = {API, Version}, Val) ->
@@ -137,14 +137,15 @@ check_api_immutability(#{release := Rel1, api := APIs1}, #{release := Rel2, api
                 Val ->
                     ok;
                 undefined ->
-                    case lists:member({API, Version}, ?FORCE_DELETED_APIS) of
+                    case lists:member(Key, ?FORCE_DELETED_APIS) of
                         true ->
                             ok;
                         false ->
                             setnok(),
                             logger:error(
-                                "API ~p v~p was removed in release ~p without being deprecated.",
+                                "API ~p v~p was removed in release ~p without being deprecated. "
-                                [API, Version, Rel2]
+                                "Old release: ~p",
+                                [API, Version, Rel2, Rel1]
                             )
                     end;
                 _Val ->
@@ -157,8 +158,6 @@ check_api_immutability(#{release := Rel1, api := APIs1}, #{release := Rel2, api
         end,
         APIs1
     ),
-    ok;
-check_api_immutability(_, _) ->
     ok.
 
 filter_calls(Calls) ->
@@ -181,8 +180,8 @@ typecheck_apis(
     AllCalls = filter_calls(AllCalls0),
     lists:foreach(
         fun({From, To}) ->
-            Caller = get_param_types(CallerSigs, From),
+            Caller = get_param_types(CallerSigs, From, From),
-            Callee = get_param_types(CalleeSigs, To),
+            Callee = get_param_types(CalleeSigs, From, To),
             %% TODO: check return types
             case typecheck_rpc(Caller, Callee) of
                 [] ->
@@ -226,8 +225,8 @@ typecheck_rpc(Caller, Callee) ->
         Callee
     ).
 
--spec get_param_types(dialyzer_dump(), emqx_bpapi:call()) -> param_types().
+%%-spec get_param_types(dialyzer_dump(), emqx_bpapi:call()) -> param_types().
-get_param_types(Signatures, {M, F, A}) ->
+get_param_types(Signatures, From, {M, F, A}) ->
     Arity = length(A),
     case Signatures of
         #{{M, F, Arity} := {_RetType, AttrTypes}} ->
@@ -235,7 +234,7 @@ get_param_types(Signatures, {M, F, A}) ->
             Arity = length(AttrTypes),
             maps:from_list(lists:zip(A, AttrTypes));
         _ ->
-            logger:critical("Call ~p:~p/~p is not found in PLT~n", [M, F, Arity]),
+            logger:critical("Call ~p:~p/~p from ~p is not found in PLT~n", [M, F, Arity, From]),
             error({badkey, {M, F, A}})
     end.
 

apps/emqx_bridge_hstreamdb/rebar.config

@@ -3,7 +3,8 @@
 {erl_opts, [debug_info]}.
 {deps, [
     {hstreamdb_erl,
-        {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.5.18+v0.18.1"}}},
+        {git, "https://github.com/hstreamdb/hstreamdb_erl.git",
+            {tag, "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"}}},
     {emqx, {path, "../../apps/emqx"}},
     {emqx_utils, {path, "../../apps/emqx_utils"}}
 ]}.

changes/e5.6.0.en.md

@@ -155,22 +155,18 @@
 
 - [#12336](https://github.com/emqx/emqx/pull/12336) Refined the approach to managing asynchronous tasks by segregating the cleanup of channels into its own dedicated pool. This separation addresses performance issues encountered during channels cleanup under conditions of high network latency, ensuring that such tasks do not impede the efficiency of other asynchronous operations, such as route cleanup.
 
-- [#12494](https://github.com/emqx/emqx/pull/12494) Improved MongoDB connector performance.
-
-  - [mongodb-erlang PR](https://github.com/emqx/mongodb-erlang/pull/41)
-
 - [#12725](https://github.com/emqx/emqx/pull/12725) Implemented REST API to list the available source types.
 
 - [#12746](https://github.com/emqx/emqx/pull/12746) Added `username` log field. If MQTT client is connected with a non-empty username the logs and traces will include `username` field.
 
-- [#12785](https://github.com/emqx/emqx/pull/12785) Added `timestamp_format` config option to log handers.
+- [#12785](https://github.com/emqx/emqx/pull/12785) Added `timestamp_format` configuration option to log handlers. This new option allows for the following settings:
 
-This new option supports the following values:
+  - `auto`: Automatically determines the timestamp format based on the log formatter being used.
-
-- `auto`: Automatically determines the timestamp format based on the log formatter being used.
     Utilizes `rfc3339` format for text formatters, and `epoch` format for JSON formatters.
-- `epoch`: Represents timestamps in microseconds precision Unix epoch format.
+
-- `rfc3339`: Uses RFC3339 compliant format for date-time strings. For example: `2024-03-26T11:52:19.777087+00:00`.
+  - `epoch`: Represents timestamps in microseconds precision Unix epoch format.
+
+  - `rfc3339`: Uses RFC3339 compliant format for date-time strings. For example, `2024-03-26T11:52:19.777087+00:00`.
 
 ## Bug Fixes
 
@@ -184,7 +180,6 @@ This new option supports the following values:
 
 - [#12492](https://github.com/emqx/emqx/pull/12492) EMQX now returns the `Receive-Maximum` property in the `CONNACK` message for MQTT v5 clients, aligning with protocol expectations. This implementation considers the minimum value of the client's `Receive-Maximum` setting and the server's `max_inflight` configuration as the limit for the number of inflight (unacknowledged) messages permitted. Previously, the determined value was not sent back to the client in the `CONNACK` message.
 
-- [#12500](https://github.com/emqx/emqx/pull/12500) The `GET /clients` and `GET /client/:clientid` HTTP APIs have been updated to include disconnected persistent sessions in their responses.
 
   NOTE: A current known issue with these enhanced API responses is that the total client count provided may exceed the actual number of clients due to the inclusion of disconnected sessions.
 
@@ -196,7 +191,6 @@ This new option supports the following values:
 
 - [#12541](https://github.com/emqx/emqx/pull/12541) Introduced a new configuration validation step for autocluster by DNS records to ensure compatibility between `node.name` and `cluster.discover_strategy`. Specifically, when utilizing the `dns` strategy with either `a` or `aaaa` record types, it is mandatory for all nodes to use a (static) IP address as the host name.
 
-- [#12562](https://github.com/emqx/emqx/pull/12562) Added a new configuration root: `durable_storage`. This configuration tree contains the settings related to the new persistent session feature.
 
 - [#12566](https://github.com/emqx/emqx/pull/12566) Enhanced the bootstrap file for REST API keys:
 
@@ -244,18 +238,17 @@ This new option supports the following values:
 
 - [#12715](https://github.com/emqx/emqx/pull/12715) Fixed a crash that could occur during configuration updates if the connector for the ingress data integration source had active channels.
 
-- [#12740](https://github.com/emqx/emqx/pull/12740) Fixed an issue when durable sessions could not be kicked out.
 
 - [#12767](https://github.com/emqx/emqx/pull/12767) Fixed issues encountered during upgrades from 5.0.1 to 5.5.1, specifically related to Kafka Producer configurations that led to upgrade failures. The correction ensures that Kafka Producer configurations are accurately transformed into the new action and connector configuration format required by EMQX version 5.5.1 and beyond.
 
-- [#12768](https://github.com/emqx/emqx/pull/12768) Fixed an issue which may occur when performing rolling upgrade, especially when upgrading from a version earlier than 5.4.0.
+- [#12768](https://github.com/emqx/emqx/pull/12768) Addressed a startup failure issue in EMQX version 5.4.0 and later, particularly noted during rolling upgrades from versions before 5.4.0. The issue was related to the initialization of the routing schema when both v1 and v2 routing tables were empty.
 
-When the cluster is empty (more precisely, routing tables are empty), try to additionally ask the cluster nodes for the routing schema in use, to make more informed decision about routing storage schema upon startup. This should make routing storage schema less likely to diverge across cluster nodes, especially when the cluster is composed of different versions of EMQX.
+  The node now attempts to retrieve the routing schema version in use across the cluster instead of using the v2 routing table by default when local routing tables are found empty at startup. This approach mitigates potential conflicts and reduces the chances of diverging routing storage schemas among cluster nodes, especially in a mixed-version cluster scenario. 
 
-The version also logs instructions for how to manually resolve if conflict is detected in a running cluster.
+  If conflict is detected in a running cluster, EMQX writes instructions on how to manually resolve it in the log as part of the error message with `critical` severity. The same error message and instructions will also be written on standard error to make sure this message will not get lost even if no log handler is configured.
 
-- [#12786](https://github.com/emqx/emqx/pull/12786) Added a strict check that prevents replicant nodes from connecting to the core nodes running with a different version of EMQX application.
+- [#12786](https://github.com/emqx/emqx/pull/12786) Added a strict check that prevents replicant nodes from connecting to core nodes running with a different version of EMQX application.
-Effectively it means that during the rolling upgrades the replicant nodes can only work if there is at least one core node with the matching EMQX release.
+This check ensures that during the rolling upgrades, the replicant nodes can only work when at least one core node is running the same EMQX release version.
 
 ## Breaking Changes
 
@@ -267,7 +260,7 @@ Effectively it means that during the rolling upgrades the replicant nodes can on
 - [#12634](https://github.com/emqx/emqx/pull/12634) Triple-quote string values in HOCON config files no longer support escape sequence.
 
   The detailed information can be found in [this pull request](https://github.com/emqx/hocon/pull/290).
-  Here is a summary for the impact on EMQX users:
+  Here is a summary of the impact on EMQX users:
 
   - EMQX 5.6 is the first version to generate triple-quote strings in `cluster.hocon`,
   meaning for generated configs, there is no compatibility issue.

changes/v5.6.0.en.md

@@ -102,10 +102,7 @@
 
 - [#12679](https://github.com/emqx/emqx/pull/12679) Upgraded docker image base from Debian 11 to Debian 12.
 
-- [#12700](https://github.com/emqx/emqx/pull/12700) Started supporting "b" and "B" unit in bytesize hocon fields.
+- [#12700](https://github.com/emqx/emqx/pull/12700) Started supporting "b" and "B" unit in bytesize hocon fields. For example, all three fields below will have the value of 1024 bytes:
-
-
-  For example, all three fields below will have the value of 1024 bytes:
 
   ```
   bytesize_field = "1024b"
@@ -136,14 +133,14 @@
 
 - [#12746](https://github.com/emqx/emqx/pull/12746) Added `username` log field. If MQTT client is connected with a non-empty username the logs and traces will include `username` field.
 
-- [#12785](https://github.com/emqx/emqx/pull/12785) Added `timestamp_format` config option to log handers.
+- [#12785](https://github.com/emqx/emqx/pull/12785) Added `timestamp_format` configuration option to log handlers. This new option allows for the following settings:
 
-This new option supports the following values:
+  - `auto`: Automatically determines the timestamp format based on the log formatter being used.
-
-- `auto`: Automatically determines the timestamp format based on the log formatter being used.
     Utilizes `rfc3339` format for text formatters, and `epoch` format for JSON formatters.
-- `epoch`: Represents timestamps in microseconds precision Unix epoch format.
+
-- `rfc3339`: Uses RFC3339 compliant format for date-time strings. For example: `2024-03-26T11:52:19.777087+00:00`.
+  - `epoch`: Represents timestamps in microseconds precision Unix epoch format.
+
+  - `rfc3339`: Uses RFC3339 compliant format for date-time strings. For example, `2024-03-26T11:52:19.777087+00:00`.
 
 ## Bug Fixes
 
@@ -205,14 +202,14 @@ This new option supports the following values:
 
 - [#12740](https://github.com/emqx/emqx/pull/12740) Fixed an issue when durable sessions could not be kicked out.
 
-- [#12768](https://github.com/emqx/emqx/pull/12768) Fixed an issue which may occur when performing rolling upgrade, especially when upgrading from a version earlier than 5.4.0.
+- [#12768](https://github.com/emqx/emqx/pull/12768) Addressed a startup failure issue in EMQX version 5.4.0 and later, particularly noted during rolling upgrades from versions before 5.4.0. The issue was related to the initialization of the routing schema when both v1 and v2 routing tables were empty.
 
-When the cluster is empty (more precisely, routing tables are empty), try to additionally ask the cluster nodes for the routing schema in use, to make more informed decision about routing storage schema upon startup. This should make routing storage schema less likely to diverge across cluster nodes, especially when the cluster is composed of different versions of EMQX.
+  The node now attempts to retrieve the routing schema version in use across the cluster instead of using the v2 routing table by default when local routing tables are found empty at startup. This approach mitigates potential conflicts and reduces the chances of diverging routing storage schemas among cluster nodes, especially in a mixed-version cluster scenario. 
 
-The version also logs instructions for how to manually resolve if conflict is detected in a running cluster.
+  If conflict is detected in a running cluster, EMQX writes instructions on how to manually resolve it in the log as part of the error message with `critical` severity. The same error message and instructions will also be written on standard error to make sure this message will not get lost even if no log handler is configured.
 
-- [#12786](https://github.com/emqx/emqx/pull/12786) Added a strict check that prevents replicant nodes from connecting to the core nodes running with a different version of EMQX application.
+- [#12786](https://github.com/emqx/emqx/pull/12786) Added a strict check that prevents replicant nodes from connecting to core nodes running with a different version of EMQX application.
-Effectively it means that during the rolling upgrades the replicant nodes can only work if there is at least one core node with the matching EMQX release.
+  This check ensures that during the rolling upgrades, the replicant nodes can only work when at least one core node is running the same EMQX release version.
 
 ## Breaking Changes
 
@@ -224,7 +221,7 @@ Effectively it means that during the rolling upgrades the replicant nodes can on
 - [#12634](https://github.com/emqx/emqx/pull/12634) Triple-quote string values in HOCON config files no longer support escape sequence.
 
   The detailed information can be found in [this pull request](https://github.com/emqx/hocon/pull/290).
-  Here is a summary for the impact on EMQX users:
+  Here is a summary of the impact on EMQX users:
 
   - EMQX 5.6 is the first version to generate triple-quote strings in `cluster.hocon`,
   meaning for generated configs, there is no compatibility issue.

deploy/charts/emqx-enterprise/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.6.0-rc.6
+version: 5.6.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.6.0-rc.6
+appVersion: 5.6.0

deploy/charts/emqx/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.6.0-rc.6
+version: 5.6.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.6.0-rc.6
+appVersion: 5.6.0

mix.exs

@@ -200,7 +200,8 @@ defmodule EMQXUmbrella.MixProject do
 
   defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
     [
-      {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1"},
+      {:hstreamdb_erl,
+       github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"},
       {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.13", override: true},
       {:wolff, github: "kafka4beam/wolff", tag: "1.10.2"},
       {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true},