Merge pull request #10532 from id/0426-v5.0.24

0426 v5.0.24

.github/workflows/performance_test.yaml

@@ -24,14 +24,14 @@ jobs:
       with:
         fetch-depth: 0
         ref: ${{ github.event.inputs.ref }}
+    - name: Work around https://github.com/actions/checkout/issues/766
+      run: |
+        git config --global --add safe.directory "$GITHUB_WORKSPACE"
     - id: prepare
       run: |
         echo "EMQX_NAME=emqx" >> $GITHUB_ENV
         echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
         echo "BENCH_ID=$(date --utc +%F)/emqx-$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT
-    - name: Work around https://github.com/actions/checkout/issues/766
-      run: |
-        git config --global --add safe.directory "$GITHUB_WORKSPACE"
     - name: Build deb package
       run: |
         make ${EMQX_NAME}-pkg

apps/emqx/include/emqx_release.hrl

@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Community edition
--define(EMQX_RELEASE_CE, "5.0.23").
+-define(EMQX_RELEASE_CE, "5.0.24").
 
 %% Enterprise edition
 -define(EMQX_RELEASE_EE, "5.0.3-alpha.3").

changes/v5.0.24.en.md

@@ -0,0 +1,89 @@
+# v5.0.24
+
+## Enhancements
+
+- [#10457](https://github.com/emqx/emqx/pull/10457) Deprecates the integration with StatsD.
+
+  There seemd to be no user using StatsD integration, so we have decided to hide this feature
+  for now. We will either remove or revive it based on requirements in the future.
+
+- [#10458](https://github.com/emqx/emqx/pull/10458) Set the level of plugin configuration options to low level,
+  in most cases, users only need to manage plugins on the dashboard
+  without the need for manual modification, so we lowered the level.
+
+- [#10491](https://github.com/emqx/emqx/pull/10491) Rename `etcd.ssl` to `etcd.ssl_options` to keep all of SSL options consistent in the configuration file.
+
+- [#10512](https://github.com/emqx/emqx/pull/10512) Improved the storage format of Unicode characters in data files,
+  Now we can store Unicode characters normally.
+  For example: "SELECT * FROM \"t/1\" WHERE clientid = \"-测试专用-\""
+
+- [#10487](https://github.com/emqx/emqx/pull/10487) Optimize the instance of limiter for whose rate is `infinity` to reduce memory and CPU usage.
+
+- [#10490](https://github.com/emqx/emqx/pull/10490) Remove the default limit of connect rate which used to be `1000/s`
+
+## Bug Fixes
+
+- [#10407](https://github.com/emqx/emqx/pull/10407) Improve 'emqx_alarm' performance by using Mnesia dirty operations and avoiding
+  unnecessary calls from 'emqx_resource_manager' to reactivate alarms that have been already activated.
+  Use new safe 'emqx_alarm' API to activate/deactivate alarms to ensure that emqx_resource_manager
+  doesn't crash because of alarm timeouts.
+  The crashes were possible when the following conditions co-occurred:
+  - a relatively high number of failing resources, e.g. bridges tried to activate alarms on re-occurring errors;
+  - the system experienced a very high load.
+
+- [#10420](https://github.com/emqx/emqx/pull/10420) Fix HTTP path handling when composing the URL for the HTTP requests in authentication and authorization modules.
+  * Avoid unnecessary URL normalization since we cannot assume that external servers treat original and normalized URLs equally. This led to bugs like [#10411](https://github.com/emqx/emqx/issues/10411).
+  * Fix the issue that path segments could be HTTP encoded twice.
+
+- [#10422](https://github.com/emqx/emqx/pull/10422) Fixed a bug where external plugins could not be configured via environment variables in a lone-node cluster.
+
+- [#10448](https://github.com/emqx/emqx/pull/10448) Fix a compatibility issue of limiter configuration introduced by v5.0.23 which broke the upgrade from previous versions if the `capacity` is `infinity`.
+
+  In v5.0.23 we have replaced `capacity` with `burst`. After this fix, a `capacity = infinity` config will be automatically converted to equivalent `burst = 0`.
+
+- [#10449](https://github.com/emqx/emqx/pull/10449) Validate the ssl_options and header configurations when creating authentication http (`authn_http`).
+  Prior to this, incorrect `ssl` configuration could result in successful creation but the entire authn being unusable.
+
+- [#10455](https://github.com/emqx/emqx/pull/10455) Fixed an issue that could cause (otherwise harmless) noise in the logs.
+
+  During some particularly slow synchronous calls to bridges, some late replies could be sent to connections processes that were no longer expecting a reply, and then emit an error log like:
+
+  ```
+  2023-04-19T18:24:35.350233+00:00 [error] msg: unexpected_info, mfa: emqx_channel:handle_info/2, line: 1278, peername: 172.22.0.1:36384, clientid: caribdis_bench_sub_1137967633_4788, info: {#Ref<0.408802983.1941504010.189402>,{ok,200,[{<<"cache-control">>,<<"max-age=0, ...">>}}
+  ```
+
+  Those logs are harmless, but they could flood and worry the users without need.
+
+- [#10462](https://github.com/emqx/emqx/pull/10462) Deprecate config `broker.shared_dispatch_ack_enabled`.
+  This was designed to avoid dispatching messages to a shared-subscription session which has the client disconnected.
+  However since v5.0.9, this feature is no longer useful because the shared-subscrption messages in a expired session will be redispatched to other sessions in the group.
+  See also: https://github.com/emqx/emqx/pull/9104
+
+- [#10463](https://github.com/emqx/emqx/pull/10463) Improve bridges API error handling.
+  If Webhook bridge URL is not valid, bridges API will return '400' error instead of '500'.
+
+- [#10484](https://github.com/emqx/emqx/pull/10484) Fix the issue that the priority of the configuration cannot be set during rolling upgrade.
+  For example, when authorization is modified in v5.0.21 and then upgraded v5.0.23 through rolling upgrade,
+  the authorization will be restored to the default.
+
+- [#10495](https://github.com/emqx/emqx/pull/10495) Add the limiter API `/configs/limiter` which was deleted by mistake back.
+
+- [#10500](https://github.com/emqx/emqx/pull/10500) Add several fixes, enhancements and features in Mria:
+  - protect `mria:join/1,2` with a global lock to prevent conflicts between
+  two nodes trying to join each other simultaneously
+  [Mria PR](https://github.com/emqx/mria/pull/137)
+  - implement new function `mria:sync_transaction/4,3,2`, which blocks the caller until
+  a transaction is imported to the local node (if the local node is a replicant, otherwise,
+  it behaves exactly the same as `mria:transaction/3,2`)
+  [Mria PR](https://github.com/emqx/mria/pull/136)
+  - optimize `mria:running_nodes/0`
+  [Mria PR](https://github.com/emqx/mria/pull/135)
+  - optimize `mria:ro_transaction/2` when called on a replicant node
+  [Mria PR](https://github.com/emqx/mria/pull/134).
+
+- [#10518](https://github.com/emqx/emqx/pull/10518) Add the following fixes and features in Mria:
+  - call `mria_rlog:role/1` safely in mria_membership to ensure that mria_membership
+  gen_server won't crash if RPC to another node fails
+  [Mria PR](https://github.com/emqx/mria/pull/139)
+  - Add extra field to ?rlog_sync table to facilitate extending this functionality in future
+  [Mria PR](https://github.com/emqx/mria/pull/138).

deploy/charts/emqx/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.0.23
+version: 5.0.24
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.0.23
+appVersion: 5.0.24

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl

@@ -291,7 +291,7 @@ t_setup_via_config_and_publish(Config) ->
         end,
         fun(Trace0) ->
             Trace = ?of_kind(dynamo_connector_query_return, Trace0),
-            ?assertMatch([#{result := ok}], Trace),
+            ?assertMatch([#{result := {ok, _}}], Trace),
             ok
         end
     ),
@@ -328,7 +328,7 @@ t_setup_via_http_api_and_publish(Config) ->
         end,
         fun(Trace0) ->
             Trace = ?of_kind(dynamo_connector_query_return, Trace0),
-            ?assertMatch([#{result := ok}], Trace),
+            ?assertMatch([#{result := {ok, _}}], Trace),
             ok
         end
     ),

lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl

@@ -22,8 +22,6 @@
     on_stop/2,
     on_query/3,
     on_batch_query/3,
-    on_query_async/4,
-    on_batch_query_async/4,
     on_get_status/2
 ]).
 
@@ -60,7 +58,7 @@ fields(config) ->
 %% `emqx_resource' API
 %%========================================================================================
 
-callback_mode() -> async_if_possible.
+callback_mode() -> always_sync.
 
 is_buffer_supported() -> false.
 
@@ -117,32 +115,15 @@ on_stop(InstanceId, #{pool_name := PoolName}) ->
     emqx_resource_pool:stop(PoolName).
 
 on_query(InstanceId, Query, State) ->
-    do_query(InstanceId, Query, sync, State).
+    do_query(InstanceId, Query, State).
-
-on_query_async(InstanceId, Query, ReplyCtx, State) ->
-    do_query(
-        InstanceId,
-        Query,
-        {async, ReplyCtx},
-        State
-    ).
 
 %% we only support batch insert
 on_batch_query(InstanceId, [{send_message, _} | _] = Query, State) ->
-    do_query(InstanceId, Query, sync, State);
+    do_query(InstanceId, Query, State);
 on_batch_query(_InstanceId, Query, _State) ->
     {error, {unrecoverable_error, {invalid_request, Query}}}.
 
 %% we only support batch insert
-on_batch_query_async(InstanceId, [{send_message, _} | _] = Query, ReplyCtx, State) ->
-    do_query(
-        InstanceId,
-        Query,
-        {async, ReplyCtx},
-        State
-    );
-on_batch_query_async(_InstanceId, Query, _Reply, _State) ->
-    {error, {unrecoverable_error, {invalid_request, Query}}}.
 
 on_get_status(_InstanceId, #{pool_name := Pool}) ->
     Health = emqx_resource_pool:health_check_workers(
@@ -160,7 +141,6 @@ status_result(_Status = false) -> connecting.
 do_query(
     InstanceId,
     Query,
-    ApplyMode,
     #{pool_name := PoolName, templates := Templates, table := Table} = State
 ) ->
     ?TRACE(
@@ -170,7 +150,7 @@ do_query(
     ),
     Result = ecpool:pick_and_do(
         PoolName,
-        {emqx_ee_connector_dynamo_client, query, [ApplyMode, Table, Query, Templates]},
+        {emqx_ee_connector_dynamo_client, query, [Table, Query, Templates]},
         no_handover
     ),
 

lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl

@@ -9,7 +9,6 @@
 -export([
     start_link/1,
     is_connected/1,
-    query/5,
     query/4
 ]).
 
@@ -28,22 +27,22 @@
 -export([execute/2]).
 -endif.
 
+%% The default timeout for DynamoDB REST API calls is 10 seconds,
+%% but this value for `gen_server:call` is 5s,
+%% so we should pass the timeout to `gen_server:call`
+-define(HEALTH_CHECK_TIMEOUT, 10000).
+
 %%%===================================================================
 %%% API
 %%%===================================================================
 is_connected(Pid) ->
     try
-        gen_server:call(Pid, is_connected)
+        gen_server:call(Pid, is_connected, ?HEALTH_CHECK_TIMEOUT)
     catch
         _:_ ->
             false
     end.
 
-query(Pid, sync, Table, Query, Templates) ->
-    query(Pid, Table, Query, Templates);
-query(Pid, {async, ReplyCtx}, Table, Query, Templates) ->
-    gen_server:cast(Pid, {query, Table, Query, Templates, ReplyCtx}).
-
 query(Pid, Table, Query, Templates) ->
     gen_server:call(Pid, {query, Table, Query, Templates}, infinity).