Merge remote-tracking branch 'upstream/release-56' into 0422-sync-release-56

.ci/docker-compose-file/rocketmq/conf/plain_acl.yml

@@ -9,3 +9,4 @@ accounts:
     defaultGroupPerm: PUB|SUB
     topicPerms:
       - TopicTest=PUB|SUB
+      - Topic2=PUB|SUB

apps/emqx/include/emqx_release.hrl

@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Opensource edition
--define(EMQX_RELEASE_CE, "5.6.1-beta.1").
+-define(EMQX_RELEASE_CE, "5.6.1").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.6.1-beta.1").
+-define(EMQX_RELEASE_EE, "5.6.1").

apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl

@@ -202,7 +202,7 @@ on_stop(InstanceId, _State) ->
             ({_, client_id, ClientId}) ->
                 destory_producers_map(ClientId),
                 ok = rocketmq:stop_and_delete_supervised_client(ClientId);
-            ({_, _Topic, Producer}) ->
+            ({_, _ProducerGroup, Producer}) ->
                 _ = rocketmq:stop_and_delete_supervised_producers(Producer)
         end,
         emqx_resource:get_allocated_resources_list(InstanceId)
@@ -258,7 +258,7 @@ do_query(
     Data = apply_template(Query, Templates),
 
     Result = safe_do_produce(
-        InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout
+        ChannelId, InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout
     ),
     case Result of
         {error, Reason} ->
@@ -284,9 +284,11 @@ do_query(
 get_channel_id({ChannelId, _}) -> ChannelId;
 get_channel_id([{ChannelId, _} | _]) -> ChannelId.
 
-safe_do_produce(InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout) ->
+safe_do_produce(
+    ChannelId, InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout
+) ->
     try
-        Producers = get_producers(InstanceId, ClientId, TopicKey, ProducerOpts),
+        Producers = get_producers(ChannelId, InstanceId, ClientId, TopicKey, ProducerOpts),
         produce(InstanceId, QueryFunc, Producers, Data, RequestTimeout)
     catch
         _Type:Reason ->
@@ -391,16 +393,21 @@ destory_producers_map(ClientId) ->
             ets:delete(Tid)
     end.
 
-get_producers(InstanceId, ClientId, Topic, ProducerOpts) ->
-    case ets:lookup(ClientId, Topic) of
+get_producers(ChannelId, InstanceId, ClientId, Topic, ProducerOpts) ->
+    %% The topic need to be included in the name since we can have multiple
+    %% topics per channel due to templating.
+    ProducerGroup = iolist_to_binary([ChannelId, "_", Topic]),
+    case ets:lookup(ClientId, ProducerGroup) of
         [{_, Producers}] ->
             Producers;
         _ ->
-            ProducerGroup = iolist_to_binary([atom_to_list(ClientId), "_", Topic]),
+            %% TODO: the name needs to be an atom but this may cause atom leak so we
+            %% should figure out a way to avoid this
+            ProducerOpts2 = ProducerOpts#{name => binary_to_atom(ProducerGroup)},
             {ok, Producers} = rocketmq:ensure_supervised_producers(
-                ClientId, ProducerGroup, Topic, ProducerOpts
+                ClientId, ProducerGroup, Topic, ProducerOpts2
             ),
-            ok = emqx_resource:allocate_resource(InstanceId, Topic, Producers),
-            ets:insert(ClientId, {Topic, Producers}),
+            ok = emqx_resource:allocate_resource(InstanceId, ProducerGroup, Producers),
+            ets:insert(ClientId, {ProducerGroup, Producers}),
             Producers
     end.

apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl

@@ -263,6 +263,60 @@ t_setup_via_http_api_and_publish(Config) ->
     ),
     ok.
 
+t_setup_two_actions_via_http_api_and_publish(Config) ->
+    BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config),
+    Name = ?GET_CONFIG(rocketmq_name, Config),
+    RocketMQConf = ?GET_CONFIG(rocketmq_config, Config),
+    RocketMQConf2 = RocketMQConf#{
+        <<"name">> => Name,
+        <<"type">> => BridgeType
+    },
+    ?assertMatch(
+        {ok, _},
+        create_bridge_http(RocketMQConf2)
+    ),
+    {ok, #{raw_config := ActionConf}} = emqx_bridge_v2:lookup(actions, BridgeType, Name),
+    Topic2 = <<"Topic2">>,
+    ActionConf2 = emqx_utils_maps:deep_force_put(
+        [<<"parameters">>, <<"topic">>], ActionConf, Topic2
+    ),
+    Action2Name = atom_to_binary(?FUNCTION_NAME),
+    {ok, _} = emqx_bridge_v2:create(BridgeType, Action2Name, ActionConf2),
+    SentData = #{payload => ?PAYLOAD},
+    ?check_trace(
+        begin
+            ?wait_async_action(
+                ?assertEqual(ok, send_message(Config, SentData)),
+                #{?snk_kind := rocketmq_connector_query_return},
+                10_000
+            ),
+            ok
+        end,
+        fun(Trace0) ->
+            Trace = ?of_kind(rocketmq_connector_query_return, Trace0),
+            ?assertMatch([#{result := ok}], Trace),
+            ok
+        end
+    ),
+    Config2 = proplists:delete(rocketmq_name, Config),
+    Config3 = [{rocketmq_name, Action2Name} | Config2],
+    ?check_trace(
+        begin
+            ?wait_async_action(
+                ?assertEqual(ok, send_message(Config3, SentData)),
+                #{?snk_kind := rocketmq_connector_query_return},
+                10_000
+            ),
+            ok
+        end,
+        fun(Trace0) ->
+            Trace = ?of_kind(rocketmq_connector_query_return, Trace0),
+            ?assertMatch([#{result := ok}], Trace),
+            ok
+        end
+    ),
+    ok.
+
 t_get_status(Config) ->
     ?assertMatch(
         {ok, _},

build

@@ -493,7 +493,7 @@ make_docker() {
     if [ "${DOCKER_PUSH:-false}" = true ]; then
         DOCKER_BUILDX_ARGS+=(--push)
     fi
-    if [ "${DOCKER_LOAD:-false}" = true ]; then
+    if [ "${DOCKER_LOAD:-true}" = true ]; then
         DOCKER_BUILDX_ARGS+=(--load)
     fi
     if [ -d "${REBAR_GIT_CACHE_DIR:-}" ]; then

changes/e5.6.1.en.md

@@ -0,0 +1,46 @@
+# e5.6.1
+
+## Bug Fixes
+
+- [#12759](https://github.com/emqx/emqx/pull/12759) Do not save invalid uploaded backup files.
+
+- [#12766](https://github.com/emqx/emqx/pull/12766) Rename `message_queue_too_long` error reason to `mailbox_overflow`
+
+  `mailbox_overflow` is consistent with the corresponding config parameter: `force_shutdown.max_mailbox_size`.
+
+- [#12773](https://github.com/emqx/emqx/pull/12773) Upgrade HTTP client libraries.
+
+  The HTTP client library (`gun-1.3`) incorrectly appends a `:portnumber` suffix to the `Host` header for
+  standard ports (`http` on port 80, `https` on port 443). This could cause compatibility issues with servers or
+  gateways performing strict `Host` header checks (e.g., AWS Lambda, Alibaba Cloud HTTP gateways), leading to
+  errors such as `InvalidCustomDomain.NotFound` or "The specified CustomDomain does not exist."
+
+- [#12802](https://github.com/emqx/emqx/pull/12802) Improve cluster discovery behaviour when a node is manually removed from a cluster using 'emqx ctl cluster leave' command.
+  Previously, if the configured cluster 'discovery_strategy' was not 'manual', the left node might re-discover and re-join the same cluster shortly after it left (unless it was stopped).
+  After this change, 'cluster leave' command disables automatic cluster_discovery, so that the left node won't re-join the same cluster again. Cluster discovery can be re-enabled by running 'emqx ctl discovery enable` or by restarting the left node.
+
+- [#12814](https://github.com/emqx/emqx/pull/12814) Handle several errors in `/clients/{clientid}/mqueue_messages` and `/clients/{clientid}/inflight_messages` APIs:
+
+  - Internal timeout, which means that EMQX failed to get the list of Inflight/Mqueue messages within the default timeout of 5 s. This error may occur when the system is under a heavy load. The API will return 500 `{"code":"INTERNAL_ERROR","message":"timeout"}` response and log additional details.
+  - Client shutdown. The error may occur if the client connection is shutdown during the API call. The API will return 404 `{"code": "CLIENT_SHUTDOWN", "message": "Client connection has been shutdown"}` response in this case.
+
+- [#12824](https://github.com/emqx/emqx/pull/12824) Make sure stats `'subscribers.count'` `'subscribers.max'` countains shared-subscribers.
+  It only contains non-shared subscribers previously.
+
+- [#12826](https://github.com/emqx/emqx/pull/12826) Fixed an issue that prevented importing source data integrations and retained messages.
+
+  Before the fix:
+
+  - source data integrations are ignored from the backup file
+  - importing the `mnesia` table for retained messages are not supported
+
+- [#12843](https://github.com/emqx/emqx/pull/12843) Fixed `cluster_rpc_commit` transaction ID cleanup procedure after `cluster leave` on replicant nodes.
+  Previously, the transaction id of the core node would be deleted prematurely, blocking configuration updates on the core node.
+
+- [#12882](https://github.com/emqx/emqx/pull/12882) The RocketMQ action has been fixed so that the topic configiuration works correctly. If more than one action used a single connector before this fix, all actions messages got delivered to the topic that was used first.
+
+- [#12885](https://github.com/emqx/emqx/pull/12885) Fixed an issue when users were not able to see the "Retained Messages" under the "Monitoring" menu in the admin dashboard.
+
+"Retained messages" backend API uses `qlc`, and `qlc` uses `file_sorter` that puts temporary files in the working directory by default, which is not writable by emqx user since 58d0f04.
+
+This patch fixes this by making `/opt/emqx` directory owned by `emqx:emqx`.

changes/ee/fix-12882.en.md

@@ -0,0 +1 @@
+The RocketMQ action has been fixed so that the topic configiuration works correctly. If more than one action used a single connector before this fix, all actions messages got delivered to the topic that was used first.

changes/v5.6.1.en.md

@@ -0,0 +1,44 @@
+# v5.6.1
+
+## Bug Fixes
+
+- [#12759](https://github.com/emqx/emqx/pull/12759) Do not save invalid uploaded backup files.
+
+- [#12766](https://github.com/emqx/emqx/pull/12766) Rename `message_queue_too_long` error reason to `mailbox_overflow`
+
+  `mailbox_overflow` is consistent with the corresponding config parameter: `force_shutdown.max_mailbox_size`.
+
+- [#12773](https://github.com/emqx/emqx/pull/12773) Upgrade HTTP client libraries.
+
+  The HTTP client library (`gun-1.3`) incorrectly appends a `:portnumber` suffix to the `Host` header for
+  standard ports (`http` on port 80, `https` on port 443). This could cause compatibility issues with servers or
+  gateways performing strict `Host` header checks (e.g., AWS Lambda, Alibaba Cloud HTTP gateways), leading to
+  errors such as `InvalidCustomDomain.NotFound` or "The specified CustomDomain does not exist."
+
+- [#12802](https://github.com/emqx/emqx/pull/12802) Improve cluster discovery behaviour when a node is manually removed from a cluster using 'emqx ctl cluster leave' command.
+  Previously, if the configured cluster 'discovery_strategy' was not 'manual', the left node might re-discover and re-join the same cluster shortly after it left (unless it was stopped).
+  After this change, 'cluster leave' command disables automatic cluster_discovery, so that the left node won't re-join the same cluster again. Cluster discovery can be re-enabled by running 'emqx ctl discovery enable` or by restarting the left node.
+
+- [#12814](https://github.com/emqx/emqx/pull/12814) Handle several errors in `/clients/{clientid}/mqueue_messages` and `/clients/{clientid}/inflight_messages` APIs:
+
+  - Internal timeout, which means that EMQX failed to get the list of Inflight/Mqueue messages within the default timeout of 5 s. This error may occur when the system is under a heavy load. The API will return 500 `{"code":"INTERNAL_ERROR","message":"timeout"}` response and log additional details.
+  - Client shutdown. The error may occur if the client connection is shutdown during the API call. The API will return 404 `{"code": "CLIENT_SHUTDOWN", "message": "Client connection has been shutdown"}` response in this case.
+
+- [#12824](https://github.com/emqx/emqx/pull/12824) Make sure stats `'subscribers.count'` `'subscribers.max'` countains shared-subscribers.
+  It only contains non-shared subscribers previously.
+
+- [#12826](https://github.com/emqx/emqx/pull/12826) Fixed an issue that prevented importing source data integrations and retained messages.
+
+  Before the fix:
+
+  - source data integrations are ignored from the backup file
+  - importing the `mnesia` table for retained messages are not supported
+
+- [#12843](https://github.com/emqx/emqx/pull/12843) Fixed `cluster_rpc_commit` transaction ID cleanup procedure after `cluster leave` on replicant nodes.
+  Previously, the transaction id of the core node would be deleted prematurely, blocking configuration updates on the core node.
+
+- [#12885](https://github.com/emqx/emqx/pull/12885) Fixed an issue when users were not able to see the "Retained Messages" under the "Monitoring" menu in the admin dashboard.
+
+"Retained messages" backend API uses `qlc`, and `qlc` uses `file_sorter` that puts temporary files in the working directory by default, which is not writable by emqx user since 58d0f04.
+
+This patch fixes this by making `/opt/emqx` directory owned by `emqx:emqx`.

deploy/charts/emqx-enterprise/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.6.1-beta.1
+version: 5.6.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.6.1-beta.1
+appVersion: 5.6.1

deploy/charts/emqx/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.6.1-beta.1
+version: 5.6.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.6.1-beta.1
+appVersion: 5.6.1

deploy/docker/Dockerfile

@@ -47,18 +47,19 @@ ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
 COPY deploy/docker/docker-entrypoint.sh /usr/bin/
-COPY --from=builder /emqx-rel /opt/
 
 RUN set -eu; \
     apt-get update; \
     apt-get install -y --no-install-recommends ca-certificates procps $(echo "${EXTRA_DEPS}" | tr ',' ' '); \
     rm -rf /var/lib/apt/lists/*; \
-    find /opt/emqx -name 'swagger*.js.map' -exec rm {} +; \
-    ln -s /opt/emqx/bin/* /usr/local/bin/; \
     groupadd -r -g 1000 emqx; \
-    useradd -r -m -u 1000 -g emqx emqx; \
-    mkdir -p /opt/emqx/log /opt/emqx/data /opt/emqx/plugins; \
-    chown -R emqx:emqx /opt/emqx/log /opt/emqx/data /opt/emqx/plugins
+    useradd -r -m -u 1000 -g emqx emqx;
+
+COPY --from=builder --chown=emqx:emqx /emqx-rel /opt/
+
+RUN set -eu; \
+    find /opt/emqx -name 'swagger*.js.map' -exec rm {} +; \
+    ln -s /opt/emqx/bin/* /usr/local/bin/;
 
 WORKDIR /opt/emqx