Merge remote-tracking branch 'upstream/release-56' into 0422-sync-release-56

This commit is contained in:
Ivan Dyachkov 2024-04-22 08:12:05 +02:00
commit b723e87e54
11 changed files with 177 additions and 23 deletions

View File

@ -9,3 +9,4 @@ accounts:
defaultGroupPerm: PUB|SUB defaultGroupPerm: PUB|SUB
topicPerms: topicPerms:
- TopicTest=PUB|SUB - TopicTest=PUB|SUB
- Topic2=PUB|SUB

View File

@ -32,7 +32,7 @@
%% `apps/emqx/src/bpapi/README.md' %% `apps/emqx/src/bpapi/README.md'
%% Opensource edition %% Opensource edition
-define(EMQX_RELEASE_CE, "5.6.1-beta.1"). -define(EMQX_RELEASE_CE, "5.6.1").
%% Enterprise edition %% Enterprise edition
-define(EMQX_RELEASE_EE, "5.6.1-beta.1"). -define(EMQX_RELEASE_EE, "5.6.1").

View File

@ -202,7 +202,7 @@ on_stop(InstanceId, _State) ->
({_, client_id, ClientId}) -> ({_, client_id, ClientId}) ->
destory_producers_map(ClientId), destory_producers_map(ClientId),
ok = rocketmq:stop_and_delete_supervised_client(ClientId); ok = rocketmq:stop_and_delete_supervised_client(ClientId);
({_, _Topic, Producer}) -> ({_, _ProducerGroup, Producer}) ->
_ = rocketmq:stop_and_delete_supervised_producers(Producer) _ = rocketmq:stop_and_delete_supervised_producers(Producer)
end, end,
emqx_resource:get_allocated_resources_list(InstanceId) emqx_resource:get_allocated_resources_list(InstanceId)
@ -258,7 +258,7 @@ do_query(
Data = apply_template(Query, Templates), Data = apply_template(Query, Templates),
Result = safe_do_produce( Result = safe_do_produce(
InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout ChannelId, InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout
), ),
case Result of case Result of
{error, Reason} -> {error, Reason} ->
@ -284,9 +284,11 @@ do_query(
get_channel_id({ChannelId, _}) -> ChannelId; get_channel_id({ChannelId, _}) -> ChannelId;
get_channel_id([{ChannelId, _} | _]) -> ChannelId. get_channel_id([{ChannelId, _} | _]) -> ChannelId.
safe_do_produce(InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout) -> safe_do_produce(
ChannelId, InstanceId, QueryFunc, ClientId, TopicKey, Data, ProducerOpts, RequestTimeout
) ->
try try
Producers = get_producers(InstanceId, ClientId, TopicKey, ProducerOpts), Producers = get_producers(ChannelId, InstanceId, ClientId, TopicKey, ProducerOpts),
produce(InstanceId, QueryFunc, Producers, Data, RequestTimeout) produce(InstanceId, QueryFunc, Producers, Data, RequestTimeout)
catch catch
_Type:Reason -> _Type:Reason ->
@ -391,16 +393,21 @@ destory_producers_map(ClientId) ->
ets:delete(Tid) ets:delete(Tid)
end. end.
get_producers(InstanceId, ClientId, Topic, ProducerOpts) -> get_producers(ChannelId, InstanceId, ClientId, Topic, ProducerOpts) ->
case ets:lookup(ClientId, Topic) of %% The topic need to be included in the name since we can have multiple
%% topics per channel due to templating.
ProducerGroup = iolist_to_binary([ChannelId, "_", Topic]),
case ets:lookup(ClientId, ProducerGroup) of
[{_, Producers}] -> [{_, Producers}] ->
Producers; Producers;
_ -> _ ->
ProducerGroup = iolist_to_binary([atom_to_list(ClientId), "_", Topic]), %% TODO: the name needs to be an atom but this may cause atom leak so we
%% should figure out a way to avoid this
ProducerOpts2 = ProducerOpts#{name => binary_to_atom(ProducerGroup)},
{ok, Producers} = rocketmq:ensure_supervised_producers( {ok, Producers} = rocketmq:ensure_supervised_producers(
ClientId, ProducerGroup, Topic, ProducerOpts ClientId, ProducerGroup, Topic, ProducerOpts2
), ),
ok = emqx_resource:allocate_resource(InstanceId, Topic, Producers), ok = emqx_resource:allocate_resource(InstanceId, ProducerGroup, Producers),
ets:insert(ClientId, {Topic, Producers}), ets:insert(ClientId, {ProducerGroup, Producers}),
Producers Producers
end. end.

View File

@ -263,6 +263,60 @@ t_setup_via_http_api_and_publish(Config) ->
), ),
ok. ok.
t_setup_two_actions_via_http_api_and_publish(Config) ->
BridgeType = ?GET_CONFIG(rocketmq_bridge_type, Config),
Name = ?GET_CONFIG(rocketmq_name, Config),
RocketMQConf = ?GET_CONFIG(rocketmq_config, Config),
RocketMQConf2 = RocketMQConf#{
<<"name">> => Name,
<<"type">> => BridgeType
},
?assertMatch(
{ok, _},
create_bridge_http(RocketMQConf2)
),
{ok, #{raw_config := ActionConf}} = emqx_bridge_v2:lookup(actions, BridgeType, Name),
Topic2 = <<"Topic2">>,
ActionConf2 = emqx_utils_maps:deep_force_put(
[<<"parameters">>, <<"topic">>], ActionConf, Topic2
),
Action2Name = atom_to_binary(?FUNCTION_NAME),
{ok, _} = emqx_bridge_v2:create(BridgeType, Action2Name, ActionConf2),
SentData = #{payload => ?PAYLOAD},
?check_trace(
begin
?wait_async_action(
?assertEqual(ok, send_message(Config, SentData)),
#{?snk_kind := rocketmq_connector_query_return},
10_000
),
ok
end,
fun(Trace0) ->
Trace = ?of_kind(rocketmq_connector_query_return, Trace0),
?assertMatch([#{result := ok}], Trace),
ok
end
),
Config2 = proplists:delete(rocketmq_name, Config),
Config3 = [{rocketmq_name, Action2Name} | Config2],
?check_trace(
begin
?wait_async_action(
?assertEqual(ok, send_message(Config3, SentData)),
#{?snk_kind := rocketmq_connector_query_return},
10_000
),
ok
end,
fun(Trace0) ->
Trace = ?of_kind(rocketmq_connector_query_return, Trace0),
?assertMatch([#{result := ok}], Trace),
ok
end
),
ok.
t_get_status(Config) -> t_get_status(Config) ->
?assertMatch( ?assertMatch(
{ok, _}, {ok, _},

2
build
View File

@ -493,7 +493,7 @@ make_docker() {
if [ "${DOCKER_PUSH:-false}" = true ]; then if [ "${DOCKER_PUSH:-false}" = true ]; then
DOCKER_BUILDX_ARGS+=(--push) DOCKER_BUILDX_ARGS+=(--push)
fi fi
if [ "${DOCKER_LOAD:-false}" = true ]; then if [ "${DOCKER_LOAD:-true}" = true ]; then
DOCKER_BUILDX_ARGS+=(--load) DOCKER_BUILDX_ARGS+=(--load)
fi fi
if [ -d "${REBAR_GIT_CACHE_DIR:-}" ]; then if [ -d "${REBAR_GIT_CACHE_DIR:-}" ]; then

46
changes/e5.6.1.en.md Normal file
View File

@ -0,0 +1,46 @@
# e5.6.1
## Bug Fixes
- [#12759](https://github.com/emqx/emqx/pull/12759) Do not save invalid uploaded backup files.
- [#12766](https://github.com/emqx/emqx/pull/12766) Rename `message_queue_too_long` error reason to `mailbox_overflow`
`mailbox_overflow` is consistent with the corresponding config parameter: `force_shutdown.max_mailbox_size`.
- [#12773](https://github.com/emqx/emqx/pull/12773) Upgrade HTTP client libraries.
The HTTP client library (`gun-1.3`) incorrectly appends a `:portnumber` suffix to the `Host` header for
standard ports (`http` on port 80, `https` on port 443). This could cause compatibility issues with servers or
gateways performing strict `Host` header checks (e.g., AWS Lambda, Alibaba Cloud HTTP gateways), leading to
errors such as `InvalidCustomDomain.NotFound` or "The specified CustomDomain does not exist."
- [#12802](https://github.com/emqx/emqx/pull/12802) Improve cluster discovery behaviour when a node is manually removed from a cluster using 'emqx ctl cluster leave' command.
Previously, if the configured cluster 'discovery_strategy' was not 'manual', the left node might re-discover and re-join the same cluster shortly after it left (unless it was stopped).
After this change, 'cluster leave' command disables automatic cluster_discovery, so that the left node won't re-join the same cluster again. Cluster discovery can be re-enabled by running 'emqx ctl discovery enable` or by restarting the left node.
- [#12814](https://github.com/emqx/emqx/pull/12814) Handle several errors in `/clients/{clientid}/mqueue_messages` and `/clients/{clientid}/inflight_messages` APIs:
- Internal timeout, which means that EMQX failed to get the list of Inflight/Mqueue messages within the default timeout of 5 s. This error may occur when the system is under a heavy load. The API will return 500 `{"code":"INTERNAL_ERROR","message":"timeout"}` response and log additional details.
- Client shutdown. The error may occur if the client connection is shutdown during the API call. The API will return 404 `{"code": "CLIENT_SHUTDOWN", "message": "Client connection has been shutdown"}` response in this case.
- [#12824](https://github.com/emqx/emqx/pull/12824) Make sure stats `'subscribers.count'` `'subscribers.max'` countains shared-subscribers.
It only contains non-shared subscribers previously.
- [#12826](https://github.com/emqx/emqx/pull/12826) Fixed an issue that prevented importing source data integrations and retained messages.
Before the fix:
- source data integrations are ignored from the backup file
- importing the `mnesia` table for retained messages are not supported
- [#12843](https://github.com/emqx/emqx/pull/12843) Fixed `cluster_rpc_commit` transaction ID cleanup procedure after `cluster leave` on replicant nodes.
Previously, the transaction id of the core node would be deleted prematurely, blocking configuration updates on the core node.
- [#12882](https://github.com/emqx/emqx/pull/12882) The RocketMQ action has been fixed so that the topic configiuration works correctly. If more than one action used a single connector before this fix, all actions messages got delivered to the topic that was used first.
- [#12885](https://github.com/emqx/emqx/pull/12885) Fixed an issue when users were not able to see the "Retained Messages" under the "Monitoring" menu in the admin dashboard.
"Retained messages" backend API uses `qlc`, and `qlc` uses `file_sorter` that puts temporary files in the working directory by default, which is not writable by emqx user since 58d0f04.
This patch fixes this by making `/opt/emqx` directory owned by `emqx:emqx`.

View File

@ -0,0 +1 @@
The RocketMQ action has been fixed so that the topic configiuration works correctly. If more than one action used a single connector before this fix, all actions messages got delivered to the topic that was used first.

44
changes/v5.6.1.en.md Normal file
View File

@ -0,0 +1,44 @@
# v5.6.1
## Bug Fixes
- [#12759](https://github.com/emqx/emqx/pull/12759) Do not save invalid uploaded backup files.
- [#12766](https://github.com/emqx/emqx/pull/12766) Rename `message_queue_too_long` error reason to `mailbox_overflow`
`mailbox_overflow` is consistent with the corresponding config parameter: `force_shutdown.max_mailbox_size`.
- [#12773](https://github.com/emqx/emqx/pull/12773) Upgrade HTTP client libraries.
The HTTP client library (`gun-1.3`) incorrectly appends a `:portnumber` suffix to the `Host` header for
standard ports (`http` on port 80, `https` on port 443). This could cause compatibility issues with servers or
gateways performing strict `Host` header checks (e.g., AWS Lambda, Alibaba Cloud HTTP gateways), leading to
errors such as `InvalidCustomDomain.NotFound` or "The specified CustomDomain does not exist."
- [#12802](https://github.com/emqx/emqx/pull/12802) Improve cluster discovery behaviour when a node is manually removed from a cluster using 'emqx ctl cluster leave' command.
Previously, if the configured cluster 'discovery_strategy' was not 'manual', the left node might re-discover and re-join the same cluster shortly after it left (unless it was stopped).
After this change, 'cluster leave' command disables automatic cluster_discovery, so that the left node won't re-join the same cluster again. Cluster discovery can be re-enabled by running 'emqx ctl discovery enable` or by restarting the left node.
- [#12814](https://github.com/emqx/emqx/pull/12814) Handle several errors in `/clients/{clientid}/mqueue_messages` and `/clients/{clientid}/inflight_messages` APIs:
- Internal timeout, which means that EMQX failed to get the list of Inflight/Mqueue messages within the default timeout of 5 s. This error may occur when the system is under a heavy load. The API will return 500 `{"code":"INTERNAL_ERROR","message":"timeout"}` response and log additional details.
- Client shutdown. The error may occur if the client connection is shutdown during the API call. The API will return 404 `{"code": "CLIENT_SHUTDOWN", "message": "Client connection has been shutdown"}` response in this case.
- [#12824](https://github.com/emqx/emqx/pull/12824) Make sure stats `'subscribers.count'` `'subscribers.max'` countains shared-subscribers.
It only contains non-shared subscribers previously.
- [#12826](https://github.com/emqx/emqx/pull/12826) Fixed an issue that prevented importing source data integrations and retained messages.
Before the fix:
- source data integrations are ignored from the backup file
- importing the `mnesia` table for retained messages are not supported
- [#12843](https://github.com/emqx/emqx/pull/12843) Fixed `cluster_rpc_commit` transaction ID cleanup procedure after `cluster leave` on replicant nodes.
Previously, the transaction id of the core node would be deleted prematurely, blocking configuration updates on the core node.
- [#12885](https://github.com/emqx/emqx/pull/12885) Fixed an issue when users were not able to see the "Retained Messages" under the "Monitoring" menu in the admin dashboard.
"Retained messages" backend API uses `qlc`, and `qlc` uses `file_sorter` that puts temporary files in the working directory by default, which is not writable by emqx user since 58d0f04.
This patch fixes this by making `/opt/emqx` directory owned by `emqx:emqx`.

View File

@ -14,8 +14,8 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes # This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version. # to the chart and its templates, including the app version.
version: 5.6.1-beta.1 version: 5.6.1
# This is the version number of the application being deployed. This version number should be # This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. # incremented each time you make changes to the application.
appVersion: 5.6.1-beta.1 appVersion: 5.6.1

View File

@ -14,8 +14,8 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes # This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version. # to the chart and its templates, including the app version.
version: 5.6.1-beta.1 version: 5.6.1
# This is the version number of the application being deployed. This version number should be # This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. # incremented each time you make changes to the application.
appVersion: 5.6.1-beta.1 appVersion: 5.6.1

View File

@ -47,18 +47,19 @@ ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8 ENV LANG=C.UTF-8
COPY deploy/docker/docker-entrypoint.sh /usr/bin/ COPY deploy/docker/docker-entrypoint.sh /usr/bin/
COPY --from=builder /emqx-rel /opt/
RUN set -eu; \ RUN set -eu; \
apt-get update; \ apt-get update; \
apt-get install -y --no-install-recommends ca-certificates procps $(echo "${EXTRA_DEPS}" | tr ',' ' '); \ apt-get install -y --no-install-recommends ca-certificates procps $(echo "${EXTRA_DEPS}" | tr ',' ' '); \
rm -rf /var/lib/apt/lists/*; \ rm -rf /var/lib/apt/lists/*; \
find /opt/emqx -name 'swagger*.js.map' -exec rm {} +; \
ln -s /opt/emqx/bin/* /usr/local/bin/; \
groupadd -r -g 1000 emqx; \ groupadd -r -g 1000 emqx; \
useradd -r -m -u 1000 -g emqx emqx; \ useradd -r -m -u 1000 -g emqx emqx;
mkdir -p /opt/emqx/log /opt/emqx/data /opt/emqx/plugins; \
chown -R emqx:emqx /opt/emqx/log /opt/emqx/data /opt/emqx/plugins COPY --from=builder --chown=emqx:emqx /emqx-rel /opt/
RUN set -eu; \
find /opt/emqx -name 'swagger*.js.map' -exec rm {} +; \
ln -s /opt/emqx/bin/* /usr/local/bin/;
WORKDIR /opt/emqx WORKDIR /opt/emqx