From 3d7201502b5bebfd96c6a68587af12f76cedaa01 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 17 Apr 2023 17:04:24 +0200 Subject: [PATCH 1/7] perf(config): avoid build new map in emqx_mqtt_caps:get_caps --- apps/emqx/src/emqx_mqtt_caps.erl | 18 ++++++++++-------- changes/ce/perf-10525.en.md | 2 ++ 2 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 changes/ce/perf-10525.en.md diff --git a/apps/emqx/src/emqx_mqtt_caps.erl b/apps/emqx/src/emqx_mqtt_caps.erl index 1806ede1d..1ead953a6 100644 --- a/apps/emqx/src/emqx_mqtt_caps.erl +++ b/apps/emqx/src/emqx_mqtt_caps.erl @@ -88,7 +88,7 @@ check_pub(Zone, Flags) when is_map(Flags) -> error -> Flags end, - maps:with(?PUBCAP_KEYS, get_caps(Zone)) + get_caps(?PUBCAP_KEYS, Zone) ). do_check_pub(#{topic_levels := Levels}, #{max_topic_levels := Limit}) when @@ -111,7 +111,7 @@ do_check_pub(_Flags, _Caps) -> ) -> ok_or_error(emqx_types:reason_code()). check_sub(ClientInfo = #{zone := Zone}, Topic, SubOpts) -> - Caps = maps:with(?SUBCAP_KEYS, get_caps(Zone)), + Caps = get_caps(?SUBCAP_KEYS, Zone), Flags = lists:foldl( fun (max_topic_levels, Map) -> @@ -152,10 +152,12 @@ do_check_sub(_Flags, _Caps, _, _) -> ok. get_caps(Zone) -> - lists:foldl( - fun({K, V}, Acc) -> - Acc#{K => emqx_config:get_zone_conf(Zone, [mqtt, K], V)} - end, - #{}, - maps:to_list(?DEFAULT_CAPS) + get_caps(maps:keys(?DEFAULT_CAPS), Zone). +get_caps(Keys, Zone) -> + maps:with( + Keys, + maps:merge( + ?DEFAULT_CAPS, + emqx_config:get_zone_conf(Zone, [mqtt]) + ) ). diff --git a/changes/ce/perf-10525.en.md b/changes/ce/perf-10525.en.md new file mode 100644 index 000000000..b67e88289 --- /dev/null +++ b/changes/ce/perf-10525.en.md @@ -0,0 +1,2 @@ +Reduce resource usage per MQTT packet handling. + From 265c9ea6689363fdec878e7b2d05683a6553ec34 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 27 Apr 2023 16:37:23 +0200 Subject: [PATCH 2/7] chore: bump emqx app vsn to 5.0.25 --- apps/emqx/src/emqx.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index d42478fea..5ca8fc797 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.24"}, + {vsn, "5.0.25"}, {modules, []}, {registered, []}, {applications, [ From 77405d9cd70c285d07ff8147b9fe3a688cae6ce7 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 28 Apr 2023 13:45:29 +0200 Subject: [PATCH 3/7] perf: keep defaults of zone.mqtt --- apps/emqx/src/emqx_zone_schema.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx/src/emqx_zone_schema.erl b/apps/emqx/src/emqx_zone_schema.erl index 5d6720986..ffa64572e 100644 --- a/apps/emqx/src/emqx_zone_schema.erl +++ b/apps/emqx/src/emqx_zone_schema.erl @@ -61,6 +61,9 @@ hidden() -> "flapping_detect" ]. +%% optimized for mqtt field +fields("mqtt") -> + emqx_schema:fields("mqtt"); %% zone schemas are clones from the same name from root level %% only not allowed to have default values. fields(Name) -> From df0911be6ed74a7f6b28927d6cb18da2dedc20fc Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 2 May 2023 11:37:36 +0200 Subject: [PATCH 4/7] Revert "perf: keep defaults of zone.mqtt" This reverts commit 6184bc02ee30aa2d022793762e626893a8a32599. --- apps/emqx/src/emqx_zone_schema.erl | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/emqx/src/emqx_zone_schema.erl b/apps/emqx/src/emqx_zone_schema.erl index ffa64572e..5d6720986 100644 --- a/apps/emqx/src/emqx_zone_schema.erl +++ b/apps/emqx/src/emqx_zone_schema.erl @@ -61,9 +61,6 @@ hidden() -> "flapping_detect" ]. -%% optimized for mqtt field -fields("mqtt") -> - emqx_schema:fields("mqtt"); %% zone schemas are clones from the same name from root level %% only not allowed to have default values. fields(Name) -> From 9903f34470cf5222feb69e9ef447acbcebc92d88 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 2 May 2023 11:40:02 +0200 Subject: [PATCH 5/7] test(docker): set global mqtt config Set global mqtt config instead of default zone --- .ci/docker-compose-file/scripts/run-emqx.sh | 4 ++-- .github/workflows/run_fvt_tests.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/docker-compose-file/scripts/run-emqx.sh b/.ci/docker-compose-file/scripts/run-emqx.sh index 8f124aa63..fc61c9da1 100755 --- a/.ci/docker-compose-file/scripts/run-emqx.sh +++ b/.ci/docker-compose-file/scripts/run-emqx.sh @@ -20,8 +20,8 @@ esac { echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_" - echo "EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s" - echo "EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10" + echo "EMQX_MQTT__RETRY_INTERVAL=2s" + echo "EMQX_MQTT__MAX_TOPIC_ALIAS=10" echo "EMQX_AUTHORIZATION__SOURCES=[]" echo "EMQX_AUTHORIZATION__NO_MATCH=allow" } >> .ci/docker-compose-file/conf.cluster.env diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 1bb0486f1..185c76be1 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -167,8 +167,8 @@ jobs: --set image.pullPolicy=Never \ --set image.tag=$EMQX_TAG \ --set emqxAclConfig="" \ - --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \ - --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ + --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ deploy/charts/${{ matrix.profile }} \ @@ -185,8 +185,8 @@ jobs: --set image.pullPolicy=Never \ --set image.tag=$EMQX_TAG \ --set emqxAclConfig="" \ - --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \ - --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ + --set emqxConfig.EMQX_MQTT__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 \ --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \ --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \ deploy/charts/${{ matrix.profile }} \ From be6cd5705f8ffc212f65a1963c256b8e532e4407 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 2 May 2023 12:36:34 +0200 Subject: [PATCH 6/7] fix(mqtt-cap): use global config as defaults Don't use default values in module, use global mqtt config as default --- apps/emqx/src/emqx_mqtt_caps.erl | 28 +++++++++++++-------------- apps/emqx/test/emqx_channel_SUITE.erl | 2 ++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/apps/emqx/src/emqx_mqtt_caps.erl b/apps/emqx/src/emqx_mqtt_caps.erl index 1ead953a6..ec550bac5 100644 --- a/apps/emqx/src/emqx_mqtt_caps.erl +++ b/apps/emqx/src/emqx_mqtt_caps.erl @@ -58,18 +58,18 @@ exclusive_subscription ]). --define(DEFAULT_CAPS, #{ - max_packet_size => ?MAX_PACKET_SIZE, - max_clientid_len => ?MAX_CLIENTID_LEN, - max_topic_alias => ?MAX_TOPIC_AlIAS, - max_topic_levels => ?MAX_TOPIC_LEVELS, - max_qos_allowed => ?QOS_2, - retain_available => true, - wildcard_subscription => true, - subscription_identifiers => true, - shared_subscription => true, - exclusive_subscription => false -}). +-define(DEFAULT_CAPS_KEYS, [ + max_packet_size, + max_clientid_len, + max_topic_alias, + max_topic_levels, + max_qos_allowed, + retain_available, + wildcard_subscription, + subscription_identifiers, + shared_subscription, + exclusive_subscription +]). -spec check_pub( emqx_types:zone(), @@ -152,12 +152,12 @@ do_check_sub(_Flags, _Caps, _, _) -> ok. get_caps(Zone) -> - get_caps(maps:keys(?DEFAULT_CAPS), Zone). + get_caps(?DEFAULT_CAPS_KEYS, Zone). get_caps(Keys, Zone) -> maps:with( Keys, maps:merge( - ?DEFAULT_CAPS, + emqx_config:get([mqtt]), emqx_config:get_zone_conf(Zone, [mqtt]) ) ). diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index eccb5c865..d5265c1dd 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -267,6 +267,8 @@ t_chan_info(_) -> t_chan_caps(_) -> ?assertMatch( #{ + exclusive_subscription := false, + max_packet_size := 1048576, max_clientid_len := 65535, max_qos_allowed := 2, max_topic_alias := 65535, From e5884e0e87789f5b0da7ed25c710e2c77ec56126 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 2 May 2023 17:36:44 +0200 Subject: [PATCH 7/7] chore: remove unused mqtt cap 'subscription_identifiers' --- apps/emqx/src/emqx_mqtt_caps.erl | 2 -- apps/emqx/test/emqx_channel_SUITE.erl | 1 - 2 files changed, 3 deletions(-) diff --git a/apps/emqx/src/emqx_mqtt_caps.erl b/apps/emqx/src/emqx_mqtt_caps.erl index ec550bac5..897bb93c4 100644 --- a/apps/emqx/src/emqx_mqtt_caps.erl +++ b/apps/emqx/src/emqx_mqtt_caps.erl @@ -37,7 +37,6 @@ max_qos_allowed => emqx_types:qos(), retain_available => boolean(), wildcard_subscription => boolean(), - subscription_identifiers => boolean(), shared_subscription => boolean(), exclusive_subscription => boolean() }. @@ -66,7 +65,6 @@ max_qos_allowed, retain_available, wildcard_subscription, - subscription_identifiers, shared_subscription, exclusive_subscription ]). diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index d5265c1dd..717614430 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -275,7 +275,6 @@ t_chan_caps(_) -> max_topic_levels := Level, retain_available := true, shared_subscription := true, - subscription_identifiers := true, wildcard_subscription := true } when is_integer(Level), emqx_channel:caps(channel())