From 08e1566490c8e8c1bfa7d7753a55df3339f0031f Mon Sep 17 00:00:00 2001
From: Mayer Maximilian <Maximilian.Mayer@liebherr.com>
Date: Mon, 24 Oct 2022 09:51:49 +0200
Subject: [PATCH 001/144] feat: Add pod disruption budget to helm chart

Now it is possible to define the pod disruption budget.

Closes: #8222
---
 deploy/charts/emqx/templates/pdb.yaml | 18 ++++++++++++++++++
 deploy/charts/emqx/values.yaml        |  7 +++++++
 2 files changed, 25 insertions(+)
 create mode 100644 deploy/charts/emqx/templates/pdb.yaml

diff --git a/deploy/charts/emqx/templates/pdb.yaml b/deploy/charts/emqx/templates/pdb.yaml
new file mode 100644
index 000000000..a3f233064
--- /dev/null
+++ b/deploy/charts/emqx/templates/pdb.yaml
@@ -0,0 +1,18 @@
+{{- if and (.Values.pdb.enabled) (.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "emqx.fullname" . }}-pdb
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  maxUnavailable: {{ .Values.pdb.maxUnavailable }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "emqx.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index b648f070f..7af0eff71 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -211,3 +211,10 @@ ssl:
   issuer:
     name: letsencrypt-dns
     kind: ClusterIssuer
+
+## Setting PodDisruptionBudget.
+## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb
+##
+pdb:
+  enabled: false
+  maxUnavailable: 1

From cf77dcf25eded3b3058b8192ce470f28edd11a38 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Thu, 2 Feb 2023 14:53:17 +0100
Subject: [PATCH 002/144] test(emqx_ee_connector): add basic tests for influxdb
 incl. SSL opts

This adds a test suite for the emqx_ee_connector_influxdb. We add it so
that SSL transport options are properly tested.
---
 lib-ee/emqx_ee_connector/docker-ct            |   2 +
 .../src/emqx_ee_connector_influxdb.erl        |  14 ++
 .../test/emqx_ee_connector_influxdb_SUITE.erl | 196 ++++++++++++++++++
 3 files changed, 212 insertions(+)
 create mode 100644 lib-ee/emqx_ee_connector/docker-ct
 create mode 100644 lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl

diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct
new file mode 100644
index 000000000..ef579c036
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/docker-ct
@@ -0,0 +1,2 @@
+toxiproxy
+influxdb
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index d689f4bf3..988c19156 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -29,6 +29,7 @@
 -export([reply_callback/2]).
 
 -export([
+    roots/0,
     namespace/0,
     fields/1,
     desc/1
@@ -139,6 +140,19 @@ on_get_status(_InstId, #{client := Client}) ->
 %% schema
 namespace() -> connector_influxdb.
 
+roots() ->
+    [
+        {config, #{
+            type => hoconsc:union(
+                [
+                    hoconsc:ref(?MODULE, influxdb_udp),
+                    hoconsc:ref(?MODULE, influxdb_api_v1),
+                    hoconsc:ref(?MODULE, influxdb_api_v2)
+                ]
+            )
+        }}
+    ].
+
 fields(common) ->
     [
         {server, server()},
diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
new file mode 100644
index 000000000..01bb8a08e
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
@@ -0,0 +1,196 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_connector_influxdb_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include("emqx_connector.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-define(INFLUXDB_RESOURCE_MOD, emqx_ee_connector_influxdb).
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+groups() ->
+    [].
+
+init_per_suite(Config) ->
+    InfluxDBTCPHost = os:getenv("INFLUXDB_APIV2_TCP_HOST", "toxiproxy"),
+    InfluxDBTCPPort = list_to_integer(os:getenv("INFLUXDB_APIV2_TCP_PORT", "8086")),
+    InfluxDBTLSHost = os:getenv("INFLUXDB_APIV2_TLS_HOST", "toxiproxy"),
+    InfluxDBTLSPort = list_to_integer(os:getenv("INFLUXDB_APIV2_TLS_PORT", "8087")),
+    case emqx_common_test_helpers:is_tcp_server_available(InfluxDBTCPHost, InfluxDBTCPPort) of
+        true ->
+            ok = emqx_common_test_helpers:start_apps([emqx_conf]),
+            ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
+            {ok, _} = application:ensure_all_started(emqx_connector),
+            [
+                {influxdb_tcp_host, InfluxDBTCPHost},
+                {influxdb_tcp_port, InfluxDBTCPPort},
+                {influxdb_tls_host, InfluxDBTLSHost},
+                {influxdb_tls_port, InfluxDBTLSPort}
+                | Config
+            ];
+        false ->
+            {skip, no_influxdb}
+    end.
+
+end_per_suite(_Config) ->
+    ok = emqx_common_test_helpers:stop_apps([emqx_conf]),
+    ok = emqx_connector_test_helpers:stop_apps([emqx_resource]),
+    _ = application:stop(emqx_connector).
+
+init_per_testcase(_, Config) ->
+    Config.
+
+end_per_testcase(_, _Config) ->
+    ok.
+
+% %%------------------------------------------------------------------------------
+% %% Testcases
+% %%------------------------------------------------------------------------------
+
+t_lifecycle(Config) ->
+    Host = ?config(influxdb_tcp_host, Config),
+    Port = ?config(influxdb_tcp_port, Config),
+    perform_lifecycle_check(
+        <<"emqx_ee_connector_influxdb_SUITE">>,
+        influxdb_config(Host, Port, false, "verify_none")
+    ).
+
+perform_lifecycle_check(PoolName, InitialConfig) ->
+    {ok, #{config := CheckedConfig}} =
+        emqx_resource:check_config(?INFLUXDB_RESOURCE_MOD, InitialConfig),
+    % We need to add a write_syntax to the config since the connector
+    % expects this
+    FullConfig = CheckedConfig#{write_syntax => influxdb_write_syntax()},
+    {ok, #{
+        state := #{client := #{pool := ReturnedPoolName}} = State,
+        status := InitialStatus
+    }} = emqx_resource:create_local(
+        PoolName,
+        ?CONNECTOR_RESOURCE_GROUP,
+        ?INFLUXDB_RESOURCE_MOD,
+        FullConfig,
+        #{}
+    ),
+    ?assertEqual(InitialStatus, connected),
+    % Instance should match the state and status of the just started resource
+    {ok, ?CONNECTOR_RESOURCE_GROUP, #{
+        state := State,
+        status := InitialStatus
+    }} =
+        emqx_resource:get_instance(PoolName),
+    ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)),
+    % % Perform query as further check that the resource is working as expected
+    ?assertMatch(ok, emqx_resource:query(PoolName, test_query())),
+    ?assertEqual(ok, emqx_resource:stop(PoolName)),
+    % Resource will be listed still, but state will be changed and healthcheck will fail
+    % as the worker no longer exists.
+    {ok, ?CONNECTOR_RESOURCE_GROUP, #{
+        state := State,
+        status := StoppedStatus
+    }} =
+        emqx_resource:get_instance(PoolName),
+    ?assertEqual(stopped, StoppedStatus),
+    ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)),
+    % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself.
+    ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
+    % Can call stop/1 again on an already stopped instance
+    ?assertEqual(ok, emqx_resource:stop(PoolName)),
+    % Make sure it can be restarted and the healthchecks and queries work properly
+    ?assertEqual(ok, emqx_resource:restart(PoolName)),
+    % async restart, need to wait resource
+    timer:sleep(500),
+    {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} =
+        emqx_resource:get_instance(PoolName),
+    ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)),
+    ?assertMatch(ok, emqx_resource:query(PoolName, test_query())),
+    % Stop and remove the resource in one go.
+    ?assertEqual(ok, emqx_resource:remove_local(PoolName)),
+    ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)),
+    % Should not even be able to get the resource data out of ets now unlike just stopping.
+    ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)).
+
+t_tls_opts(Config) ->
+    PoolName = <<"emqx_ee_connector_influxdb_SUITE">>,
+    Host = ?config(influxdb_tls_host, Config),
+    Port = ?config(influxdb_tls_port, Config),
+    VerifyNoneStatus = perform_tls_opts_check(
+        PoolName, influxdb_config(Host, Port, true, "verify_none")
+    ),
+    ?assertEqual(connected, VerifyNoneStatus),
+    VerifyPeerStatus = perform_tls_opts_check(
+        PoolName, influxdb_config(Host, Port, true, "verify_peer")
+    ),
+    ?assertEqual(disconnected, VerifyPeerStatus),
+    ok.
+
+perform_tls_opts_check(PoolName, InitialConfig) ->
+    {ok, #{config := CheckedConfig}} =
+        emqx_resource:check_config(?INFLUXDB_RESOURCE_MOD, InitialConfig),
+    % We need to add a write_syntax to the config since the connector
+    % expects this
+    FullConfig = CheckedConfig#{write_syntax => influxdb_write_syntax()},
+    {ok, #{
+        config := #{ssl := #{enable := SslEnabled}},
+        status := Status
+    }} = emqx_resource:create_local(
+        PoolName,
+        ?CONNECTOR_RESOURCE_GROUP,
+        ?INFLUXDB_RESOURCE_MOD,
+        FullConfig,
+        #{}
+    ),
+    ?assert(SslEnabled),
+    % Stop and remove the resource in one go.
+    ?assertEqual(ok, emqx_resource:remove_local(PoolName)),
+    Status.
+
+% %%------------------------------------------------------------------------------
+% %% Helpers
+% %%------------------------------------------------------------------------------
+
+influxdb_config(Host, Port, SslEnabled, Verify) ->
+    RawConfig = list_to_binary(
+        io_lib:format(
+            ""
+            "\n"
+            "  bucket = mqtt\n"
+            "  org = emqx\n"
+            "  token = abcdefg\n"
+            "  server = \"~s:~b\"\n"
+            "  ssl {\n"
+            "    enable = ~s\n"
+            "    verify = ~s\n"
+            "  }\n"
+            "  "
+            "",
+            [Host, Port, SslEnabled, Verify]
+        )
+    ),
+
+    {ok, ResourceConfig} = hocon:binary(RawConfig),
+    #{<<"config">> => ResourceConfig}.
+
+influxdb_write_syntax() ->
+    [
+        #{
+            measurement => "${topic}",
+            tags => [{"clientid", "${clientid}"}],
+            fields => [{"payload", "${payload}"}],
+            timestamp => undefined
+        }
+    ].
+
+test_query() ->
+    {send_message, #{
+        <<"clientid">> => <<"something">>,
+        <<"payload">> => #{bool => true},
+        <<"topic">> => <<"connector_test">>
+    }}.

From cfd0e9ebdd9dbf3fef3d9b0c4d8ced08008ea7ba Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Thu, 9 Feb 2023 17:08:36 +0100
Subject: [PATCH 003/144] refactor(emqx_ee_connector): use emqx_tls_lib for
 influx ssl opts

We used to simply pass on SSL options given to the influxdb EE
connector, but we now pass them to emqx_tls_lib instead. This ensures a
proper handling of SSL options and also allow us to use meck to inject
custom options in tests.
---
 lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index 988c19156..f056f4af2 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -341,7 +341,7 @@ ssl_config(SSL = #{enable := true}) ->
     [
         {https_enabled, true},
         {transport, ssl},
-        {transport_opts, maps:to_list(maps:remove(enable, SSL))}
+        {transport_opts, emqx_tls_lib:to_client_opts(SSL)}
     ].
 
 username(#{username := Username}) ->

From bc6c653c9fd83dbd446de42b9465fc8289820cc0 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Thu, 9 Feb 2023 17:12:01 +0100
Subject: [PATCH 004/144] test(emqx_ee_connector): rework and improve influxdb
 tests

---
 .../test/emqx_ee_connector_influxdb_SUITE.erl | 88 +++++++++++++------
 1 file changed, 62 insertions(+), 26 deletions(-)

diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
index 01bb8a08e..a2e6c7c8f 100644
--- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
+++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
@@ -117,40 +117,64 @@ perform_lifecycle_check(PoolName, InitialConfig) ->
     % Should not even be able to get the resource data out of ets now unlike just stopping.
     ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)).
 
-t_tls_opts(Config) ->
+t_tls_verify_none(Config) ->
     PoolName = <<"emqx_ee_connector_influxdb_SUITE">>,
     Host = ?config(influxdb_tls_host, Config),
     Port = ?config(influxdb_tls_port, Config),
-    VerifyNoneStatus = perform_tls_opts_check(
-        PoolName, influxdb_config(Host, Port, true, "verify_none")
-    ),
-    ?assertEqual(connected, VerifyNoneStatus),
-    VerifyPeerStatus = perform_tls_opts_check(
-        PoolName, influxdb_config(Host, Port, true, "verify_peer")
-    ),
-    ?assertEqual(disconnected, VerifyPeerStatus),
+    InitialConfig = influxdb_config(Host, Port, true, "verify_none"),
+    ValidStatus = perform_tls_opts_check(PoolName, InitialConfig, valid),
+    ?assertEqual(connected, ValidStatus),
+    InvalidStatus = perform_tls_opts_check(PoolName, InitialConfig, fail),
+    ?assertEqual(disconnected, InvalidStatus),
     ok.
 
-perform_tls_opts_check(PoolName, InitialConfig) ->
+t_tls_verify_peer(Config) ->
+    PoolName = <<"emqx_ee_connector_influxdb_SUITE">>,
+    Host = ?config(influxdb_tls_host, Config),
+    Port = ?config(influxdb_tls_port, Config),
+    InitialConfig = influxdb_config(Host, Port, true, "verify_peer"),
+    ValidStatus = perform_tls_opts_check(PoolName, InitialConfig, valid),
+    ?assertEqual(connected, ValidStatus),
+    InvalidStatus = perform_tls_opts_check(PoolName, InitialConfig, fail),
+    ?assertEqual(disconnected, InvalidStatus),
+    ok.
+
+perform_tls_opts_check(PoolName, InitialConfig, VerifyReturn) ->
     {ok, #{config := CheckedConfig}} =
         emqx_resource:check_config(?INFLUXDB_RESOURCE_MOD, InitialConfig),
-    % We need to add a write_syntax to the config since the connector
-    % expects this
-    FullConfig = CheckedConfig#{write_syntax => influxdb_write_syntax()},
-    {ok, #{
-        config := #{ssl := #{enable := SslEnabled}},
-        status := Status
-    }} = emqx_resource:create_local(
-        PoolName,
-        ?CONNECTOR_RESOURCE_GROUP,
-        ?INFLUXDB_RESOURCE_MOD,
-        FullConfig,
-        #{}
+    % Meck handling of TLS opt handling so that we can inject custom
+    % verification returns
+    meck:new(emqx_tls_lib, [passthrough, no_link]),
+    meck:expect(
+        emqx_tls_lib,
+        to_client_opts,
+        fun(Opts) ->
+            Verify = {verify_fun, {custom_verify(), {return, VerifyReturn}}},
+            [Verify | meck:passthrough([Opts])]
+        end
     ),
-    ?assert(SslEnabled),
-    % Stop and remove the resource in one go.
-    ?assertEqual(ok, emqx_resource:remove_local(PoolName)),
-    Status.
+    try
+        % We need to add a write_syntax to the config since the connector
+        % expects this
+        FullConfig = CheckedConfig#{write_syntax => influxdb_write_syntax()},
+        {ok, #{
+            config := #{ssl := #{enable := SslEnabled}},
+            status := Status
+        }} = emqx_resource:create_local(
+            PoolName,
+            ?CONNECTOR_RESOURCE_GROUP,
+            ?INFLUXDB_RESOURCE_MOD,
+            FullConfig,
+            #{}
+        ),
+        ?assert(SslEnabled),
+        ?assert(meck:validate(emqx_tls_lib)),
+        % Stop and remove the resource in one go.
+        ?assertEqual(ok, emqx_resource:remove_local(PoolName)),
+        Status
+    after
+        meck:unload(emqx_tls_lib)
+    end.
 
 % %%------------------------------------------------------------------------------
 % %% Helpers
@@ -178,6 +202,18 @@ influxdb_config(Host, Port, SslEnabled, Verify) ->
     {ok, ResourceConfig} = hocon:binary(RawConfig),
     #{<<"config">> => ResourceConfig}.
 
+custom_verify() ->
+    fun
+        (_, {bad_cert, unknown_ca} = Event, {return, Return} = UserState) ->
+            ct:pal("Call to custom verify fun. Event: ~p UserState: ~p", [Event, UserState]),
+            {Return, UserState};
+        (_, Event, UserState) ->
+            ct:pal("Unexpected call to custom verify fun. Event: ~p UserState: ~p", [
+                Event, UserState
+            ]),
+            {fail, unexpected_call_to_verify_fun}
+    end.
+
 influxdb_write_syntax() ->
     [
         #{

From 4eb4430fe22e7f024ae8fadc5047993dab856271 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Thu, 9 Feb 2023 17:22:19 +0100
Subject: [PATCH 005/144] chore: bump VSN

---
 lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index 56d128601..1d9927d3f 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -1,6 +1,6 @@
 {application, emqx_ee_connector, [
     {description, "EMQX Enterprise connectors"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {applications, [
         kernel,

From 28560e460366682a51a977bf2d35987e916dd49c Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 13 Feb 2023 11:23:43 +0100
Subject: [PATCH 006/144] refactor: remove unused code

---
 apps/emqx_management/src/emqx_mgmt_api.erl | 78 ++++++----------------
 1 file changed, 19 insertions(+), 59 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index e46047521..110a69e60 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -23,8 +23,7 @@
 -define(LONG_QUERY_TIMEOUT, 50000).
 
 -export([
-    paginate/3,
-    paginate/4
+    paginate/3
 ]).
 
 %% first_next query APIs
@@ -58,14 +57,14 @@
 
 -export([do_query/2, apply_total_query/1]).
 
-paginate(Tables, Params, {Module, FormatFun}) ->
-    Qh = query_handle(Tables),
-    Count = count(Tables),
-    do_paginate(Qh, Count, Params, {Module, FormatFun}).
-
-paginate(Tables, MatchSpec, Params, {Module, FormatFun}) ->
-    Qh = query_handle(Tables, MatchSpec),
-    Count = count(Tables, MatchSpec),
+-spec paginate(atom(), map(), {atom(), fun()}) ->
+    #{
+        meta => #{page => pos_integer(), limit => pos_integer(), count => pos_integer()},
+        data => list(term())
+    }.
+paginate(Table, Params, {Module, FormatFun}) ->
+    Qh = query_handle(Table),
+    Count = count(Table),
     do_paginate(Qh, Count, Params, {Module, FormatFun}).
 
 do_paginate(Qh, Count, Params, {Module, FormatFun}) ->
@@ -86,57 +85,17 @@ do_paginate(Qh, Count, Params, {Module, FormatFun}) ->
         data => [erlang:apply(Module, FormatFun, [Row]) || Row <- Rows]
     }.
 
-query_handle(Table) when is_atom(Table) ->
-    qlc:q([R || R <- ets:table(Table)]);
-query_handle({Table, Opts}) when is_atom(Table) ->
-    qlc:q([R || R <- ets:table(Table, Opts)]);
-query_handle([Table]) when is_atom(Table) ->
-    qlc:q([R || R <- ets:table(Table)]);
-query_handle([{Table, Opts}]) when is_atom(Table) ->
-    qlc:q([R || R <- ets:table(Table, Opts)]);
-query_handle(Tables) ->
-    %
-    qlc:append([query_handle(T) || T <- Tables]).
+query_handle(Table) ->
+    qlc:q([R || R <- ets:table(Table)]).
 
-query_handle(Table, MatchSpec) when is_atom(Table) ->
-    Options = {traverse, {select, MatchSpec}},
-    qlc:q([R || R <- ets:table(Table, Options)]);
-query_handle([Table], MatchSpec) when is_atom(Table) ->
-    Options = {traverse, {select, MatchSpec}},
-    qlc:q([R || R <- ets:table(Table, Options)]);
-query_handle(Tables, MatchSpec) ->
-    Options = {traverse, {select, MatchSpec}},
-    qlc:append([qlc:q([E || E <- ets:table(T, Options)]) || T <- Tables]).
+count(Table) ->
+    ets:info(Table, size).
 
-count(Table) when is_atom(Table) ->
-    ets:info(Table, size);
-count({Table, _}) when is_atom(Table) ->
-    ets:info(Table, size);
-count([Table]) when is_atom(Table) ->
-    ets:info(Table, size);
-count([{Table, _}]) when is_atom(Table) ->
-    ets:info(Table, size);
-count(Tables) ->
-    lists:sum([count(T) || T <- Tables]).
-
-count(Table, MatchSpec) when is_atom(Table) ->
-    [{MatchPattern, Where, _Re}] = MatchSpec,
-    NMatchSpec = [{MatchPattern, Where, [true]}],
-    ets:select_count(Table, NMatchSpec);
-count([Table], MatchSpec) when is_atom(Table) ->
-    count(Table, MatchSpec);
-count(Tables, MatchSpec) ->
-    lists:sum([count(T, MatchSpec) || T <- Tables]).
-
-page(Params) when is_map(Params) ->
-    maps:get(<<"page">>, Params, 1);
 page(Params) ->
-    proplists:get_value(<<"page">>, Params, <<"1">>).
+    maps:get(<<"page">>, Params, 1).
 
-limit(Params) when is_map(Params) ->
-    maps:get(<<"limit">>, Params, emqx_mgmt:max_row_limit());
 limit(Params) ->
-    proplists:get_value(<<"limit">>, Params, emqx_mgmt:max_row_limit()).
+    maps:get(<<"limit">>, Params, emqx_mgmt:max_row_limit()).
 
 %%--------------------------------------------------------------------
 %% Node Query
@@ -605,7 +564,7 @@ to_type(V, TargetType) ->
 to_type_(V, atom) -> to_atom(V);
 to_type_(V, integer) -> to_integer(V);
 to_type_(V, timestamp) -> to_timestamp(V);
-to_type_(V, ip) -> aton(V);
+to_type_(V, ip) -> to_ip(V);
 to_type_(V, ip_port) -> to_ip_port(V);
 to_type_(V, _) -> V.
 
@@ -624,8 +583,9 @@ to_timestamp(I) when is_integer(I) ->
 to_timestamp(B) when is_binary(B) ->
     binary_to_integer(B).
 
-aton(B) when is_binary(B) ->
-    list_to_tuple([binary_to_integer(T) || T <- re:split(B, "[.]")]).
+to_ip(IP0) when is_binary(IP0) ->
+    {ok, IP} = inet:parse_address(binary_to_list(IP0)),
+    IP.
 
 to_ip_port(IPAddress) ->
     [IP0, Port0] = string:tokens(binary_to_list(IPAddress), ":"),

From 743ced17c99db29b9df061eb57dc1788eb0f8b20 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Fri, 10 Feb 2023 14:40:45 +0100
Subject: [PATCH 007/144] test: add test for more types in schema

---
 apps/emqx_management/src/emqx_mgmt_api.erl | 31 ++++++++++++++++------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 110a69e60..9b2e6981e 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -605,40 +605,55 @@ b2i(Any) ->
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 
-params2qs_test() ->
+params2qs_test_() ->
     QSchema = [
         {<<"str">>, binary},
         {<<"int">>, integer},
+        {<<"binatom">>, atom},
         {<<"atom">>, atom},
         {<<"ts">>, timestamp},
         {<<"gte_range">>, integer},
         {<<"lte_range">>, integer},
         {<<"like_fuzzy">>, binary},
-        {<<"match_topic">>, binary}
+        {<<"match_topic">>, binary},
+        {<<"ip">>, ip},
+        {<<"ip_port">>, ip_port}
     ],
     QString = [
         {<<"str">>, <<"abc">>},
         {<<"int">>, <<"123">>},
-        {<<"atom">>, <<"connected">>},
+        {<<"binatom">>, <<"connected">>},
+        {<<"atom">>, ok},
         {<<"ts">>, <<"156000">>},
         {<<"gte_range">>, <<"1">>},
         {<<"lte_range">>, <<"5">>},
         {<<"like_fuzzy">>, <<"user">>},
-        {<<"match_topic">>, <<"t/#">>}
+        {<<"match_topic">>, <<"t/#">>},
+        {<<"ip">>, <<"127.0.0.1">>},
+        {<<"ip_port">>, <<"127.0.0.1:8888">>}
     ],
     ExpectedQs = [
         {str, '=:=', <<"abc">>},
         {int, '=:=', 123},
-        {atom, '=:=', connected},
+        {binatom, '=:=', connected},
+        {atom, '=:=', ok},
         {ts, '=:=', 156000},
-        {range, '>=', 1, '=<', 5}
+        {range, '>=', 1, '=<', 5},
+        {ip, '=:=', {127, 0, 0, 1}},
+        {ip_port, '=:=', {{127, 0, 0, 1}, 8888}}
     ],
     FuzzyNQString = [
         {fuzzy, like, <<"user">>},
         {topic, match, <<"t/#">>}
     ],
-    ?assertEqual({7, {ExpectedQs, FuzzyNQString}}, parse_qstring(QString, QSchema)),
 
-    {0, {[], []}} = parse_qstring([{not_a_predefined_params, val}], QSchema).
+    [
+        ?_assertEqual({10, {ExpectedQs, FuzzyNQString}}, parse_qstring(QString, QSchema)),
+        ?_assertEqual({0, {[], []}}, parse_qstring([{not_a_predefined_params, val}], QSchema)),
+        ?_assertThrow(
+            {bad_value_type, {<<"ip">>, ip, <<"helloworld">>}},
+            parse_qstring([{<<"ip">>, <<"helloworld">>}], QSchema)
+        )
+    ].
 
 -endif.

From 0e9289e48cf656fcc3840725c326235188911a63 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 13 Feb 2023 15:40:31 +0100
Subject: [PATCH 008/144] refactor: remove unused clause

---
 apps/emqx_management/src/emqx_mgmt_api.erl | 2 --
 1 file changed, 2 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 9b2e6981e..f3fb87bd2 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -169,8 +169,6 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) ->
     end.
 
 %% @private
-do_cluster_query([], QueryState, ResultAcc) ->
-    finalize_query(ResultAcc, mark_complete(QueryState));
 do_cluster_query(
     [Node | Tail] = Nodes,
     QueryState,

From 9e954585ae8768bade0cc2197258c949a4708013 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 13 Feb 2023 15:40:53 +0100
Subject: [PATCH 009/144] test: add node query tests and some more

---
 .../test/emqx_mgmt_api_SUITE.erl              | 74 ++++++++++++++++++-
 1 file changed, 70 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
index a8bbfa6d9..8e95ad116 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
@@ -67,7 +67,7 @@ t_cluster_query(_Config) ->
 
         %% assert: AllPage = Page1 + Page2 + Page3 + Page4
         %% !!!Note: this equation requires that the queried tables must be ordered_set
-        {200, ClientsPage2} = query_clients(Node1, #{<<"page">> => 2, <<"limit">> => 5}),
+        {200, ClientsPage2} = query_clients(Node1, #{<<"page">> => <<"2">>, <<"limit">> => 5}),
         {200, ClientsPage3} = query_clients(Node2, #{<<"page">> => 3, <<"limit">> => 5}),
         {200, ClientsPage4} = query_clients(Node1, #{<<"page">> => 4, <<"limit">> => 5}),
         GetClientIds = fun(L) -> lists:map(fun(#{clientid := Id}) -> Id end, L) end,
@@ -79,6 +79,72 @@ t_cluster_query(_Config) ->
             )
         ),
 
+        %% Scroll past count
+        {200, ClientsPage10} = query_clients(Node1, #{<<"page">> => <<"10">>, <<"limit">> => 5}),
+        ?assertEqual(
+            #{data => [], meta => #{page => 10, limit => 5, count => 20, hasnext => false}},
+            ClientsPage10
+        ),
+
+        %% Node queries
+        {200, ClientsNode2} = query_clients(Node1, #{<<"node">> => Node2}),
+        ?assertEqual({200, ClientsNode2}, query_clients(Node2, #{<<"node">> => Node2})),
+        ?assertMatch(
+            #{page := 1, limit := 100, count := 10},
+            maps:get(meta, ClientsNode2)
+        ),
+        ?assertMatch(10, length(maps:get(data, ClientsNode2))),
+
+        {200, ClientsNode2Page1} = query_clients(Node2, #{<<"node">> => Node2, <<"limit">> => 5}),
+        {200, ClientsNode2Page2} = query_clients(Node1, #{
+            <<"node">> => Node2, <<"page">> => <<"2">>, <<"limit">> => 5
+        }),
+        {200, ClientsNode2Page3} = query_clients(Node2, #{
+            <<"node">> => Node2, <<"page">> => 3, <<"limit">> => 5
+        }),
+        {200, ClientsNode2Page4} = query_clients(Node1, #{
+            <<"node">> => Node2, <<"page">> => 4, <<"limit">> => 5
+        }),
+        ?assertEqual(
+            GetClientIds(maps:get(data, ClientsNode2)),
+            GetClientIds(
+                lists:append([
+                    maps:get(data, Page)
+                 || Page <- [
+                        ClientsNode2Page1,
+                        ClientsNode2Page2,
+                        ClientsNode2Page3,
+                        ClientsNode2Page4
+                    ]
+                ])
+            )
+        ),
+
+        %% Scroll past count
+        {200, ClientsNode2Page10} = query_clients(Node1, #{
+            <<"node">> => Node2, <<"page">> => <<"10">>, <<"limit">> => 5
+        }),
+        ?assertEqual(
+            #{data => [], meta => #{page => 10, limit => 5, count => 10, hasnext => false}},
+            ClientsNode2Page10
+        ),
+
+        %% Query with bad params
+        ?assertEqual(
+            {400, #{
+                code => <<"INVALID_PARAMETER">>,
+                message => <<"page_limit_invalid">>
+            }},
+            query_clients(Node1, #{<<"page">> => -1})
+        ),
+        ?assertEqual(
+            {400, #{
+                code => <<"INVALID_PARAMETER">>,
+                message => <<"page_limit_invalid">>
+            }},
+            query_clients(Node1, #{<<"node">> => Node1, <<"page">> => -1})
+        ),
+
         %% exact match can return non-zero total
         {200, ClientsNode1} = query_clients(Node2, #{<<"username">> => <<"corenode1@127.0.0.1">>}),
         ?assertMatch(
@@ -87,11 +153,11 @@ t_cluster_query(_Config) ->
         ),
 
         %% fuzzy searching can't return total
-        {200, ClientsNode2} = query_clients(Node2, #{<<"like_username">> => <<"corenode2">>}),
-        MetaNode2 = maps:get(meta, ClientsNode2),
+        {200, ClientsFuzzyNode2} = query_clients(Node2, #{<<"like_username">> => <<"corenode2">>}),
+        MetaNode2 = maps:get(meta, ClientsFuzzyNode2),
         ?assertNotMatch(#{count := _}, MetaNode2),
         ?assertMatch(#{hasnext := false}, MetaNode2),
-        ?assertMatch(10, length(maps:get(data, ClientsNode2))),
+        ?assertMatch(10, length(maps:get(data, ClientsFuzzyNode2))),
 
         _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1),
         _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs2)

From 0c87d238e9e09203f1715c54d1b733b9d78362b7 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 13 Feb 2023 15:46:27 +0100
Subject: [PATCH 010/144] chore: bump version

---
 apps/emqx_management/src/emqx_management.app.src | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index fdd2af9b2..8a10f1b6b 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.0.14"},
+    {vsn, "5.0.15"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},

From 89630a8a74b3f7b455e27c1163ca01b7311bc76d Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 13 Feb 2023 16:09:04 +0100
Subject: [PATCH 011/144] test: query bad node

---
 apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
index 8e95ad116..5cda176ab 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
@@ -145,6 +145,12 @@ t_cluster_query(_Config) ->
             query_clients(Node1, #{<<"node">> => Node1, <<"page">> => -1})
         ),
 
+        %% Query bad node
+        ?assertMatch(
+            {500, #{code := <<"NODE_DOWN">>}},
+            query_clients(Node1, #{<<"node">> => 'nonode@nohost'})
+        ),
+
         %% exact match can return non-zero total
         {200, ClientsNode1} = query_clients(Node2, #{<<"username">> => <<"corenode1@127.0.0.1">>}),
         ?assertMatch(

From a28588ebf064758e793f452b04d231c2a6879154 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 13 Feb 2023 17:14:02 +0100
Subject: [PATCH 012/144] fix: bad spec

---
 apps/emqx_management/src/emqx_mgmt_api.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index f3fb87bd2..fa0bc6e7a 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -57,7 +57,7 @@
 
 -export([do_query/2, apply_total_query/1]).
 
--spec paginate(atom(), map(), {atom(), fun()}) ->
+-spec paginate(atom(), map(), {atom(), atom()}) ->
     #{
         meta => #{page => pos_integer(), limit => pos_integer(), count => pos_integer()},
         data => list(term())

From 8d34a72ab640610a917849ca21e6e46113b111bf Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 13 Feb 2023 17:14:15 +0100
Subject: [PATCH 013/144] test: add test for clustered badrpc

---
 apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
index 5cda176ab..b051982d7 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
@@ -173,6 +173,17 @@ t_cluster_query(_Config) ->
     end,
     ok.
 
+t_bad_rpc(_) ->
+    emqx_mgmt_api_test_util:init_suite(),
+    meck:expect(mria_mnesia, running_nodes, 0, ['fake@nohost']),
+    try
+        Path = emqx_mgmt_api_test_util:api_path(["clients"]),
+        {error, {_, 500, _}} = emqx_mgmt_api_test_util:request_api(get, Path)
+    after
+        meck:unload(mria_mnesia),
+        emqx_mgmt_api_test_util:end_suite()
+    end.
+
 %%--------------------------------------------------------------------
 %% helpers
 %%--------------------------------------------------------------------

From 8b47acd58d4267a72809424bbf381602dc2d33df Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 13 Feb 2023 17:46:26 +0100
Subject: [PATCH 014/144] test: page overflow and bad rpc in cluster

---
 .../test/emqx_mgmt_api_SUITE.erl              | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
index b051982d7..2223df449 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
@@ -173,13 +173,30 @@ t_cluster_query(_Config) ->
     end,
     ok.
 
+t_paging(_) ->
+    emqx_mgmt_api_test_util:init_suite(),
+    try
+        Path0 = emqx_mgmt_api_test_util:api_path(["banned?page=1"]),
+        {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path0)
+        Path1 = emqx_mgmt_api_test_util:api_path(["banned?page=10"]),
+        {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path1)
+    after
+        emqx_mgmt_api_test_util:end_suite()
+    end.
+
 t_bad_rpc(_) ->
     emqx_mgmt_api_test_util:init_suite(),
-    meck:expect(mria_mnesia, running_nodes, 0, ['fake@nohost']),
+    process_flag(trap_exit, true),
+    ClientLs1 = [start_emqtt_client(node(), I, 1883) || I <- lists:seq(1, 10)],
+    Path = emqx_mgmt_api_test_util:api_path(["clients?limit=2&page=2"]),
     try
-        Path = emqx_mgmt_api_test_util:api_path(["clients"]),
+        meck:expect(mria_mnesia, running_nodes, 0, ['fake@nohost']),
+        {error, {_, 500, _}} = emqx_mgmt_api_test_util:request_api(get, Path),
+        %% good cop, bad cop
+        meck:expect(mria_mnesia, running_nodes, 0, [node(), 'fake@nohost']),
         {error, {_, 500, _}} = emqx_mgmt_api_test_util:request_api(get, Path)
     after
+        _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1),
         meck:unload(mria_mnesia),
         emqx_mgmt_api_test_util:end_suite()
     end.

From ab63954ce7d78c6c3ea7bcbf388c8bd58f286b32 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 13 Feb 2023 14:56:54 +0100
Subject: [PATCH 015/144] refactor: check if node is already started before
 default-cookie warning

---
 bin/emqx | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/bin/emqx b/bin/emqx
index 0f6a18437..2bea2716a 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -913,11 +913,14 @@ if [ -z "$COOKIE" ]; then
     COOKIE="$(get_boot_config 'node.cookie')"
 fi
 [ -z "$COOKIE" ] && COOKIE="$EMQX_DEFAULT_ERLANG_COOKIE"
-if [ $IS_BOOT_COMMAND = 'yes' ] && [ "$COOKIE" = "$EMQX_DEFAULT_ERLANG_COOKIE" ]; then
-    logwarn "Default (insecure) Erlang cookie is in use."
-    logwarn "Configure node.cookie in $EMQX_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE"
-    logwarn "NOTE: Use the same cookie for all nodes in the cluster."
-fi
+
+maybe_warn_default_cookie() {
+    if [ $IS_BOOT_COMMAND = 'yes' ] && [ "$COOKIE" = "$EMQX_DEFAULT_ERLANG_COOKIE" ]; then
+        logwarn "Default (insecure) Erlang cookie is in use."
+        logwarn "Configure node.cookie in $EMQX_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE"
+        logwarn "NOTE: Use the same cookie for all nodes in the cluster."
+    fi
+}
 
 ## check if OTP version has mnesia_hook feature; if not, fallback to
 ## using Mnesia DB backend.
@@ -937,6 +940,7 @@ case "${COMMAND}" in
         if relx_nodetool "ping" >/dev/null 2>&1; then
             die "Node $NAME is already running!"
         fi
+        maybe_warn_default_cookie
 
         # this flag passes down to console mode
         # so we know it's intended to be run in daemon mode
@@ -1107,7 +1111,12 @@ case "${COMMAND}" in
         if [ "${_EMQX_START_DAEMON_MODE:-}" = 1 ]; then
             tr_log_to_env
         else
+            # Make sure a node IS not running
+            if relx_nodetool "ping" >/dev/null 2>&1; then
+                die "Node $NAME is already running!"
+            fi
             maybe_log_to_console
+            maybe_warn_default_cookie
         fi
 
         #generate app.config and vm.args

From 12199e7987b0f01c5b3f434eebfa58c6b41e10ee Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Mon, 13 Feb 2023 22:57:49 +0100
Subject: [PATCH 016/144] refactor(bin/emqx): check if node is already running
 with ps command

---
 bin/emqx | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/bin/emqx b/bin/emqx
index 2bea2716a..6f953b777 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -453,9 +453,25 @@ if [ "$IS_ENTERPRISE" = 'yes' ]; then
     CONF_KEYS+=( 'license.key' )
 fi
 
+
+## Find the running node from 'ps -ef'
+## The primary grep pattern is $RUNNER_ROOT_DIR because one can start multiple nodes at the same time
+# shellcheck disable=SC2009
+PS_LINE="$(ps -ef | $GREP '[e]mqx' | $GREP -v -E '(remsh|nodetool)' | $GREP -oE "\-[r]oot ${RUNNER_ROOT_DIR}.*" || true)"
+if [ -n "${PS_LINE}" ]; then
+    RUNNING_NODES_COUNT="$(echo -e "$PS_LINE" | wc -l)"
+else
+    RUNNING_NODES_COUNT=0
+fi
+
 # Turn off debug as the ps output can be quite noisy
 set +x
 if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
+    if [ "$RUNNING_NODES_COUNT" -gt 0 ] && [ "$COMMAND" != 'check_config' ]; then
+        tmp_nodename=$(echo -e "$PS_LINE" | $GREP -oE "\s\-s?name.*" | awk '{print $2}' || true)
+        echo "Node ${tmp_nodename} is already running!"
+        exit 1
+    fi
     [ -f "$EMQX_ETC_DIR"/emqx.conf ] || die "emqx.conf is not found in $EMQX_ETC_DIR" 1
     maybe_use_portable_dynlibs
     if [ "${EMQX_BOOT_CONFIGS:-}" = '' ]; then
@@ -486,14 +502,9 @@ else
     #    then update the config in the file to 'node.name = "emqx@local.net"', after this change,
     #    there would be no way stop the running node 'emqx@127.0.0.1', because 'emqx stop' command
     #    would try to stop the new node instead.
-    #  * The primary grep pattern is $RUNNER_ROOT_DIR because one can start multiple nodes at the same time
     #  * The grep args like '[e]mqx' but not 'emqx' is to avoid greping the grep command itself
     #  * The running 'remsh' and 'nodetool' processes must be excluded
-    # shellcheck disable=SC2009
-    PS_LINE="$(ps -ef | $GREP '[e]mqx' | $GREP -v -E '(remsh|nodetool)' | $GREP -oE "\-[r]oot ${RUNNER_ROOT_DIR}.*" || true)"
-    [ "$DEBUG" -eq 1 ] && echo "EMQX processes: $PS_LINE"
-    running_nodes_count="$(echo -e "$PS_LINE" | wc -l)"
-    if [ "$running_nodes_count" -eq 1 ]; then
+    if [ "$RUNNING_NODES_COUNT" -eq 1 ]; then
         ## only one emqx node is running, get running args from 'ps -ef' output
         tmp_nodename=$(echo -e "$PS_LINE" | $GREP -oE "\s\-s?name.*" | awk '{print $2}' || true)
         tmp_cookie=$(echo -e "$PS_LINE" | $GREP -oE "\s\-setcookie.*" | awk '{print $2}' || true)
@@ -936,10 +947,6 @@ cd "$RUNNER_ROOT_DIR"
 
 case "${COMMAND}" in
     start)
-        # Make sure a node IS not running
-        if relx_nodetool "ping" >/dev/null 2>&1; then
-            die "Node $NAME is already running!"
-        fi
         maybe_warn_default_cookie
 
         # this flag passes down to console mode
@@ -1111,10 +1118,6 @@ case "${COMMAND}" in
         if [ "${_EMQX_START_DAEMON_MODE:-}" = 1 ]; then
             tr_log_to_env
         else
-            # Make sure a node IS not running
-            if relx_nodetool "ping" >/dev/null 2>&1; then
-                die "Node $NAME is already running!"
-            fi
             maybe_log_to_console
             maybe_warn_default_cookie
         fi

From 36b1c3eb5be81a94ae83bb65fff5d6cdcab3a1b0 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Mon, 13 Feb 2023 11:58:13 +0800
Subject: [PATCH 017/144] fix(api): fix bad return error message if client id
 is not found

---
 .../src/emqx_mgmt_api_clients.erl             | 54 ++++++++++---------
 .../test/emqx_mgmt_api_clients_SUITE.erl      | 43 +++++++++++++++
 2 files changed, 71 insertions(+), 26 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index 7c45206fd..571f190f2 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -76,9 +76,10 @@
 
 -define(FORMAT_FUN, {?MODULE, format_channel_info}).
 
--define(CLIENT_ID_NOT_FOUND,
-    <<"{\"code\": \"RESOURCE_NOT_FOUND\", \"reason\": \"Client id not found\"}">>
-).
+-define(CLIENTID_NOT_FOUND, #{
+    code => 'CLIENTID_NOT_FOUND',
+    message => <<"Client ID not found">>
+}).
 
 api_spec() ->
     emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).
@@ -219,7 +220,7 @@ schema("/clients/:clientid") ->
             responses => #{
                 200 => hoconsc:mk(hoconsc:ref(?MODULE, client), #{}),
                 404 => emqx_dashboard_swagger:error_codes(
-                    ['CLIENTID_NOT_FOUND'], <<"Client id not found">>
+                    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
                 )
             }
         },
@@ -232,7 +233,7 @@ schema("/clients/:clientid") ->
             responses => #{
                 204 => <<"Kick out client successfully">>,
                 404 => emqx_dashboard_swagger:error_codes(
-                    ['CLIENTID_NOT_FOUND'], <<"Client id not found">>
+                    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
                 )
             }
         }
@@ -247,7 +248,7 @@ schema("/clients/:clientid/authorization/cache") ->
             responses => #{
                 200 => hoconsc:mk(hoconsc:ref(?MODULE, authz_cache), #{}),
                 404 => emqx_dashboard_swagger:error_codes(
-                    ['CLIENTID_NOT_FOUND'], <<"Client id not found">>
+                    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
                 )
             }
         },
@@ -256,9 +257,9 @@ schema("/clients/:clientid/authorization/cache") ->
             tags => ?TAGS,
             parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}],
             responses => #{
-                204 => <<"Kick out client successfully">>,
+                204 => <<"Clean client authz cache successfully">>,
                 404 => emqx_dashboard_swagger:error_codes(
-                    ['CLIENTID_NOT_FOUND'], <<"Client id not found">>
+                    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
                 )
             }
         }
@@ -273,10 +274,11 @@ schema("/clients/:clientid/subscriptions") ->
             responses => #{
                 200 => hoconsc:mk(
                     hoconsc:array(hoconsc:ref(emqx_mgmt_api_subscriptions, subscription)), #{}
-                ),
-                404 => emqx_dashboard_swagger:error_codes(
-                    ['CLIENTID_NOT_FOUND'], <<"Client id not found">>
                 )
+                %% returns [] if client not existed in cluster
+                %404 => emqx_dashboard_swagger:error_codes(
+                %    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
+                %)
             }
         }
     };
@@ -291,7 +293,7 @@ schema("/clients/:clientid/subscribe") ->
             responses => #{
                 200 => hoconsc:ref(emqx_mgmt_api_subscriptions, subscription),
                 404 => emqx_dashboard_swagger:error_codes(
-                    ['CLIENTID_NOT_FOUND'], <<"Client id not found">>
+                    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
                 )
             }
         }
@@ -307,7 +309,7 @@ schema("/clients/:clientid/subscribe/bulk") ->
             responses => #{
                 200 => hoconsc:array(hoconsc:ref(emqx_mgmt_api_subscriptions, subscription)),
                 404 => emqx_dashboard_swagger:error_codes(
-                    ['CLIENTID_NOT_FOUND'], <<"Client id not found">>
+                    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
                 )
             }
         }
@@ -323,7 +325,7 @@ schema("/clients/:clientid/unsubscribe") ->
             responses => #{
                 204 => <<"Unsubscribe OK">>,
                 404 => emqx_dashboard_swagger:error_codes(
-                    ['CLIENTID_NOT_FOUND'], <<"Client id not found">>
+                    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
                 )
             }
         }
@@ -339,7 +341,7 @@ schema("/clients/:clientid/unsubscribe/bulk") ->
             responses => #{
                 204 => <<"Unsubscribe OK">>,
                 404 => emqx_dashboard_swagger:error_codes(
-                    ['CLIENTID_NOT_FOUND'], <<"Client id not found">>
+                    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
                 )
             }
         }
@@ -355,7 +357,7 @@ schema("/clients/:clientid/keepalive") ->
             responses => #{
                 200 => hoconsc:mk(hoconsc:ref(?MODULE, client), #{}),
                 404 => emqx_dashboard_swagger:error_codes(
-                    ['CLIENTID_NOT_FOUND'], <<"Client id not found">>
+                    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
                 )
             }
         }
@@ -621,7 +623,7 @@ set_keepalive(put, #{bindings := #{clientid := ClientID}, body := Body}) ->
         {ok, Interval} ->
             case emqx_mgmt:set_keepalive(emqx_mgmt_util:urldecode(ClientID), Interval) of
                 ok -> lookup(#{clientid => ClientID});
-                {error, not_found} -> {404, ?CLIENT_ID_NOT_FOUND};
+                {error, not_found} -> {404, ?CLIENTID_NOT_FOUND};
                 {error, Reason} -> {400, #{code => 'PARAMS_ERROR', message => Reason}}
             end
     end.
@@ -669,7 +671,7 @@ list_clients(QString) ->
 lookup(#{clientid := ClientID}) ->
     case emqx_mgmt:lookup_client({clientid, ClientID}, ?FORMAT_FUN) of
         [] ->
-            {404, ?CLIENT_ID_NOT_FOUND};
+            {404, ?CLIENTID_NOT_FOUND};
         ClientInfo ->
             {200, hd(ClientInfo)}
     end.
@@ -677,7 +679,7 @@ lookup(#{clientid := ClientID}) ->
 kickout(#{clientid := ClientID}) ->
     case emqx_mgmt:kickout_client({ClientID, ?FORMAT_FUN}) of
         {error, not_found} ->
-            {404, ?CLIENT_ID_NOT_FOUND};
+            {404, ?CLIENTID_NOT_FOUND};
         _ ->
             {204}
     end.
@@ -685,7 +687,7 @@ kickout(#{clientid := ClientID}) ->
 get_authz_cache(#{clientid := ClientID}) ->
     case emqx_mgmt:list_authz_cache(ClientID) of
         {error, not_found} ->
-            {404, ?CLIENT_ID_NOT_FOUND};
+            {404, ?CLIENTID_NOT_FOUND};
         {error, Reason} ->
             Message = list_to_binary(io_lib:format("~p", [Reason])),
             {500, #{code => <<"UNKNOW_ERROR">>, message => Message}};
@@ -699,7 +701,7 @@ clean_authz_cache(#{clientid := ClientID}) ->
         ok ->
             {204};
         {error, not_found} ->
-            {404, ?CLIENT_ID_NOT_FOUND};
+            {404, ?CLIENTID_NOT_FOUND};
         {error, Reason} ->
             Message = list_to_binary(io_lib:format("~p", [Reason])),
             {500, #{code => <<"UNKNOW_ERROR">>, message => Message}}
@@ -709,7 +711,7 @@ subscribe(#{clientid := ClientID, topic := Topic} = Sub) ->
     Opts = maps:with([qos, nl, rap, rh], Sub),
     case do_subscribe(ClientID, Topic, Opts) of
         {error, channel_not_found} ->
-            {404, ?CLIENT_ID_NOT_FOUND};
+            {404, ?CLIENTID_NOT_FOUND};
         {error, Reason} ->
             Message = list_to_binary(io_lib:format("~p", [Reason])),
             {500, #{code => <<"UNKNOW_ERROR">>, message => Message}};
@@ -723,7 +725,7 @@ subscribe_batch(#{clientid := ClientID, topics := Topics}) ->
     %% has returned. So if one want to subscribe topics in this hook, it will fail.
     case ets:lookup(emqx_channel, ClientID) of
         [] ->
-            {404, ?CLIENT_ID_NOT_FOUND};
+            {404, ?CLIENTID_NOT_FOUND};
         _ ->
             ArgList = [
                 [ClientID, Topic, maps:with([qos, nl, rap, rh], Sub)]
@@ -735,7 +737,7 @@ subscribe_batch(#{clientid := ClientID, topics := Topics}) ->
 unsubscribe(#{clientid := ClientID, topic := Topic}) ->
     case do_unsubscribe(ClientID, Topic) of
         {error, channel_not_found} ->
-            {404, ?CLIENT_ID_NOT_FOUND};
+            {404, ?CLIENTID_NOT_FOUND};
         {unsubscribe, [{Topic, #{}}]} ->
             {204}
     end.
@@ -745,8 +747,8 @@ unsubscribe_batch(#{clientid := ClientID, topics := Topics}) ->
         {200, _} ->
             _ = emqx_mgmt:unsubscribe_batch(ClientID, Topics),
             {204};
-        {404, ?CLIENT_ID_NOT_FOUND} ->
-            {404, ?CLIENT_ID_NOT_FOUND}
+        {404, NotFound} ->
+            {404, NotFound}
     end.
 
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
index 16ba99ad6..1a74d3af6 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
@@ -247,6 +247,49 @@ t_keepalive(_Config) ->
     emqtt:disconnect(C1),
     ok.
 
+t_client_id_not_found(_Config) ->
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    Http = {"HTTP/1.1", 404, "Not Found"},
+    Body = "{\"code\":\"CLIENTID_NOT_FOUND\",\"message\":\"Client ID not found\"}",
+
+    PathFun = fun(Suffix) ->
+        emqx_mgmt_api_test_util:api_path(["clients", "no_existed_clientid"] ++ Suffix)
+    end,
+    ReqFun = fun(Method, Path) ->
+        emqx_mgmt_api_test_util:request_api(
+            Method, Path, "", AuthHeader, [], #{return_all => true}
+        )
+    end,
+
+    PostFun = fun(Method, Path, Data) ->
+        emqx_mgmt_api_test_util:request_api(
+            Method, Path, "", AuthHeader, Data, #{return_all => true}
+        )
+    end,
+
+    %% Client lookup
+    ?assertMatch({error, {Http, _, Body}}, ReqFun(get, PathFun([]))),
+    %% Client kickout
+    ?assertMatch({error, {Http, _, Body}}, ReqFun(delete, PathFun([]))),
+    %% Client Subscription list
+    ?assertMatch({ok, {{"HTTP/1.1", 200, "OK"}, _, "[]"}}, ReqFun(get, PathFun(["subscriptions"]))),
+    %% AuthZ Cache lookup
+    ?assertMatch({error, {Http, _, Body}}, ReqFun(get, PathFun(["authorization", "cache"]))),
+    %% AuthZ Cache clean
+    ?assertMatch({error, {Http, _, Body}}, ReqFun(delete, PathFun(["authorization", "cache"]))),
+    %% Client Subscribe
+    SubBody = #{topic => <<"testtopic">>, qos => 1, nl => 1, rh => 1},
+    ?assertMatch({error, {Http, _, Body}}, PostFun(post, PathFun(["subscribe"]), SubBody)),
+    ?assertMatch(
+        {error, {Http, _, Body}}, PostFun(post, PathFun(["subscribe", "bulk"]), [SubBody])
+    ),
+    %% Client Unsubscribe
+    UnsubBody = #{topic => <<"testtopic">>},
+    ?assertMatch({error, {Http, _, Body}}, PostFun(post, PathFun(["unsubscribe"]), UnsubBody)),
+    ?assertMatch(
+        {error, {Http, _, Body}}, PostFun(post, PathFun(["unsubscribe", "bulk"]), [UnsubBody])
+    ).
+
 time_string_to_epoch_millisecond(DateTime) ->
     time_string_to_epoch(DateTime, millisecond).
 

From a9b0885af0fcea6b45af6972a88c33bbd655875f Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Mon, 13 Feb 2023 13:54:25 +0800
Subject: [PATCH 018/144] chore: update changes

---
 changes/v5.0.17/fix-9958.en.md | 1 +
 changes/v5.0.17/fix-9958.zh.md | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changes/v5.0.17/fix-9958.en.md
 create mode 100644 changes/v5.0.17/fix-9958.zh.md

diff --git a/changes/v5.0.17/fix-9958.en.md b/changes/v5.0.17/fix-9958.en.md
new file mode 100644
index 000000000..821934ad0
--- /dev/null
+++ b/changes/v5.0.17/fix-9958.en.md
@@ -0,0 +1 @@
+Fix bad http response format when client ID is not found in `clients` APIs
diff --git a/changes/v5.0.17/fix-9958.zh.md b/changes/v5.0.17/fix-9958.zh.md
new file mode 100644
index 000000000..a26fbb7fe
--- /dev/null
+++ b/changes/v5.0.17/fix-9958.zh.md
@@ -0,0 +1 @@
+修复 `clients` API 在 Client ID 不存在时返回的错误的 HTTP 应答格式。

From 0d63cfdc9793f5946c647bcc110a91b812e1c280 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Tue, 14 Feb 2023 15:31:42 +0800
Subject: [PATCH 019/144] chore: remove changes to 5.0.18

---
 apps/emqx_management/src/emqx_management.app.src | 2 +-
 changes/{v5.0.17 => v5.0.18}/fix-9958.en.md      | 0
 changes/{v5.0.17 => v5.0.18}/fix-9958.zh.md      | 0
 3 files changed, 1 insertion(+), 1 deletion(-)
 rename changes/{v5.0.17 => v5.0.18}/fix-9958.en.md (100%)
 rename changes/{v5.0.17 => v5.0.18}/fix-9958.zh.md (100%)

diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index fdd2af9b2..8a10f1b6b 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.0.14"},
+    {vsn, "5.0.15"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},
diff --git a/changes/v5.0.17/fix-9958.en.md b/changes/v5.0.18/fix-9958.en.md
similarity index 100%
rename from changes/v5.0.17/fix-9958.en.md
rename to changes/v5.0.18/fix-9958.en.md
diff --git a/changes/v5.0.17/fix-9958.zh.md b/changes/v5.0.18/fix-9958.zh.md
similarity index 100%
rename from changes/v5.0.17/fix-9958.zh.md
rename to changes/v5.0.18/fix-9958.zh.md

From 4417ea9db7af54d17c35e58732ba91e4d90b3850 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 14 Feb 2023 09:13:45 +0100
Subject: [PATCH 020/144] style: missing comma

---
 apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
index 2223df449..da9a338f2 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
@@ -177,7 +177,7 @@ t_paging(_) ->
     emqx_mgmt_api_test_util:init_suite(),
     try
         Path0 = emqx_mgmt_api_test_util:api_path(["banned?page=1"]),
-        {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path0)
+        {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path0),
         Path1 = emqx_mgmt_api_test_util:api_path(["banned?page=10"]),
         {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path1)
     after

From 530b21268a86cb589a2c5dd531b1809b0de68e10 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Tue, 7 Feb 2023 19:36:03 +0800
Subject: [PATCH 021/144] feat(bridges): add TDengine

---
 .github/workflows/run_test_cases.yaml         |   1 +
 .../src/emqx_connector_mysql.erl              |  17 +-
 .../src/emqx_plugin_libs_rule.erl             |  17 +-
 .../i18n/emqx_ee_bridge_tdengine.conf         |  74 ++++++
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl  |  17 +-
 .../src/emqx_ee_bridge_tdengine.erl           | 123 +++++++++
 .../i18n/emqx_ee_connector_tdengine.conf      |  22 ++
 lib-ee/emqx_ee_connector/rebar.config         |   1 +
 .../src/emqx_ee_connector.app.src             |   1 +
 .../src/emqx_ee_connector_tdengine.erl        | 241 ++++++++++++++++++
 scripts/spellcheck/dicts/emqx.txt             |   1 +
 11 files changed, 495 insertions(+), 20 deletions(-)
 create mode 100644 lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf
 create mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl
 create mode 100644 lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf
 create mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 6b4357abc..0abca58c3 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -161,6 +161,7 @@ jobs:
             PGSQL_TAG: "13"
             REDIS_TAG: "7.0"
             INFLUXDB_TAG: "2.5.0"
+            TDENGINE_TAG: "3.0.2.4"
             PROFILE: ${{ matrix.profile }}
             CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
           run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }}
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index 066e053d4..e06d6a9d7 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -391,22 +391,7 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens})
     end.
 
 on_batch_insert(InstId, BatchReqs, InsertPart, Tokens, State) ->
-    JoinFun = fun
-        ([Msg]) ->
-            emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Msg);
-        ([H | T]) ->
-            lists:foldl(
-                fun(Msg, Acc) ->
-                    Value = emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Msg),
-                    <<Acc/binary, ", ", Value/binary>>
-                end,
-                emqx_plugin_libs_rule:proc_sql_param_str(Tokens, H),
-                T
-            )
-    end,
-    {_, Msgs} = lists:unzip(BatchReqs),
-    JoinPart = JoinFun(Msgs),
-    SQL = <<InsertPart/binary, " values ", JoinPart/binary>>,
+    SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, InsertPart, Tokens),
     on_sql_query(InstId, query, SQL, [], default_timeout, State).
 
 on_sql_query(
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
index 969374309..a60c94a7b 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl
@@ -31,7 +31,8 @@
     proc_sql_param_str/2,
     proc_cql_param_str/2,
     split_insert_sql/1,
-    detect_sql_type/1
+    detect_sql_type/1,
+    proc_batch_sql/3
 ]).
 
 %% type converting
@@ -164,6 +165,20 @@ detect_sql_type(SQL) ->
             {error, invalid_sql}
     end.
 
+-spec proc_batch_sql(
+    BatchReqs :: list({atom(), map()}),
+    InsertPart :: binary(),
+    Tokens :: tmpl_token()
+) -> InsertSQL :: binary().
+proc_batch_sql(BatchReqs, InsertPart, Tokens) ->
+    ValuesPart = erlang:iolist_to_binary(
+        lists:join(", ", [
+            emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Msg)
+         || {_, Msg} <- BatchReqs
+        ])
+    ),
+    <<InsertPart/binary, " values ", ValuesPart/binary>>.
+
 unsafe_atom_key(Key) when is_atom(Key) ->
     Key;
 unsafe_atom_key(Key) when is_binary(Key) ->
diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf
new file mode 100644
index 000000000..2d5af9f16
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf
@@ -0,0 +1,74 @@
+emqx_ee_bridge_tdengine {
+
+    local_topic {
+        desc {
+            en: """The MQTT topic filter to be forwarded to TDengine. All MQTT 'PUBLISH' messages with the topic
+matching the local_topic will be forwarded.</br>
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is
+configured, then both the data got from the rule and the MQTT messages that match local_topic
+will be forwarded.
+"""
+            zh: """发送到 'local_topic' 的消息都会转发到 TDengine。 </br>
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。
+"""
+        }
+        label {
+                en: "Local Topic"
+                zh: "本地 Topic"
+            }
+    }
+
+    sql_template {
+        desc {
+            en: """SQL Template"""
+            zh: """SQL 模板"""
+            }
+        label {
+            en: "SQL Template"
+            zh: "SQL 模板"
+        }
+    }
+    config_enable {
+        desc {
+            en: """Enable or disable this bridge"""
+            zh: """启用/禁用桥接"""
+        }
+        label {
+            en: "Enable Or Disable Bridge"
+            zh: "启用/禁用桥接"
+        }
+        }
+
+    desc_config {
+        desc {
+            en: """Configuration for an TDengine bridge."""
+            zh: """TDengine 桥接配置"""
+        }
+        label: {
+            en: "TDengine Bridge Configuration"
+            zh: "TDengine 桥接配置"
+        }
+    }
+
+    desc_type {
+        desc {
+            en: """The Bridge Type"""
+            zh: """Bridge 类型"""
+        }
+        label {
+            en: "Bridge Type"
+            zh: "桥接类型"
+        }
+    }
+
+    desc_name {
+        desc {
+            en: """Bridge name."""
+            zh: """桥接名字"""
+        }
+        label {
+            en: "Bridge Name"
+            zh: "桥接名字"
+        }
+    }
+}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
index 43a26111a..1a358fdfe 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl
@@ -28,7 +28,8 @@ api_schemas(Method) ->
         ref(emqx_ee_bridge_redis, Method ++ "_sentinel"),
         ref(emqx_ee_bridge_redis, Method ++ "_cluster"),
         ref(emqx_ee_bridge_timescale, Method),
-        ref(emqx_ee_bridge_matrix, Method)
+        ref(emqx_ee_bridge_matrix, Method),
+        ref(emqx_ee_bridge_tdengine, Method)
     ].
 
 schema_modules() ->
@@ -42,7 +43,8 @@ schema_modules() ->
         emqx_ee_bridge_redis,
         emqx_ee_bridge_pgsql,
         emqx_ee_bridge_timescale,
-        emqx_ee_bridge_matrix
+        emqx_ee_bridge_matrix,
+        emqx_ee_bridge_tdengine
     ].
 
 examples(Method) ->
@@ -72,7 +74,8 @@ resource_type(redis_sentinel) -> emqx_ee_connector_redis;
 resource_type(redis_cluster) -> emqx_ee_connector_redis;
 resource_type(pgsql) -> emqx_connector_pgsql;
 resource_type(timescale) -> emqx_connector_pgsql;
-resource_type(matrix) -> emqx_connector_pgsql.
+resource_type(matrix) -> emqx_connector_pgsql;
+resource_type(tdengine) -> emqx_ee_connector_tdengine.
 
 fields(bridges) ->
     [
@@ -107,6 +110,14 @@ fields(bridges) ->
                     desc => <<"MySQL Bridge Config">>,
                     required => false
                 }
+            )},
+        {tdengine,
+            mk(
+                hoconsc:map(name, ref(emqx_ee_bridge_tdengine, "config")),
+                #{
+                    desc => <<"TDengine Bridge Config">>,
+                    required => false
+                }
             )}
     ] ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ pgsql_structs().
 
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl
new file mode 100644
index 000000000..35e81efa3
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl
@@ -0,0 +1,123 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+-module(emqx_ee_bridge_tdengine).
+
+-include_lib("typerefl/include/types.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+-include_lib("emqx_bridge/include/emqx_bridge.hrl").
+-include_lib("emqx_resource/include/emqx_resource.hrl").
+
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-export([
+    conn_bridge_examples/1,
+    values/1
+]).
+
+-export([
+    namespace/0,
+    roots/0,
+    fields/1,
+    desc/1
+]).
+
+-define(DEFAULT_SQL, <<
+    "insert into mqtt.t_mqtt_msg(ts, msgid, mqtt_topic, qos, payload, arrived) "
+    "values (${ts}, ${id}, ${topic}, ${qos}, ${payload}, ${timestamp})"
+>>).
+
+%% -------------------------------------------------------------------------------------------------
+%% api
+
+conn_bridge_examples(Method) ->
+    [
+        #{
+            <<"tdengine">> => #{
+                summary => <<"TDengine Bridge">>,
+                value => values(Method)
+            }
+        }
+    ].
+
+values(get) ->
+    maps:merge(values(post), ?METRICS_EXAMPLE);
+values(post) ->
+    #{
+        enable => true,
+        type => tdengine,
+        name => <<"foo">>,
+        server => <<"127.0.0.1:6041">>,
+        database => <<"mqtt">>,
+        pool_size => 8,
+        username => <<"root">>,
+        password => <<"taosdata">>,
+        sql => ?DEFAULT_SQL,
+        local_topic => <<"local/topic/#">>,
+        resource_opts => #{
+            worker_pool_size => 8,
+            health_check_interval => ?HEALTHCHECK_INTERVAL_RAW,
+            auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW,
+            batch_size => ?DEFAULT_BATCH_SIZE,
+            batch_time => ?DEFAULT_BATCH_TIME,
+            query_mode => sync,
+            max_queue_bytes => ?DEFAULT_QUEUE_SIZE
+        }
+    };
+values(put) ->
+    values(post).
+
+%% -------------------------------------------------------------------------------------------------
+%% Hocon Schema Definitions
+namespace() -> "bridge_tdengine".
+
+roots() -> [].
+
+fields("config") ->
+    [
+        {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
+        {sql,
+            mk(
+                binary(),
+                #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>}
+            )},
+        {local_topic,
+            mk(
+                binary(),
+                #{desc => ?DESC("local_topic"), default => undefined}
+            )},
+        {resource_opts,
+            mk(
+                ref(?MODULE, "creation_opts"),
+                #{
+                    required => false,
+                    default => #{},
+                    desc => ?DESC(emqx_resource_schema, <<"resource_opts">>)
+                }
+            )}
+    ] ++ emqx_ee_connector_tdengine:fields(config);
+fields("creation_opts") ->
+    emqx_resource_schema:fields("creation_opts_sync_only");
+fields("post") ->
+    [type_field(), name_field() | fields("config")];
+fields("put") ->
+    fields("config");
+fields("get") ->
+    emqx_bridge_schema:status_fields() ++ fields("post").
+
+desc("config") ->
+    ?DESC("desc_config");
+desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
+    ["Configuration for TDengine using `", string:to_upper(Method), "` method."];
+desc("creation_opts" = Name) ->
+    emqx_resource_schema:desc(Name);
+desc(_) ->
+    undefined.
+
+%% -------------------------------------------------------------------------------------------------
+
+type_field() ->
+    {type, mk(enum([tdengine]), #{required => true, desc => ?DESC("desc_type")})}.
+
+name_field() ->
+    {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}.
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf
new file mode 100644
index 000000000..c6c58d82d
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf
@@ -0,0 +1,22 @@
+emqx_ee_connector_tdengine {
+
+    server {
+        desc {
+          en: """
+The IPv4 or IPv6 address or the hostname to connect to.<br/>
+A host entry has the following form: `Host[:Port]`.<br/>
+The TDengine default port 6041 is used if `[:Port]` is not specified.
+"""
+          zh: """
+将要连接的 IPv4 或 IPv6 地址，或者主机名。<br/>
+主机名具有以下形式：`Host[:Port]`。<br/>
+如果未指定 `[:Port]`，则使用 TDengine 默认端口 6041。
+"""
+        }
+        label: {
+              en: "Server Host"
+              zh: "服务器地址"
+            }
+    }
+
+}
diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config
index 00421e4f6..54c471f96 100644
--- a/lib-ee/emqx_ee_connector/rebar.config
+++ b/lib-ee/emqx_ee_connector/rebar.config
@@ -2,6 +2,7 @@
 {deps, [
   {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
   {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.8"}}},
+  {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.5"}}},
   {emqx, {path, "../../apps/emqx"}}
 ]}.
 
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index 56d128601..c45f5478a 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -7,6 +7,7 @@
         stdlib,
         hstreamdb_erl,
         influxdb,
+        tdengine,
         wolff,
         brod
     ]},
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl
new file mode 100644
index 000000000..7ab0c5078
--- /dev/null
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl
@@ -0,0 +1,241 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_connector_tdengine).
+
+-behaviour(emqx_resource).
+
+-include_lib("emqx_resource/include/emqx_resource.hrl").
+-include_lib("typerefl/include/types.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+
+-export([roots/0, fields/1]).
+
+%% `emqx_resource' API
+-export([
+    callback_mode/0,
+    is_buffer_supported/0,
+    on_start/2,
+    on_stop/2,
+    on_query/3,
+    on_batch_query/3,
+    on_get_status/2
+]).
+
+-export([connect/1, do_get_status/1, execute/3]).
+
+-import(hoconsc, [mk/2, enum/1, ref/2]).
+
+-define(TD_HOST_OPTIONS, #{
+    default_port => 6041
+}).
+
+%%=====================================================================
+%% Hocon schema
+roots() ->
+    [{config, #{type => hoconsc:ref(?MODULE, config)}}].
+
+fields(config) ->
+    [
+        {server, server()}
+        | add_default_username(emqx_connector_schema_lib:relational_db_fields())
+    ].
+
+add_default_username(Fields) ->
+    lists:map(
+        fun
+            ({username, OrigUsernameFn}) ->
+                {username, add_default_fn(OrigUsernameFn, <<"root">>)};
+            (Field) ->
+                Field
+        end,
+        Fields
+    ).
+
+add_default_fn(OrigFn, Default) ->
+    fun
+        (default) -> Default;
+        (Field) -> OrigFn(Field)
+    end.
+
+server() ->
+    Meta = #{desc => ?DESC("server")},
+    emqx_schema:servers_sc(Meta, ?TD_HOST_OPTIONS).
+
+%%========================================================================================
+%% `emqx_resource' API
+%%========================================================================================
+
+callback_mode() -> always_sync.
+
+is_buffer_supported() -> false.
+
+on_start(
+    InstanceId,
+    #{
+        server := Server,
+        username := Username,
+        password := Password,
+        pool_size := PoolSize
+    } = Config
+) ->
+    ?SLOG(info, #{
+        msg => "starting_tdengine_connector",
+        connector => InstanceId,
+        config => emqx_misc:redact(Config)
+    }),
+
+    {Host, Port} = emqx_schema:parse_server(Server, ?TD_HOST_OPTIONS),
+    Options = [
+        {host, to_bin(Host)},
+        {port, Port},
+        {username, Username},
+        {password, Password},
+        {pool_size, PoolSize},
+        {pool, binary_to_atom(InstanceId, utf8)}
+    ],
+
+    Prepares = parse_prepare_sql(Config),
+    State = maps:merge(Prepares, #{poolname => InstanceId, query_opts => query_opts(Config)}),
+    case emqx_plugin_libs_pool:start_pool(InstanceId, ?MODULE, Options) of
+        ok ->
+            {ok, State};
+        Error ->
+            Error
+    end.
+
+on_stop(InstanceId, #{poolname := PoolName} = _State) ->
+    ?SLOG(info, #{
+        msg => "stopping_tdengine_connector",
+        connector => InstanceId
+    }),
+    emqx_plugin_libs_pool:stop_pool(PoolName).
+
+on_query(InstanceId, {query, SQL}, State) ->
+    do_query(InstanceId, SQL, State);
+on_query(InstanceId, Request, State) ->
+    %% because the `emqx-tdengine` client only supports a single SQL cmd
+    %% so the `on_query` and `on_batch_query` have the same process, that is:
+    %% we need to collect all data into one SQL cmd and then call the insert API
+    on_batch_query(InstanceId, [Request], State).
+
+on_batch_query(
+    InstanceId,
+    BatchReq,
+    #{batch_inserts := Inserts, batch_params_tokens := ParamsTokens} = State
+) ->
+    case hd(BatchReq) of
+        {Key, _} ->
+            case maps:get(Key, Inserts, undefined) of
+                undefined ->
+                    {error, {unrecoverable_error, batch_prepare_not_implemented}};
+                InsertSQL ->
+                    Tokens = maps:get(Key, ParamsTokens),
+                    do_batch_insert(InstanceId, BatchReq, InsertSQL, Tokens, State)
+            end;
+        Request ->
+            LogMeta = #{connector => InstanceId, first_request => Request, state => State},
+            ?SLOG(error, LogMeta#{msg => "invalid request"}),
+            {error, {unrecoverable_error, invalid_request}}
+    end.
+
+on_get_status(_InstanceId, #{poolname := Pool}) ->
+    Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1),
+    status_result(Health).
+
+do_get_status(Conn) ->
+    case tdengine:insert(Conn, "select server_version()", []) of
+        {ok, _} -> true;
+        _ -> false
+    end.
+
+status_result(_Status = true) -> connected;
+status_result(_Status = false) -> connecting.
+
+%%========================================================================================
+%% Helper fns
+%%========================================================================================
+
+do_batch_insert(InstanceId, BatchReqs, InsertPart, Tokens, State) ->
+    SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, InsertPart, Tokens),
+    do_query(InstanceId, SQL, State).
+
+do_query(InstanceId, Query, #{poolname := PoolName, query_opts := Opts} = State) ->
+    ?TRACE(
+        "QUERY",
+        "tdengine_connector_received",
+        #{connector => InstanceId, query => Query, state => State}
+    ),
+    Result = ecpool:pick_and_do(PoolName, {?MODULE, execute, [Query, Opts]}, no_handover),
+
+    case Result of
+        {error, Reason} ->
+            ?tp(
+                tdengine_connector_query_return,
+                #{error => Reason}
+            ),
+            ?SLOG(error, #{
+                msg => "tdengine_connector_do_query_failed",
+                connector => InstanceId,
+                query => Query,
+                reason => Reason
+            }),
+            Result;
+        _ ->
+            ?tp(
+                tdengine_connector_query_return,
+                #{result => Result}
+            ),
+            Result
+    end.
+
+execute(Conn, Query, Opts) ->
+    tdengine:insert(Conn, Query, Opts).
+
+connect(Opts) ->
+    tdengine:start_link(Opts).
+
+query_opts(#{database := Database} = _Opts) ->
+    [{db_name, Database}].
+
+parse_prepare_sql(Config) ->
+    SQL =
+        case maps:get(sql, Config, undefined) of
+            undefined -> #{};
+            Template -> #{send_message => Template}
+        end,
+
+    parse_batch_prepare_sql(maps:to_list(SQL), #{}, #{}).
+
+parse_batch_prepare_sql([{Key, H} | T], BatchInserts, BatchTks) ->
+    case emqx_plugin_libs_rule:detect_sql_type(H) of
+        {ok, select} ->
+            parse_batch_prepare_sql(T, BatchInserts, BatchTks);
+        {ok, insert} ->
+            case emqx_plugin_libs_rule:split_insert_sql(H) of
+                {ok, {InsertSQL, Params}} ->
+                    ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(Params),
+                    parse_batch_prepare_sql(
+                        T,
+                        BatchInserts#{Key => InsertSQL},
+                        BatchTks#{Key => ParamsTks}
+                    );
+                {error, Reason} ->
+                    ?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}),
+                    parse_batch_prepare_sql(T, BatchInserts, BatchTks)
+            end;
+        {error, Reason} ->
+            ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}),
+            parse_batch_prepare_sql(T, BatchInserts, BatchTks)
+    end;
+parse_batch_prepare_sql([], BatchInserts, BatchTks) ->
+    #{
+        batch_inserts => BatchInserts,
+        batch_params_tokens => BatchTks
+    }.
+
+to_bin(List) when is_list(List) ->
+    unicode:characters_to_binary(List, utf8).
diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt
index 065cf1a3a..388cfed16 100644
--- a/scripts/spellcheck/dicts/emqx.txt
+++ b/scripts/spellcheck/dicts/emqx.txt
@@ -265,3 +265,4 @@ GSSAPI
 keytab
 jq
 nif
+TDengine

From 0420e9acb54f3fe66233c6e4357fb4bb9fc489e8 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Wed, 8 Feb 2023 17:44:49 +0800
Subject: [PATCH 022/144] test(bridges): add test cases for TDEngine

---
 .ci/docker-compose-file/.env                  |   1 +
 .ci/docker-compose-file/Makefile.local        |   3 +
 .../docker-compose-tdengine-restful.yaml      |  11 +
 .../docker-compose-toxiproxy.yaml             |   1 +
 .ci/docker-compose-file/toxiproxy.json        |   6 +
 lib-ee/emqx_ee_bridge/docker-ct               |   1 +
 .../test/emqx_ee_bridge_tdengine_SUITE.erl    | 432 ++++++++++++++++++
 scripts/ct/run.sh                             |   3 +
 8 files changed, 458 insertions(+)
 create mode 100644 .ci/docker-compose-file/docker-compose-tdengine-restful.yaml
 create mode 100644 lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl

diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env
index a71e174d9..e3beb1bc7 100644
--- a/.ci/docker-compose-file/.env
+++ b/.ci/docker-compose-file/.env
@@ -4,5 +4,6 @@ MONGO_TAG=5
 PGSQL_TAG=13
 LDAP_TAG=2.4.50
 INFLUXDB_TAG=2.5.0
+TDENGINE_TAG=3.0.2.4
 
 TARGET=emqx/emqx
diff --git a/.ci/docker-compose-file/Makefile.local b/.ci/docker-compose-file/Makefile.local
index 9c12255e4..2ce1bf170 100644
--- a/.ci/docker-compose-file/Makefile.local
+++ b/.ci/docker-compose-file/Makefile.local
@@ -16,6 +16,7 @@ up:
 		REDIS_TAG=7.0 \
 		MONGO_TAG=5 \
 		PGSQL_TAG=13 \
+        TDENGINE_TAG=3.0.2.4 \
 	docker-compose \
 		-f .ci/docker-compose-file/docker-compose.yaml \
 		-f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
@@ -31,6 +32,7 @@ up:
 		-f .ci/docker-compose-file/docker-compose-redis-cluster-tcp.yaml \
 		-f .ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml \
 		-f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
+		-f .ci/docker-compose-file/docker-compose-tdengine-restful.yaml \
 		up -d --build --remove-orphans
 
 down:
@@ -49,6 +51,7 @@ down:
 		-f .ci/docker-compose-file/docker-compose-redis-cluster-tcp.yaml \
 		-f .ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml \
 		-f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
+		-f .ci/docker-compose-file/docker-compose-tdengine-restful.yaml \
 		down --remove-orphans
 
 ct:
diff --git a/.ci/docker-compose-file/docker-compose-tdengine-restful.yaml b/.ci/docker-compose-file/docker-compose-tdengine-restful.yaml
new file mode 100644
index 000000000..6cd7f2669
--- /dev/null
+++ b/.ci/docker-compose-file/docker-compose-tdengine-restful.yaml
@@ -0,0 +1,11 @@
+version: '3.9'
+
+services:
+  tdengine_server:
+    container_name: tdengine
+    image: tdengine/tdengine:${TDENGINE_TAG}
+    restart: always
+    ports:
+      - "6041:6041"
+    networks:
+      - emqx_bridge
diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
index ce4f28ba7..3f526978e 100644
--- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
+++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml
@@ -17,6 +17,7 @@ services:
       - 13307:3307
       - 15432:5432
       - 15433:5433
+      - 16041:6041
     command:
       - "-host=0.0.0.0"
       - "-config=/config/toxiproxy.json"
diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json
index f4b11116b..e26134ec8 100644
--- a/.ci/docker-compose-file/toxiproxy.json
+++ b/.ci/docker-compose-file/toxiproxy.json
@@ -41,5 +41,11 @@
     "listen": "0.0.0.0:5433",
     "upstream": "pgsql-tls:5432",
     "enabled": true
+  },
+  {
+    "name": "tdengine_restful",
+    "listen": "0.0.0.0:6041",
+    "upstream": "tdengine:6041",
+    "enabled": true
   }
 ]
diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct
index bf990bd7c..967faa343 100644
--- a/lib-ee/emqx_ee_bridge/docker-ct
+++ b/lib-ee/emqx_ee_bridge/docker-ct
@@ -7,3 +7,4 @@ mysql
 redis
 redis_cluster
 pgsql
+tdengine
diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl
new file mode 100644
index 000000000..4c17ba1a1
--- /dev/null
+++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl
@@ -0,0 +1,432 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ee_bridge_tdengine_SUITE).
+
+-compile(nowarn_export_all).
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+% SQL definitions
+-define(SQL_BRIDGE,
+    "insert into mqtt.t_mqtt_msg(ts, payload) values (${timestamp}, ${payload})"
+).
+
+-define(SQL_CREATE_DATABASE, "CREATE DATABASE IF NOT EXISTS mqtt; USE mqtt;").
+-define(SQL_CREATE_TABLE,
+    "CREATE TABLE t_mqtt_msg (\n"
+    "  ts timestamp,\n"
+    "  payload BINARY(1024)\n"
+    ");"
+).
+-define(SQL_DROP_TABLE, "DROP TABLE t_mqtt_msg").
+-define(SQL_DELETE, "DELETE from t_mqtt_msg").
+-define(SQL_SELECT, "SELECT payload FROM t_mqtt_msg").
+
+% DB defaults
+-define(TD_DATABASE, "mqtt").
+-define(TD_USERNAME, "root").
+-define(TD_PASSWORD, "taosdata").
+-define(BATCH_SIZE, 10).
+-define(PAYLOAD, <<"HELLO">>).
+
+-define(WITH_CON(Process),
+    Con = connect_direct_tdengine(Config),
+    Process,
+    ok = tdengine:stop(Con)
+).
+
+%%------------------------------------------------------------------------------
+%% CT boilerplate
+%%------------------------------------------------------------------------------
+
+all() ->
+    [
+        {group, with_batch},
+        {group, without_batch}
+    ].
+
+groups() ->
+    TCs = emqx_common_test_helpers:all(?MODULE),
+    NonBatchCases = [t_write_timeout],
+    [
+        {with_batch, TCs -- NonBatchCases},
+        {without_batch, TCs}
+    ].
+
+init_per_group(with_batch, Config0) ->
+    Config = [{enable_batch, true} | Config0],
+    common_init(Config);
+init_per_group(without_batch, Config0) ->
+    Config = [{enable_batch, false} | Config0],
+    common_init(Config);
+init_per_group(_Group, Config) ->
+    Config.
+
+end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch ->
+    connect_and_drop_table(Config),
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+    ok;
+end_per_group(_Group, _Config) ->
+    ok.
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_mgmt_api_test_util:end_suite(),
+    ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]),
+    ok.
+
+init_per_testcase(_Testcase, Config) ->
+    connect_and_clear_table(Config),
+    delete_bridge(Config),
+    Config.
+
+end_per_testcase(_Testcase, Config) ->
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+    connect_and_clear_table(Config),
+    ok = snabbkaffe:stop(),
+    delete_bridge(Config),
+    ok.
+
+%%------------------------------------------------------------------------------
+%% Helper fns
+%%------------------------------------------------------------------------------
+
+common_init(ConfigT) ->
+    Host = os:getenv("TDENGINE_HOST", "toxiproxy"),
+    Port = list_to_integer(os:getenv("TDENGINE_PORT", "6041")),
+
+    Config0 = [
+        {td_host, Host},
+        {td_port, Port},
+        {query_mode, sync},
+        {proxy_name, "tdengine_restful"}
+        | ConfigT
+    ],
+
+    BridgeType = proplists:get_value(bridge_type, Config0, <<"tdengine">>),
+    case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of
+        true ->
+            % Setup toxiproxy
+            ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"),
+            ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")),
+            emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort),
+            % Ensure EE bridge module is loaded
+            _ = application:load(emqx_ee_bridge),
+            _ = emqx_ee_bridge:module_info(),
+            ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]),
+            emqx_mgmt_api_test_util:init_suite(),
+            % Connect to tdengine directly and create the table
+            connect_and_create_table(Config0),
+            {Name, TDConf} = tdengine_config(BridgeType, Config0),
+            Config =
+                [
+                    {tdengine_config, TDConf},
+                    {tdengine_bridge_type, BridgeType},
+                    {tdengine_name, Name},
+                    {proxy_host, ProxyHost},
+                    {proxy_port, ProxyPort}
+                    | Config0
+                ],
+            Config;
+        false ->
+            case os:getenv("IS_CI") of
+                "yes" ->
+                    throw(no_tdengine);
+                _ ->
+                    {skip, no_tdengine}
+            end
+    end.
+
+tdengine_config(BridgeType, Config) ->
+    Port = integer_to_list(?config(td_port, Config)),
+    Server = ?config(td_host, Config) ++ ":" ++ Port,
+    Name = atom_to_binary(?MODULE),
+    BatchSize =
+        case ?config(enable_batch, Config) of
+            true -> ?BATCH_SIZE;
+            false -> 1
+        end,
+    QueryMode = ?config(query_mode, Config),
+    ConfigString =
+        io_lib:format(
+            "bridges.~s.~s {\n"
+            "  enable = true\n"
+            "  server = ~p\n"
+            "  database = ~p\n"
+            "  username = ~p\n"
+            "  password = ~p\n"
+            "  sql = ~p\n"
+            "  resource_opts = {\n"
+            "    request_timeout = 500ms\n"
+            "    batch_size = ~b\n"
+            "    query_mode = ~s\n"
+            "  }\n"
+            "}",
+            [
+                BridgeType,
+                Name,
+                Server,
+                ?TD_DATABASE,
+                ?TD_USERNAME,
+                ?TD_PASSWORD,
+                ?SQL_BRIDGE,
+                BatchSize,
+                QueryMode
+            ]
+        ),
+    {Name, parse_and_check(ConfigString, BridgeType, Name)}.
+
+parse_and_check(ConfigString, BridgeType, Name) ->
+    {ok, RawConf} = hocon:binary(ConfigString, #{format => map}),
+    hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}),
+    #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf,
+    Config.
+
+create_bridge(Config) ->
+    BridgeType = ?config(tdengine_bridge_type, Config),
+    Name = ?config(tdengine_name, Config),
+    TDConfig = ?config(tdengine_config, Config),
+    emqx_bridge:create(BridgeType, Name, TDConfig).
+
+delete_bridge(Config) ->
+    BridgeType = ?config(tdengine_bridge_type, Config),
+    Name = ?config(tdengine_name, Config),
+    emqx_bridge:remove(BridgeType, Name).
+
+create_bridge_http(Params) ->
+    Path = emqx_mgmt_api_test_util:api_path(["bridges"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of
+        {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])};
+        Error -> Error
+    end.
+
+send_message(Config, Payload) ->
+    Name = ?config(tdengine_name, Config),
+    BridgeType = ?config(tdengine_bridge_type, Config),
+    BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name),
+    emqx_bridge:send_message(BridgeID, Payload).
+
+query_resource(Config, Request) ->
+    Name = ?config(tdengine_name, Config),
+    BridgeType = ?config(tdengine_bridge_type, Config),
+    ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
+    emqx_resource:query(ResourceID, Request, #{timeout => 1_000}).
+
+connect_direct_tdengine(Config) ->
+    Opts = [
+        {host, to_bin(?config(td_host, Config))},
+        {port, ?config(td_port, Config)},
+        {username, to_bin(?TD_USERNAME)},
+        {password, to_bin(?TD_PASSWORD)},
+        {pool_size, 8}
+    ],
+
+    {ok, Con} = tdengine:start_link(Opts),
+    Con.
+
+% These funs connect and then stop the tdengine connection
+connect_and_create_table(Config) ->
+    ?WITH_CON(begin
+        {ok, _} = directly_query(Con, ?SQL_CREATE_DATABASE, []),
+        {ok, _} = directly_query(Con, ?SQL_CREATE_TABLE)
+    end).
+
+connect_and_drop_table(Config) ->
+    ?WITH_CON({ok, _} = directly_query(Con, ?SQL_DROP_TABLE)).
+
+connect_and_clear_table(Config) ->
+    ?WITH_CON({ok, _} = directly_query(Con, ?SQL_DELETE)).
+
+connect_and_get_payload(Config) ->
+    ?WITH_CON(
+        {ok, #{<<"code">> := 0, <<"data">> := [[Result]]}} = directly_query(Con, ?SQL_SELECT)
+    ),
+    Result.
+
+directly_query(Con, Query) ->
+    directly_query(Con, Query, [{db_name, ?TD_DATABASE}]).
+
+directly_query(Con, Query, QueryOpts) ->
+    tdengine:insert(Con, Query, QueryOpts).
+
+%%------------------------------------------------------------------------------
+%% Testcases
+%%------------------------------------------------------------------------------
+
+t_setup_via_config_and_publish(Config) ->
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000},
+    ?check_trace(
+        begin
+            ?wait_async_action(
+                ?assertMatch(
+                    {ok, #{<<"code">> := 0, <<"rows">> := 1}}, send_message(Config, SentData)
+                ),
+                #{?snk_kind := tdengine_connector_query_return},
+                10_000
+            ),
+            ?assertMatch(
+                ?PAYLOAD,
+                connect_and_get_payload(Config)
+            ),
+            ok
+        end,
+        fun(Trace0) ->
+            Trace = ?of_kind(tdengine_connector_query_return, Trace0),
+            ?assertMatch([#{result := {ok, #{<<"code">> := 0, <<"rows">> := 1}}}], Trace),
+            ok
+        end
+    ),
+    ok.
+
+t_setup_via_http_api_and_publish(Config) ->
+    BridgeType = ?config(tdengine_bridge_type, Config),
+    Name = ?config(tdengine_name, Config),
+    PgsqlConfig0 = ?config(tdengine_config, Config),
+    PgsqlConfig = PgsqlConfig0#{
+        <<"name">> => Name,
+        <<"type">> => BridgeType
+    },
+    ?assertMatch(
+        {ok, _},
+        create_bridge_http(PgsqlConfig)
+    ),
+    SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000},
+    ?check_trace(
+        begin
+            ?wait_async_action(
+                ?assertMatch(
+                    {ok, #{<<"code">> := 0, <<"rows">> := 1}}, send_message(Config, SentData)
+                ),
+                #{?snk_kind := tdengine_connector_query_return},
+                10_000
+            ),
+            ?assertMatch(
+                ?PAYLOAD,
+                connect_and_get_payload(Config)
+            ),
+            ok
+        end,
+        fun(Trace0) ->
+            Trace = ?of_kind(tdengine_connector_query_return, Trace0),
+            ?assertMatch([#{result := {ok, #{<<"code">> := 0, <<"rows">> := 1}}}], Trace),
+            ok
+        end
+    ),
+    ok.
+
+t_get_status(Config) ->
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    ProxyPort = ?config(proxy_port, Config),
+    ProxyHost = ?config(proxy_host, Config),
+    ProxyName = ?config(proxy_name, Config),
+
+    Name = ?config(tdengine_name, Config),
+    BridgeType = ?config(tdengine_bridge_type, Config),
+    ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
+
+    ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)),
+    emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
+        ?assertMatch(
+            {ok, Status} when Status =:= disconnected orelse Status =:= connecting,
+            emqx_resource_manager:health_check(ResourceID)
+        )
+    end),
+    ok.
+
+t_write_failure(Config) ->
+    ProxyName = ?config(proxy_name, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    ProxyHost = ?config(proxy_host, Config),
+    {ok, _} = create_bridge(Config),
+    SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000},
+    emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() ->
+        ?assertMatch({error, econnrefused}, send_message(Config, SentData))
+    end),
+    ok.
+
+% This test doesn't work with batch enabled since it is not possible
+% to set the timeout directly for batch queries
+t_write_timeout(Config) ->
+    ProxyName = ?config(proxy_name, Config),
+    ProxyPort = ?config(proxy_port, Config),
+    ProxyHost = ?config(proxy_host, Config),
+    {ok, _} = create_bridge(Config),
+    SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000},
+    emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() ->
+        ?assertMatch(
+            {error, {resource_error, #{reason := timeout}}},
+            query_resource(Config, {send_message, SentData})
+        )
+    end),
+    ok.
+
+t_simple_sql_query(Config) ->
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    Request = {query, <<"SELECT count(1) AS T">>},
+    Result = query_resource(Config, Request),
+    case ?config(enable_batch, Config) of
+        true ->
+            ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result);
+        false ->
+            ?assertMatch({ok, #{<<"code">> := 0, <<"data">> := [[1]]}}, Result)
+    end,
+    ok.
+
+t_missing_data(Config) ->
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    Result = send_message(Config, #{}),
+    ?assertMatch(
+        {error, #{
+            <<"code">> := 534,
+            <<"desc">> := _
+        }},
+        Result
+    ),
+    ok.
+
+t_bad_sql_parameter(Config) ->
+    ?assertMatch(
+        {ok, _},
+        create_bridge(Config)
+    ),
+    Request = {sql, <<"">>, [bad_parameter]},
+    Result = query_resource(Config, Request),
+    case ?config(enable_batch, Config) of
+        true ->
+            ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result);
+        false ->
+            ?assertMatch(
+                {error, {unrecoverable_error, _}}, Result
+            )
+    end,
+    ok.
+
+to_bin(List) when is_list(List) ->
+    unicode:characters_to_binary(List, utf8);
+to_bin(Bin) when is_binary(Bin) ->
+    Bin.
diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 0641707e8..372f5ca11 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -153,6 +153,9 @@ for dep in ${CT_DEPS}; do
             NEED_ROOT=yes
             FILES+=( '.ci/docker-compose-file/docker-compose-kafka.yaml' )
             ;;
+        tdengine)
+            FILES+=( '.ci/docker-compose-file/docker-compose-tdengine-restful.yaml' )
+            ;; 
         *)
             echo "unknown_ct_dependency $dep"
             exit 1

From 81adb0240284bb678b7d15fb7a3e626a4b4a9646 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Fri, 10 Feb 2023 17:09:29 +0800
Subject: [PATCH 023/144] chore: bump apps version

---
 apps/emqx_connector/src/emqx_connector.app.src         | 2 +-
 apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src     | 2 +-
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src       | 2 +-
 lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src
index aedc17c33..dfcf52902 100644
--- a/apps/emqx_connector/src/emqx_connector.app.src
+++ b/apps/emqx_connector/src/emqx_connector.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "EMQX Data Integration Connectors"},
-    {vsn, "0.1.14"},
+    {vsn, "0.1.15"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [
diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
index 3120b8503..7acf7433b 100644
--- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
+++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_plugin_libs, [
     {description, "EMQX Plugin utility libs"},
-    {vsn, "4.3.5"},
+    {vsn, "4.3.6"},
     {modules, []},
     {applications, [kernel, stdlib]},
     {env, []}
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
index b2a3c80c6..c30c927f2 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
@@ -1,6 +1,6 @@
 {application, emqx_ee_bridge, [
     {description, "EMQX Enterprise data bridges"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {applications, [
         kernel,
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
index c45f5478a..5017abd21 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src
@@ -1,6 +1,6 @@
 {application, emqx_ee_connector, [
     {description, "EMQX Enterprise connectors"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {applications, [
         kernel,

From 572e50e0a241054f95dd45fe35b9794e054bbd36 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 13 Feb 2023 17:55:03 +0100
Subject: [PATCH 024/144] perf: add TLS config hibernate_after, reduce memory
 footprint

---
 apps/emqx/i18n/emqx_schema_i18n.conf    | 11 ++++++++
 apps/emqx/src/emqx_schema.erl           | 10 +++++++
 apps/emqx/test/emqx_listeners_SUITE.erl | 35 +++++++++++++++++++++++++
 3 files changed, 56 insertions(+)

diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf
index 2ba5e7f52..6faa0c511 100644
--- a/apps/emqx/i18n/emqx_schema_i18n.conf
+++ b/apps/emqx/i18n/emqx_schema_i18n.conf
@@ -1495,6 +1495,17 @@ In case PSK cipher suites are intended, make sure to configure
     }
 }
 
+common_ssl_opts_schema_hibernate_after {
+    desc {
+        en: """ Hibernate the SSL process after idling for amount of time reducing its memory footprint. """
+        zh: """ 在闲置一定时间后休眠 SSL 进程，减少其内存占用。"""
+    }
+    label: {
+        en: "hibernate after"
+        zh: "闲置多久后休眠"
+    }
+}
+
 ciphers_schema_common {
     desc {
         en: """This config holds TLS cipher suite names separated by comma,
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index ba813d2c8..a75c3a968 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -1929,6 +1929,16 @@ common_ssl_opts_schema(Defaults) ->
                     default => Df("secure_renegotiate", true),
                     desc => ?DESC(common_ssl_opts_schema_secure_renegotiate)
                 }
+            )},
+
+        {"hibernate_after",
+            sc(
+                duration(),
+                #{
+                    default => undefined,
+                    required => false,
+                    desc => ?DESC(common_ssl_opts_schema_hibernate_after)
+                }
             )}
     ].
 
diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl
index 6a7cd2791..8c69e7fda 100644
--- a/apps/emqx/test/emqx_listeners_SUITE.erl
+++ b/apps/emqx/test/emqx_listeners_SUITE.erl
@@ -138,6 +138,41 @@ t_restart_listeners(_) ->
     ok = emqx_listeners:restart(),
     ok = emqx_listeners:stop().
 
+t_restart_listeners_with_hibernate_after_disabled(Config) ->
+    OldLConf = emqx_config:get([listeners]),
+    maps:foreach(
+        fun(LType, Listeners) ->
+            maps:foreach(
+                fun(Name, Opts) ->
+                    case maps:is_key(ssl_options, Opts) of
+                        true ->
+                            emqx_config:put(
+                                [
+                                    listeners,
+                                    LType,
+                                    Name,
+                                    ssl_options,
+                                    hibernate_after
+                                ],
+                                5000
+                            );
+                        _ ->
+                            skip
+                    end
+                end,
+                Listeners
+            )
+        end,
+        OldLConf
+    ),
+    ok = emqx_listeners:start(),
+    ok = emqx_listeners:stop(),
+    %% flakyness: eaddrinuse
+    timer:sleep(timer:seconds(2)),
+    ok = emqx_listeners:restart(),
+    ok = emqx_listeners:stop(),
+    emqx_config:put([listeners], OldLConf).
+
 t_max_conns_tcp(_) ->
     %% Note: Using a string representation for the bind address like
     %% "127.0.0.1" does not work

From d424d77e8490402899750722e6a4d745ce5d28ac Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 13 Feb 2023 18:04:03 +0100
Subject: [PATCH 025/144] chore(TLS):  hibernate_after default to 5s, follow
 version 4.x

---
 apps/emqx/src/emqx_schema.erl           | 3 +--
 apps/emqx/test/emqx_listeners_SUITE.erl | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index a75c3a968..d1be888c3 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -1935,8 +1935,7 @@ common_ssl_opts_schema(Defaults) ->
             sc(
                 duration(),
                 #{
-                    default => undefined,
-                    required => false,
+                    default => Df("hibernate_after", "5s"),
                     desc => ?DESC(common_ssl_opts_schema_hibernate_after)
                 }
             )}
diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl
index 8c69e7fda..015439587 100644
--- a/apps/emqx/test/emqx_listeners_SUITE.erl
+++ b/apps/emqx/test/emqx_listeners_SUITE.erl
@@ -138,7 +138,7 @@ t_restart_listeners(_) ->
     ok = emqx_listeners:restart(),
     ok = emqx_listeners:stop().
 
-t_restart_listeners_with_hibernate_after_disabled(Config) ->
+t_restart_listeners_with_hibernate_after_disabled(_Config) ->
     OldLConf = emqx_config:get([listeners]),
     maps:foreach(
         fun(LType, Listeners) ->
@@ -154,7 +154,7 @@ t_restart_listeners_with_hibernate_after_disabled(Config) ->
                                     ssl_options,
                                     hibernate_after
                                 ],
-                                5000
+                                undefined
                             );
                         _ ->
                             skip

From 4e6c7bfcbe676b1e9e3acc2454335deffa6e898d Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 14 Feb 2023 08:50:22 +0100
Subject: [PATCH 026/144] chore: changelog for hibernate_after

---
 apps/emqx/src/emqx.app.src      | 2 +-
 changes/v5.0.18/perf-9967-en.md | 1 +
 changes/v5.0.18/perf-9967-zh.md | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)
 create mode 100644 changes/v5.0.18/perf-9967-en.md
 create mode 100644 changes/v5.0.18/perf-9967-zh.md

diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src
index c812b2217..3030ccb06 100644
--- a/apps/emqx/src/emqx.app.src
+++ b/apps/emqx/src/emqx.app.src
@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.17"},
+    {vsn, "5.0.18"},
     {modules, []},
     {registered, []},
     {applications, [
diff --git a/changes/v5.0.18/perf-9967-en.md b/changes/v5.0.18/perf-9967-en.md
new file mode 100644
index 000000000..fadba24c9
--- /dev/null
+++ b/changes/v5.0.18/perf-9967-en.md
@@ -0,0 +1 @@
+New common TLS option 'hibernate_after' to reduce memory footprint per idle connecion, default: 5s.
diff --git a/changes/v5.0.18/perf-9967-zh.md b/changes/v5.0.18/perf-9967-zh.md
new file mode 100644
index 000000000..7b73f9bd0
--- /dev/null
+++ b/changes/v5.0.18/perf-9967-zh.md
@@ -0,0 +1 @@
+新的通用 TLS 选项 'hibernate_after'， 以减少空闲连接的内存占用，默认： 5s 。

From 4f75dbbe7fddcf42270764e69c685d250e26eb5c Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Wed, 15 Feb 2023 10:21:07 +0100
Subject: [PATCH 027/144] chore: remove otp version from rpm package name

---
 deploy/packages/rpm/Makefile  | 4 +---
 deploy/packages/rpm/emqx.spec | 4 ++--
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile
index 67ae7c907..526761380 100644
--- a/deploy/packages/rpm/Makefile
+++ b/deploy/packages/rpm/Makefile
@@ -7,13 +7,12 @@ none :=
 space := $(none) $(none)
 ## RPM does not allow '-' in version number and release string, replace with '_'
 RPM_VSN := $(subst -,_,$(PKG_VSN))
-RPM_REL := otp$(subst -,_,$(OTP_VSN))
 
 EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD))
 
 TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx
 TAR_PKG     := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz
-SOURCE_PKG  := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m)
+SOURCE_PKG  := emqx-$(RPM_VSN).$(shell uname -m)
 TARGET_PKG  := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long)
 
 # Not $(PWD) as it does not work for make -C
@@ -33,7 +32,6 @@ all: | $(BUILT)
 		--define "_topdir $(TOPDIR)" \
 		--define "_version $(RPM_VSN)" \
 		--define "_reldir $(SRCDIR)" \
-		--define "_release $(RPM_REL)" \
 		--define "_post_addition $(POST_ADDITION)" \
 		--define "_preun_addition $(PREUN_ADDITION)" \
 		--define "_sharedstatedir /var/lib" \
diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec
index 366f85396..b2b58ac23 100644
--- a/deploy/packages/rpm/emqx.spec
+++ b/deploy/packages/rpm/emqx.spec
@@ -5,12 +5,12 @@
 %define _log_dir %{_var}/log/%{_name}
 %define _lib_home /usr/lib/%{_name}
 %define _var_home %{_sharedstatedir}/%{_name}
-%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}.%{_arch}.rpm
+%define _build_name_fmt %{_arch}/%{_name}-%{_version}.%{_arch}.rpm
 %define _build_id_links none
 
 Name: %{_package_name}
 Version: %{_version}
-Release: %{_release}%{?dist}
+Release: 1%{?dist}
 Summary: emqx
 Group: System Environment/Daemons
 License: Apache License Version 2.0

From 9be73bfff048b48bd083ab525e89a892ab554a29 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 15 Feb 2023 10:37:13 +0100
Subject: [PATCH 028/144] fix: support ipv6 at ip_port

---
 apps/emqx_management/src/emqx_mgmt_api.erl | 29 +++++++++++++++++-----
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index fa0bc6e7a..12292b7d9 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -582,14 +582,15 @@ to_timestamp(B) when is_binary(B) ->
     binary_to_integer(B).
 
 to_ip(IP0) when is_binary(IP0) ->
-    {ok, IP} = inet:parse_address(binary_to_list(IP0)),
-    IP.
+    ensure_ok(inet:parse_address(binary_to_list(IP0))).
 
 to_ip_port(IPAddress) ->
-    [IP0, Port0] = string:tokens(binary_to_list(IPAddress), ":"),
-    {ok, IP} = inet:parse_address(IP0),
-    Port = list_to_integer(Port0),
-    {IP, Port}.
+    ensure_ok(emqx_schema:to_ip_port(IPAddress)).
+
+ensure_ok({ok, V}) ->
+    V;
+ensure_ok({error, _R} = E) ->
+    throw(E).
 
 b2i(Bin) when is_binary(Bin) ->
     binary_to_integer(Bin);
@@ -648,9 +649,25 @@ params2qs_test_() ->
     [
         ?_assertEqual({10, {ExpectedQs, FuzzyNQString}}, parse_qstring(QString, QSchema)),
         ?_assertEqual({0, {[], []}}, parse_qstring([{not_a_predefined_params, val}], QSchema)),
+        ?_assertEqual(
+            {1, {[{ip, '=:=', {0, 0, 0, 0, 0, 0, 0, 1}}], []}},
+            parse_qstring([{<<"ip">>, <<"::1">>}], QSchema)
+        ),
+        ?_assertEqual(
+            {1, {[{ip_port, '=:=', {{0, 0, 0, 0, 0, 0, 0, 1}, 8888}}], []}},
+            parse_qstring([{<<"ip_port">>, <<"::1:8888">>}], QSchema)
+        ),
         ?_assertThrow(
             {bad_value_type, {<<"ip">>, ip, <<"helloworld">>}},
             parse_qstring([{<<"ip">>, <<"helloworld">>}], QSchema)
+        ),
+        ?_assertThrow(
+            {bad_value_type, {<<"ip_port">>, ip_port, <<"127.0.0.1">>}},
+            parse_qstring([{<<"ip_port">>, <<"127.0.0.1">>}], QSchema)
+        ),
+        ?_assertThrow(
+            {bad_value_type, {<<"ip_port">>, ip_port, <<"helloworld:abcd">>}},
+            parse_qstring([{<<"ip_port">>, <<"helloworld:abcd">>}], QSchema)
         )
     ].
 

From 21b8ff6d141b74acfc9c9ba878c62868ac4a1399 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 15 Feb 2023 13:31:48 +0100
Subject: [PATCH 029/144] test: use unit test to test paginate

---
 apps/emqx_management/src/emqx_mgmt_api.erl    | 48 +++++++++++++++++++
 .../test/emqx_mgmt_api_SUITE.erl              | 11 -----
 2 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 12292b7d9..a2538d06b 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -33,6 +33,10 @@
     b2i/1
 ]).
 
+-ifdef(TEST).
+-export([paginate_test_format/1]).
+-endif.
+
 -export_type([
     match_spec_and_filter/0
 ]).
@@ -671,4 +675,48 @@ params2qs_test_() ->
         )
     ].
 
+paginate_test_format(Row) ->
+    Row.
+
+paginate_test_() ->
+    _ = ets:new(?MODULE, [named_table]),
+    Size = 1000,
+    MyLimit = 10,
+    ets:insert(?MODULE, [{I, foo} || I <- lists:seq(1, Size)]),
+    DefaultLimit = emqx_mgmt:max_row_limit(),
+    NoParamsResult = paginate(?MODULE, #{}, {?MODULE, paginate_test_format}),
+    PaginateResults = [
+        paginate(
+            ?MODULE, #{<<"page">> => I, <<"limit">> => MyLimit}, {?MODULE, paginate_test_format}
+        )
+     || I <- lists:seq(1, floor(Size / MyLimit))
+    ],
+    [
+        ?_assertMatch(
+            #{meta := #{count := Size, page := 1, limit := DefaultLimit}}, NoParamsResult
+        ),
+        ?_assertEqual(DefaultLimit, length(maps:get(data, NoParamsResult))),
+        ?_assertEqual(
+            #{data => [], meta => #{count => Size, limit => DefaultLimit, page => 100}},
+            paginate(?MODULE, #{<<"page">> => <<"100">>}, {?MODULE, paginate_test_format})
+        )
+    ] ++ assertPaginateResults(PaginateResults, Size, MyLimit).
+
+assertPaginateResults(Results, Size, Limit) ->
+    AllData = lists:flatten([Data || #{data := Data} <- Results]),
+    [
+        begin
+            Result = lists:nth(I, Results),
+            [
+                ?_assertMatch(#{meta := #{count := Size, limit := Limit, page := I}}, Result),
+                ?_assertEqual(Limit, length(maps:get(data, Result)))
+            ]
+        end
+     || I <- lists:seq(1, floor(Size / Limit))
+    ] ++
+        [
+            ?_assertEqual(floor(Size / Limit), length(Results)),
+            ?_assertEqual(Size, length(AllData)),
+            ?_assertEqual(Size, sets:size(sets:from_list(AllData)))
+        ].
 -endif.
diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
index da9a338f2..4d0262e6a 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl
@@ -173,17 +173,6 @@ t_cluster_query(_Config) ->
     end,
     ok.
 
-t_paging(_) ->
-    emqx_mgmt_api_test_util:init_suite(),
-    try
-        Path0 = emqx_mgmt_api_test_util:api_path(["banned?page=1"]),
-        {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path0),
-        Path1 = emqx_mgmt_api_test_util:api_path(["banned?page=10"]),
-        {ok, _} = emqx_mgmt_api_test_util:request_api(get, Path1)
-    after
-        emqx_mgmt_api_test_util:end_suite()
-    end.
-
 t_bad_rpc(_) ->
     emqx_mgmt_api_test_util:init_suite(),
     process_flag(trap_exit, true),

From 921b5b2c5c6f7be58f3a5e582e11d65497b991ff Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 15 Feb 2023 13:38:14 +0100
Subject: [PATCH 030/144] style: rename function for compliance

---
 apps/emqx_management/src/emqx_mgmt_api.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index a2538d06b..3c4d787d3 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -700,9 +700,9 @@ paginate_test_() ->
             #{data => [], meta => #{count => Size, limit => DefaultLimit, page => 100}},
             paginate(?MODULE, #{<<"page">> => <<"100">>}, {?MODULE, paginate_test_format})
         )
-    ] ++ assertPaginateResults(PaginateResults, Size, MyLimit).
+    ] ++ assert_paginate_results(PaginateResults, Size, MyLimit).
 
-assertPaginateResults(Results, Size, Limit) ->
+assert_paginate_results(Results, Size, Limit) ->
     AllData = lists:flatten([Data || #{data := Data} <- Results]),
     [
         begin

From 1cf88ea0b4c141a21a37372925b7178bac088f54 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 15 Feb 2023 15:24:10 +0100
Subject: [PATCH 031/144] docs: improve bridge_mode description

---
 apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf
index f9f79beb8..0de97d84b 100644
--- a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf
+++ b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf
@@ -114,9 +114,13 @@ topic filters for <code>remote.topic</code> of ingress connections."""
         desc {
             en: """If enable bridge mode.
 NOTE: This setting is only for MQTT protocol version older than 5.0, and the remote MQTT
-broker MUST support this feature."""
+broker MUST support this feature.
+If bridge_mode is set to true, the bridge will indicate to the remote broker that it is a bridge not an ordinary client.
+This means that loop detection will be more effective and that retained messages will be propagated correctly."""
             zh: """是否启用 Bridge Mode。
-注意：此设置只针对 MQTT 协议版本 < 5.0 有效，并且需要远程 MQTT Broker 支持 Bridge Mode。"""
+注意：此设置只针对 MQTT 协议版本 < 5.0 有效，并且需要远程 MQTT Broker 支持 Bridge Mode。
+如果设置为 true ，桥接会告诉远端服务器当前连接是一个桥接而不是一个普通的客户端。
+这意味着消息回环检测会更加高效，并且远端服务器收到的保留消息的标志位会透传给本地。"""
         }
         label {
             en: "Bridge Mode"

From 719e02cc96e6002fa9c88049e49772aa64030f90 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 30 Jun 2022 20:41:29 +0200
Subject: [PATCH 032/144] fix(metrics): use the same data source for RAM usage
 info

---
 apps/emqx/src/emqx.appup.src                  | 32 ++-----------------
 apps/emqx/src/emqx_vm.erl                     |  7 ----
 apps/emqx/test/emqx_vm_SUITE.erl              |  6 ----
 apps/emqx_management/src/emqx_mgmt.erl        | 22 ++++++++++++-
 .../src/emqx_prometheus.app.src               |  2 +-
 apps/emqx_prometheus/src/emqx_prometheus.erl  | 15 +--------
 apps/emqx_statsd/src/emqx_statsd.app.src      |  2 +-
 apps/emqx_statsd/src/emqx_statsd.erl          | 16 +---------
 8 files changed, 27 insertions(+), 75 deletions(-)

diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src
index d3121c97b..04bf1f428 100644
--- a/apps/emqx/src/emqx.appup.src
+++ b/apps/emqx/src/emqx.appup.src
@@ -1,33 +1,5 @@
 %% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
-  [{"5.0.0",
-    [{load_module,emqx_quic_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_config,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_schema,brutal_purge,soft_purge,[]},
-     {load_module,emqx_release,brutal_purge,soft_purge,[]},
-     {load_module,emqx_authentication,brutal_purge,soft_purge,[]},
-     {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
-     {add_module,emqx_exclusive_subscription},
-     {apply,{emqx_exclusive_subscription,on_add_module,[]}},
-     {load_module,emqx_broker,brutal_purge,soft_purge,[]},
-     {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]},
-     {load_module,emqx_topic,brutal_purge,soft_purge,[]},
-     {load_module,emqx_relup}]},
-   {<<".*">>,[]}],
-  [{"5.0.0",
-    [{load_module,emqx_quic_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_config,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_schema,brutal_purge,soft_purge,[]},
-     {load_module,emqx_release,brutal_purge,soft_purge,[]},
-     {load_module,emqx_authentication,brutal_purge,soft_purge,[]},
-     {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
-     {load_module,emqx_broker,brutal_purge,soft_purge,[]},
-     {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]},
-     {load_module,emqx_topic,brutal_purge,soft_purge,[]},
-     {apply,{emqx_exclusive_subscription,on_delete_module,[]}},
-     {delete_module,emqx_exclusive_subscription},
-     {load_module,emqx_relup}]},
-   {<<".*">>,[]}]}.
+  [{<<".*">>,[]}],
+  [{<<".*">>,[]}]}.
diff --git a/apps/emqx/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl
index f80d18a3a..0d861f671 100644
--- a/apps/emqx/src/emqx_vm.erl
+++ b/apps/emqx/src/emqx_vm.erl
@@ -24,7 +24,6 @@
     get_system_info/1,
     get_memory/0,
     get_memory/2,
-    mem_info/0,
     loads/0
 ]).
 
@@ -226,12 +225,6 @@ convert_allocated_areas({Key, Value1, Value2}) ->
 convert_allocated_areas({Key, Value}) ->
     {Key, Value}.
 
-mem_info() ->
-    Dataset = memsup:get_system_memory_data(),
-    Total = proplists:get_value(total_memory, Dataset),
-    Free = proplists:get_value(free_memory, Dataset),
-    [{total_memory, Total}, {used_memory, Total - Free}].
-
 %%%% erlang vm scheduler_usage  fun copied from recon
 scheduler_usage(Interval) when is_integer(Interval) ->
     %% We start and stop the scheduler_wall_time system flag
diff --git a/apps/emqx/test/emqx_vm_SUITE.erl b/apps/emqx/test/emqx_vm_SUITE.erl
index 35f37a41e..12f28ed28 100644
--- a/apps/emqx/test/emqx_vm_SUITE.erl
+++ b/apps/emqx/test/emqx_vm_SUITE.erl
@@ -50,12 +50,6 @@ t_systeminfo(_Config) ->
     ),
     ?assertEqual(undefined, emqx_vm:get_system_info(undefined)).
 
-t_mem_info(_Config) ->
-    application:ensure_all_started(os_mon),
-    MemInfo = emqx_vm:mem_info(),
-    [{total_memory, _}, {used_memory, _}] = MemInfo,
-    application:stop(os_mon).
-
 t_process_info(_Config) ->
     ProcessInfo = emqx_vm:get_process_info(),
     ?assertEqual(emqx_vm:process_info_keys(), [K || {K, _V} <- ProcessInfo]).
diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 814b39cdc..2d6fa854e 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -104,7 +104,10 @@
 ]).
 
 %% Common Table API
--export([max_row_limit/0]).
+-export([
+    max_row_limit/0,
+    vm_stats/0
+]).
 
 -define(APP, emqx_management).
 
@@ -161,6 +164,23 @@ node_info(Nodes) ->
 stopped_node_info(Node) ->
     #{name => Node, node_status => 'stopped'}.
 
+vm_stats() ->
+    Idle =
+        case cpu_sup:util([detailed]) of
+            %% Not support for Windows
+            {_, 0, 0, _} -> 0;
+            {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0)
+        end,
+    RunQueue = erlang:statistics(run_queue),
+    {MemUsedRatio, MemTotal} = get_sys_memory(),
+    [
+        {run_queue, RunQueue},
+        {cpu_idle, Idle},
+        {cpu_use, 100 - Idle},
+        {total_memory, MemTotal},
+        {used_memory, erlang:round(MemTotal * MemUsedRatio)}
+    ].
+
 %%--------------------------------------------------------------------
 %% Brokers
 %%--------------------------------------------------------------------
diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src
index 013de63fb..1ba2411ac 100644
--- a/apps/emqx_prometheus/src/emqx_prometheus.app.src
+++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src
@@ -2,7 +2,7 @@
 {application, emqx_prometheus, [
     {description, "Prometheus for EMQX"},
     % strict semver, bump manually!
-    {vsn, "5.0.5"},
+    {vsn, "5.0.6"},
     {modules, []},
     {registered, [emqx_prometheus_sup]},
     {applications, [kernel, stdlib, prometheus, emqx]},
diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl
index 4712a43c8..62e6f1d9a 100644
--- a/apps/emqx_prometheus/src/emqx_prometheus.erl
+++ b/apps/emqx_prometheus/src/emqx_prometheus.erl
@@ -590,20 +590,7 @@ emqx_vm() ->
     ].
 
 emqx_vm_data() ->
-    Idle =
-        case cpu_sup:util([detailed]) of
-            %% Not support for Windows
-            {_, 0, 0, _} -> 0;
-            {_Num, _Use, IdleList, _} -> ?C(idle, IdleList)
-        end,
-    RunQueue = erlang:statistics(run_queue),
-    [
-        {run_queue, RunQueue},
-        %% XXX: Plan removed at v5.0
-        {process_total_messages, 0},
-        {cpu_idle, Idle},
-        {cpu_use, 100 - Idle}
-    ] ++ emqx_vm:mem_info().
+    emqx_mgmt:vm_stats().
 
 emqx_cluster() ->
     [
diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src
index 638c5a33b..26a0cc278 100644
--- a/apps/emqx_statsd/src/emqx_statsd.app.src
+++ b/apps/emqx_statsd/src/emqx_statsd.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_statsd, [
     {description, "EMQX Statsd"},
-    {vsn, "5.0.4"},
+    {vsn, "5.0.5"},
     {registered, []},
     {mod, {emqx_statsd_app, []}},
     {applications, [
diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl
index defaf78e0..770320ddd 100644
--- a/apps/emqx_statsd/src/emqx_statsd.erl
+++ b/apps/emqx_statsd/src/emqx_statsd.erl
@@ -105,7 +105,7 @@ handle_info(
         timer := Ref
     }
 ) ->
-    Metrics = emqx_metrics:all() ++ emqx_stats:getstats() ++ emqx_vm_data(),
+    Metrics = emqx_metrics:all() ++ emqx_stats:getstats() ++ emqx_mgmt:vm_stats(),
     SampleRate = SampleTimeInterval / FlushTimeInterval,
     StatsdMetrics = [
         {gauge, Name, Value, SampleRate, []}
@@ -129,20 +129,6 @@ terminate(_Reason, #{estatsd_pid := Pid}) ->
 %% Internal function
 %%------------------------------------------------------------------------------
 
-emqx_vm_data() ->
-    Idle =
-        case cpu_sup:util([detailed]) of
-            %% Not support for Windows
-            {_, 0, 0, _} -> 0;
-            {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0)
-        end,
-    RunQueue = erlang:statistics(run_queue),
-    [
-        {run_queue, RunQueue},
-        {cpu_idle, Idle},
-        {cpu_use, 100 - Idle}
-    ] ++ emqx_vm:mem_info().
-
 ensure_timer(State = #{sample_time_interval := SampleTimeInterval}) ->
     State#{timer => emqx_misc:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}.
 

From 474979ae47068ada35b6e23023f3f444ece100cb Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 15 Feb 2023 14:31:14 +0100
Subject: [PATCH 033/144] docs: add changelogs for 9974

---
 changes/v5.0.18/fix-9974.en.md | 2 ++
 changes/v5.0.18/fix-9974.zh.md | 2 ++
 2 files changed, 4 insertions(+)
 create mode 100644 changes/v5.0.18/fix-9974.en.md
 create mode 100644 changes/v5.0.18/fix-9974.zh.md

diff --git a/changes/v5.0.18/fix-9974.en.md b/changes/v5.0.18/fix-9974.en.md
new file mode 100644
index 000000000..97223e03f
--- /dev/null
+++ b/changes/v5.0.18/fix-9974.en.md
@@ -0,0 +1,2 @@
+Report memory usage to statsd and prometheus using the same data source as dashboard.
+Prior to this fix, the memory usage data source was collected from an outdated source which did not work well in containers.
diff --git a/changes/v5.0.18/fix-9974.zh.md b/changes/v5.0.18/fix-9974.zh.md
new file mode 100644
index 000000000..8358204f3
--- /dev/null
+++ b/changes/v5.0.18/fix-9974.zh.md
@@ -0,0 +1,2 @@
+Statsd 和 prometheus 使用跟 Dashboard 相同的内存用量数据源。
+在此修复前，内存的总量和用量统计使用了过时的（在容器环境中不准确）的数据源。

From dbda504f2c4011940b305aacb0594ef58eabf990 Mon Sep 17 00:00:00 2001
From: Thales Macedo Garitezi <thalesmg@gmail.com>
Date: Wed, 15 Feb 2023 11:57:45 -0300
Subject: [PATCH 034/144] fix(pgsql): fix ssl option for pgsql connector to
 match previous behavior (5.0)

Fixes https://github.com/emqx/emqx/issues/9907

At v5.0.14, we changed the `ssl` option for the Postgres connector
from `true` to `required`, but there was another transformation in
`conn_opts/2` that led to an incorrect configuration.  This change
ended up preventing users from connecting to Postgres with their
previous configurations after upgrading EMQX.
---
 .ci/docker-compose-file/pgsql/Dockerfile         | 2 +-
 .ci/docker-compose-file/pgsql/pg_hba_tls.conf    | 8 ++++++++
 apps/emqx_connector/src/emqx_connector_pgsql.erl | 6 +++++-
 changes/v5.0.18/fix-9978.en.md                   | 2 ++
 changes/v5.0.18/fix-9978.zh.md                   | 2 ++
 5 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100644 .ci/docker-compose-file/pgsql/pg_hba_tls.conf
 create mode 100644 changes/v5.0.18/fix-9978.en.md
 create mode 100644 changes/v5.0.18/fix-9978.zh.md

diff --git a/.ci/docker-compose-file/pgsql/Dockerfile b/.ci/docker-compose-file/pgsql/Dockerfile
index f26e18d0e..8598d3f33 100644
--- a/.ci/docker-compose-file/pgsql/Dockerfile
+++ b/.ci/docker-compose-file/pgsql/Dockerfile
@@ -1,7 +1,7 @@
 ARG BUILD_FROM=postgres:13
 FROM ${BUILD_FROM}
 ARG POSTGRES_USER=postgres
-COPY --chown=$POSTGRES_USER ./pgsql/pg_hba.conf /var/lib/postgresql/pg_hba.conf
+COPY --chown=$POSTGRES_USER ./pgsql/pg_hba_tls.conf /var/lib/postgresql/pg_hba.conf
 COPY --chown=$POSTGRES_USER certs/server.key /var/lib/postgresql/server.key
 COPY --chown=$POSTGRES_USER certs/server.crt /var/lib/postgresql/server.crt
 COPY --chown=$POSTGRES_USER certs/ca.crt /var/lib/postgresql/root.crt
diff --git a/.ci/docker-compose-file/pgsql/pg_hba_tls.conf b/.ci/docker-compose-file/pgsql/pg_hba_tls.conf
new file mode 100644
index 000000000..356afd9a6
--- /dev/null
+++ b/.ci/docker-compose-file/pgsql/pg_hba_tls.conf
@@ -0,0 +1,8 @@
+# TYPE DATABASE USER CIDR-ADDRESS METHOD
+local   all all trust
+# TODO: also test with `cert`? will require client certs
+hostssl all all 0.0.0.0/0 password
+hostssl all all ::/0      password
+
+hostssl all www-data 0.0.0.0/0 cert clientcert=1
+hostssl all postgres 0.0.0.0/0 cert clientcert=1
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index 890227b9d..1fc994275 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -100,7 +100,11 @@ on_start(
         case maps:get(enable, SSL) of
             true ->
                 [
-                    {ssl, required},
+                    %% note: this is converted to `required' in
+                    %% `conn_opts/2', and there's a boolean guard
+                    %% there; if this is set to `required' here,
+                    %% that'll require changing `conn_opts/2''s guard.
+                    {ssl, true},
                     {ssl_opts, emqx_tls_lib:to_client_opts(SSL)}
                 ];
             false ->
diff --git a/changes/v5.0.18/fix-9978.en.md b/changes/v5.0.18/fix-9978.en.md
new file mode 100644
index 000000000..6750d136f
--- /dev/null
+++ b/changes/v5.0.18/fix-9978.en.md
@@ -0,0 +1,2 @@
+Fixed configuration issue when choosing to use SSL for a Postgres connection (`authn`, `authz` and bridge).
+The connection could fail to complete with a previously working configuration after an upgrade from 5.0.13 to newer EMQX versions.
diff --git a/changes/v5.0.18/fix-9978.zh.md b/changes/v5.0.18/fix-9978.zh.md
new file mode 100644
index 000000000..75eed3600
--- /dev/null
+++ b/changes/v5.0.18/fix-9978.zh.md
@@ -0,0 +1,2 @@
+修正了在Postgres连接中选择使用SSL时的配置问题（`authn`, `authz` 和 bridge）。
+从5.0.13升级到较新的EMQX版本后，连接可能无法完成之前的配置。

From 82b2f9f245b44bc13b9afb46e126cf71222d60ca Mon Sep 17 00:00:00 2001
From: Zhongwen Deng <zhongwencool@gmail.com>
Date: Thu, 16 Feb 2023 11:37:00 +0800
Subject: [PATCH 035/144] chore: include emqx_management in
 emqx_prometheus/emqx_statsd

---
 apps/emqx_prometheus/src/emqx_prometheus.app.src | 2 +-
 apps/emqx_statsd/src/emqx_statsd.app.src         | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src
index 1ba2411ac..6970ba777 100644
--- a/apps/emqx_prometheus/src/emqx_prometheus.app.src
+++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src
@@ -5,7 +5,7 @@
     {vsn, "5.0.6"},
     {modules, []},
     {registered, [emqx_prometheus_sup]},
-    {applications, [kernel, stdlib, prometheus, emqx]},
+    {applications, [kernel, stdlib, prometheus, emqx, emqx_management]},
     {mod, {emqx_prometheus_app, []}},
     {env, []},
     {licenses, ["Apache-2.0"]},
diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src
index 26a0cc278..27f842ce2 100644
--- a/apps/emqx_statsd/src/emqx_statsd.app.src
+++ b/apps/emqx_statsd/src/emqx_statsd.app.src
@@ -8,7 +8,8 @@
         kernel,
         stdlib,
         estatsd,
-        emqx
+        emqx,
+        emqx_management
     ]},
     {env, []},
     {modules, []},

From 6599c44213337c67688712a985d068ef791c052e Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Wed, 15 Feb 2023 14:00:40 +0800
Subject: [PATCH 036/144] chore: refactor the format-changelog script

---
 changes/ce/.gitkeep                 |  0
 changes/ee/.gitkeep                 |  0
 scripts/changelog-lang-templates/en | 12 +++++
 scripts/changelog-lang-templates/zh | 12 +++++
 scripts/format-changelog.sh         | 79 +++++++++++++++--------------
 scripts/rel/cut.sh                  | 22 ++++++--
 6 files changed, 83 insertions(+), 42 deletions(-)
 create mode 100644 changes/ce/.gitkeep
 create mode 100644 changes/ee/.gitkeep
 create mode 100644 scripts/changelog-lang-templates/en
 create mode 100644 scripts/changelog-lang-templates/zh

diff --git a/changes/ce/.gitkeep b/changes/ce/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/changes/ee/.gitkeep b/changes/ee/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/scripts/changelog-lang-templates/en b/scripts/changelog-lang-templates/en
new file mode 100644
index 000000000..05c218c7e
--- /dev/null
+++ b/scripts/changelog-lang-templates/en
@@ -0,0 +1,12 @@
+# ${version}
+
+## Enhancements
+
+$(section feat)
+
+$(section perf)
+
+## Bug fixes
+
+$(section fix)
+
diff --git a/scripts/changelog-lang-templates/zh b/scripts/changelog-lang-templates/zh
new file mode 100644
index 000000000..2bafd99d7
--- /dev/null
+++ b/scripts/changelog-lang-templates/zh
@@ -0,0 +1,12 @@
+# ${version}
+
+## 增强
+
+$(section feat)
+
+$(section perf)
+
+## 修复
+
+$(section fix)
+
diff --git a/scripts/format-changelog.sh b/scripts/format-changelog.sh
index c7b413cc8..87fbc60a2 100755
--- a/scripts/format-changelog.sh
+++ b/scripts/format-changelog.sh
@@ -3,20 +3,27 @@ set -euo pipefail
 shopt -s nullglob
 export LANG=C.UTF-8
 
-[ "$#" -ne 2 ] && {
-    echo "Usage $0 <EMQX version> <en|zh>" 1>&2;
+[ "$#" -ne 4 ] && {
+    echo "Usage $0 <emqx|emqx-enterprise> <LAST TAG> <VERSION> <OUTPUT DIR>" 1>&2;
     exit 1
 }
 
-version="${1}"
-language="${2}"
+profile="${1}"
+last_tag="${2}"
+version="${3}"
+output_dir="${4}"
+languages=("en" "zh")
+top_dir="$(git rev-parse --show-toplevel)"
+templates_dir="$top_dir/scripts/changelog-lang-templates"
+declare -a changes
+changes=("")
 
-changes_dir="$(git rev-parse --show-toplevel)/changes/${version}"
+echo "generated changelogs from tag:${last_tag} to HEAD"
 
 item() {
     local filename pr indent
     filename="${1}"
-    pr="$(echo "${filename}" | sed -E 's/.*-([0-9]+)\.(en|zh)\.md$/\1/')"
+    pr="$(echo "${filename}" | sed -E 's/.*-([0-9]+)\.[a-z]+\.md$/\1/')"
     indent="- [#${pr}](https://github.com/emqx/emqx/pull/${pr}) "
     while read -r line; do
         echo "${indent}${line}"
@@ -27,40 +34,36 @@ item() {
 
 section() {
     local prefix=$1
-    for i in "${changes_dir}"/"${prefix}"-*."${language}".md; do
-        item "${i}"
+    for file in "${changes[@]}"; do
+        if [[ $file =~ .*$prefix-.*$language.md ]]; then
+            item "$file"
+        fi
     done
 }
 
-if [ "${language}" = "en" ]; then
-    cat <<EOF
-# ${version}
+generate() {
+    local language=$1
+    local output="$output_dir/${version}_$language.md"
+    local template_file="$templates_dir/$language"
+    local template
+    if [ -f "$template_file" ]; then
+        template=$(cat "$template_file")
+        eval "echo \"$template\" > $output"
+    else
+        echo "Invalid language ${language}" 1>&2;
+        exit 1
+    fi
+}
 
-## Enhancements
-
-$(section feat)
-
-$(section perf)
-
-## Bug fixes
-
-$(section fix)
-EOF
-elif [ "${language}" = "zh" ]; then
-     cat <<EOF
-# ${version}
-
-## 增强
-
-$(section feat)
-
-$(section perf)
-
-## 修复
-
-$(section fix)
-EOF
-else
-    echo "Invalid language ${language}" 1>&2;
-    exit 1
+changes_dir=("$top_dir/changes/ce")
+if [ "$profile" == "emqx-enterprise" ]; then
+    changes_dir+=("$top_dir/changes/ee")
 fi
+
+while read -d "" -r file; do
+   changes+=("$file")
+done < <(git diff --name-only -z -a "tags/${last_tag}...HEAD" "${changes_dir[@]}")
+
+for language in "${languages[@]}"; do
+    generate "$language"
+done
diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh
index e03d5eff4..8d00694ac 100755
--- a/scripts/rel/cut.sh
+++ b/scripts/rel/cut.sh
@@ -26,6 +26,8 @@ options:
   --dryrun:  Do not actually create the git tag.
   --skip-appup: Skip checking appup
                 Useful when you are sure that appup is already updated'
+  --prev-tag: Provide the prev tag to automatically generate changelogs
+              If this option is absent, the tag found by git describe will be used
 
 NOTE: For 5.0 series the current working branch must be 'release-50' for opensource edition
       and 'release-e50' for enterprise edition.
@@ -92,6 +94,11 @@ while [ "$#" -gt 0 ]; do
             fi
             shift 2
             ;;
+        --prev-tag)
+            shift
+            PREV_TAG="$1"
+            shift
+            ;;
         *)
             logerr "Unknown option $1"
             exit 1
@@ -208,10 +215,17 @@ if [ -d "${CHECKS_DIR}" ]; then
 fi
 
 generate_changelog () {
-    local CHANGES_EN_MD="changes/${TAG}-en.md" CHANGES_ZH_MD="changes/${TAG}-zh.md"
-    ./scripts/format-changelog.sh "${TAG}" "en" > "$CHANGES_EN_MD"
-    ./scripts/format-changelog.sh "${TAG}" "zh" > "$CHANGES_ZH_MD"
-    git add "$CHANGES_EN_MD" "$CHANGES_ZH_MD"
+    local from_tag="${PREV_TAG:-}"
+    if [[ -z $from_tag ]]; then
+        if [ $PROFILE == "emqx" ]; then
+            from_tag="$(git describe --tags --abbrev=0 --match 'v*')"
+        else
+            from_tag="$(git describe --tags --abbrev=0 --match 'e*')"
+        fi
+    fi
+    local output_dir="changes"
+    ./scripts/format-changelog.sh $PROFILE "${from_tag}" "${TAG}" $output_dir
+    git add $output_dir
     [ -n "$(git status -s)" ] && git commit -m "chore: Generate changelog for ${TAG}"
 }
 

From c684730bbe8e1fd118a4367cd20a05d365c6e745 Mon Sep 17 00:00:00 2001
From: Ilya Averyanov <av@rubybox.dev>
Date: Thu, 16 Feb 2023 11:30:56 +0200
Subject: [PATCH 037/144] chore(ci): remove legacy helper Makefile

---
 .ci/docker-compose-file/Makefile.local | 67 --------------------------
 1 file changed, 67 deletions(-)
 delete mode 100644 .ci/docker-compose-file/Makefile.local

diff --git a/.ci/docker-compose-file/Makefile.local b/.ci/docker-compose-file/Makefile.local
deleted file mode 100644
index 2ce1bf170..000000000
--- a/.ci/docker-compose-file/Makefile.local
+++ /dev/null
@@ -1,67 +0,0 @@
-.PHONY: help up down ct ct-all bash run
-
-define usage
-make -f .ci/docker-compose-file/Makefile.local up
-make -f .ci/docker-compose-file/Makefile.local ct CONTAINER=erlang SUITE=apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl
-make -f .ci/docker-compose-file/Makefile.local down
-endef
-export usage
-
-help:
-	@echo "$$usage"
-
-up:
-	env \
-		MYSQL_TAG=8 \
-		REDIS_TAG=7.0 \
-		MONGO_TAG=5 \
-		PGSQL_TAG=13 \
-        TDENGINE_TAG=3.0.2.4 \
-	docker-compose \
-		-f .ci/docker-compose-file/docker-compose.yaml \
-		-f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-mongo-single-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-cluster-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
-		-f .ci/docker-compose-file/docker-compose-tdengine-restful.yaml \
-		up -d --build --remove-orphans
-
-down:
-	docker-compose \
-		-f .ci/docker-compose-file/docker-compose.yaml \
-		-f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-mongo-single-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-cluster-tcp.yaml \
-		-f .ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml \
-		-f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
-		-f .ci/docker-compose-file/docker-compose-tdengine-restful.yaml \
-		down --remove-orphans
-
-ct:
-	docker exec -i "$(CONTAINER)" bash -c "rebar3 ct --name 'test@127.0.0.1' --readable true -v --suite $(SUITE)"
-
-ct-all:
-	docker exec -i "$(CONTAINER)" bash -c "make ct"
-
-bash:
-	docker exec -it "$(CONTAINER)" bash
-
-run:
-	docker exec -it "$(CONTAINER)" bash -c "make run";

From dcf70e0e68348d24cecdbd30b66637c206c32d14 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Thu, 16 Feb 2023 14:10:42 +0100
Subject: [PATCH 038/144] refactor(emqx_resource): add more trace points for
 flushing

---
 .../src/emqx_resource_buffer_worker.erl       | 22 +++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
index b25725c41..bb4eee57d 100644
--- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
+++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl
@@ -468,7 +468,10 @@ flush(Data0) ->
         queue := Q0
     } = Data0,
     Data1 = cancel_flush_timer(Data0),
-    case {queue_count(Q0), is_inflight_full(InflightTID)} of
+    CurrentCount = queue_count(Q0),
+    IsFull = is_inflight_full(InflightTID),
+    ?tp(buffer_worker_flush, #{queue_count => CurrentCount, is_full => IsFull}),
+    case {CurrentCount, IsFull} of
         {0, _} ->
             {keep_state, Data1};
         {_, true} ->
@@ -595,8 +598,12 @@ do_flush(
                     result => Result
                 }
             ),
-            case queue_count(Q1) > 0 of
+            CurrentCount = queue_count(Q1),
+            case CurrentCount > 0 of
                 true ->
+                    ?tp(buffer_worker_flush_ack_reflush, #{
+                        batch_or_query => Request, result => Result, queue_count => CurrentCount
+                    }),
                     flush_worker(self());
                 false ->
                     ok
@@ -666,19 +673,26 @@ do_flush(Data0, #{
             {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result),
             store_async_worker_reference(InflightTID, Ref, WorkerMRef),
             emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)),
+            CurrentCount = queue_count(Q1),
             ?tp(
                 buffer_worker_flush_ack,
                 #{
                     batch_or_query => Batch,
-                    result => Result
+                    result => Result,
+                    queue_count => CurrentCount
                 }
             ),
-            CurrentCount = queue_count(Q1),
             Data2 =
                 case {CurrentCount > 0, CurrentCount >= BatchSize} of
                     {false, _} ->
                         Data1;
                     {true, true} ->
+                        ?tp(buffer_worker_flush_ack_reflush, #{
+                            batch_or_query => Batch,
+                            result => Result,
+                            queue_count => CurrentCount,
+                            batch_size => BatchSize
+                        }),
                         flush_worker(self()),
                         Data1;
                     {true, false} ->

From 2442a4dea7d04eba36ec75f112dd419427fae6e6 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Thu, 16 Feb 2023 14:14:29 +0100
Subject: [PATCH 039/144] test(emqx_resource): add regression test for
 recursive flushing

---
 .../test/emqx_resource_SUITE.erl              | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl
index a042bfb67..984b3b04a 100644
--- a/apps/emqx_resource/test/emqx_resource_SUITE.erl
+++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl
@@ -2291,6 +2291,67 @@ t_expiration_retry_batch_multiple_times(_Config) ->
     ),
     ok.
 
+t_recursive_flush(_Config) ->
+    emqx_connector_demo:set_callback_mode(async_if_possible),
+    {ok, _} = emqx_resource:create(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource},
+        #{
+            query_mode => async,
+            batch_size => 1,
+            batch_time => 10_000,
+            worker_pool_size => 1
+        }
+    ),
+    do_t_recursive_flush().
+
+t_recursive_flush_batch(_Config) ->
+    emqx_connector_demo:set_callback_mode(async_if_possible),
+    {ok, _} = emqx_resource:create(
+        ?ID,
+        ?DEFAULT_RESOURCE_GROUP,
+        ?TEST_RESOURCE,
+        #{name => test_resource},
+        #{
+            query_mode => async,
+            batch_size => 2,
+            batch_time => 10_000,
+            worker_pool_size => 1
+        }
+    ),
+    do_t_recursive_flush().
+
+do_t_recursive_flush() ->
+    ?check_trace(
+        begin
+            Timeout = 1_000,
+            Pid = spawn_link(fun S() ->
+                emqx_resource:query(?ID, {inc_counter, 1}),
+                S()
+            end),
+            %% we want two reflushes to happen before we analyze the
+            %% trace, so that we get a single full interaction
+            {ok, _} = snabbkaffe:block_until(
+                ?match_n_events(2, #{?snk_kind := buffer_worker_flush_ack_reflush}), Timeout
+            ),
+            unlink(Pid),
+            exit(Pid, kill),
+            ok
+        end,
+        fun(Trace) ->
+            %% check that a recursive flush leads to a new call to flush/1
+            Pairs = ?find_pairs(
+                #{?snk_kind := buffer_worker_flush_ack_reflush},
+                #{?snk_kind := buffer_worker_flush},
+                Trace
+            ),
+            ?assert(lists:any(fun(E) -> E end, [true || {pair, _, _} <- Pairs]))
+        end
+    ),
+    ok.
+
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------

From 6b81d9965f715c890868dc4d62289e0ffc471163 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 16 Feb 2023 09:59:20 +0100
Subject: [PATCH 040/144] fix(bin/emqx): allow starting two nodes from the same
 installation

If more than one node is boot from the same root directory
try to find the node by node name set in EMQX_NODE_NAME
or EMQX_NODE__NAME environment variable
---
 bin/emqx | 64 +++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 42 insertions(+), 22 deletions(-)

diff --git a/bin/emqx b/bin/emqx
index 6f953b777..9211bd338 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -76,6 +76,12 @@ logwarn() {
     fi
 }
 
+logdebug() {
+    if [ "$DEBUG" -eq 1 ]; then
+        echo "DEBUG: $*"
+    fi
+}
+
 die() {
     set +x
     logerr "$1"
@@ -453,24 +459,37 @@ if [ "$IS_ENTERPRISE" = 'yes' ]; then
     CONF_KEYS+=( 'license.key' )
 fi
 
-
-## Find the running node from 'ps -ef'
-## The primary grep pattern is $RUNNER_ROOT_DIR because one can start multiple nodes at the same time
-# shellcheck disable=SC2009
-PS_LINE="$(ps -ef | $GREP '[e]mqx' | $GREP -v -E '(remsh|nodetool)' | $GREP -oE "\-[r]oot ${RUNNER_ROOT_DIR}.*" || true)"
-if [ -n "${PS_LINE}" ]; then
-    RUNNING_NODES_COUNT="$(echo -e "$PS_LINE" | wc -l)"
-else
-    RUNNING_NODES_COUNT=0
+## To be backward compatible, read and then unset EMQX_NODE_NAME
+if [ -n "${EMQX_NODE_NAME:-}" ]; then
+    export EMQX_NODE__NAME="${EMQX_NODE_NAME}"
+    unset EMQX_NODE_NAME
 fi
 
 # Turn off debug as the ps output can be quite noisy
 set +x
+
+## Find the running node from 'ps -ef'
+##  * The grep args like '[e]mqx' but not 'emqx' is to avoid greping the grep command itself
+##  * The running 'remsh' and 'nodetool' processes must be excluded
+if [ -n "${EMQX_NODE__NAME:-}" ]; then
+    # if node name is provided, filter by node name
+    # shellcheck disable=SC2009
+    PS_LINE="$(ps -ef | $GREP '[e]mqx' | $GREP -v -E '(remsh|nodetool)' | $GREP -E "\s\-s?name\s${EMQX_NODE__NAME}" | $GREP -oE "\-[r]oot ${RUNNER_ROOT_DIR}.*" || true)"
+else
+    # shellcheck disable=SC2009
+    PS_LINE="$(ps -ef | $GREP '[e]mqx' | $GREP -v -E '(remsh|nodetool)' | $GREP -oE "\-[r]oot ${RUNNER_ROOT_DIR}.*" || true)"
+fi
+logdebug "PS_LINE=$PS_LINE"
+RUNNING_NODES_COUNT="$(echo -e "$PS_LINE" | sed '/^\s*$/d' | wc -l)"
+[ "$RUNNING_NODES_COUNT" -gt 1 ] && logdebug "More than one running node found: count=$RUNNING_NODES_COUNT"
+
 if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
     if [ "$RUNNING_NODES_COUNT" -gt 0 ] && [ "$COMMAND" != 'check_config' ]; then
-        tmp_nodename=$(echo -e "$PS_LINE" | $GREP -oE "\s\-s?name.*" | awk '{print $2}' || true)
-        echo "Node ${tmp_nodename} is already running!"
-        exit 1
+        running_node_name=$(echo -e "$PS_LINE" | $GREP -oE "\s\-s?name.*" | awk '{print $2}' || true)
+        if [ -n "$running_node_name" ] && [ "$running_node_name" = "${EMQX_NODE__NAME:-}" ]; then
+            echo "Node ${running_node_name} is already running!"
+            exit 1
+        fi
     fi
     [ -f "$EMQX_ETC_DIR"/emqx.conf ] || die "emqx.conf is not found in $EMQX_ETC_DIR" 1
     maybe_use_portable_dynlibs
@@ -502,8 +521,6 @@ else
     #    then update the config in the file to 'node.name = "emqx@local.net"', after this change,
     #    there would be no way stop the running node 'emqx@127.0.0.1', because 'emqx stop' command
     #    would try to stop the new node instead.
-    #  * The grep args like '[e]mqx' but not 'emqx' is to avoid greping the grep command itself
-    #  * The running 'remsh' and 'nodetool' processes must be excluded
     if [ "$RUNNING_NODES_COUNT" -eq 1 ]; then
         ## only one emqx node is running, get running args from 'ps -ef' output
         tmp_nodename=$(echo -e "$PS_LINE" | $GREP -oE "\s\-s?name.*" | awk '{print $2}' || true)
@@ -520,14 +537,22 @@ else
         ## Make the format like what call_hocon multi_get prints out, but only need 4 args
         EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}"
     else
-        ## None or more than one node is running, resolve from boot config
-        ## we have no choiece but to read the bootstrap config (with environment overrides available in the current shell)
+        if [ "$RUNNING_NODES_COUNT" -gt 1 ]; then
+            if [ -z "${EMQX_NODE__NAME:-}" ]; then
+                tmp_nodenames=$(echo -e "$PS_LINE" | $GREP -oE "\s\-s?name.*" | awk '{print $2}' | tr '\n' ' ')
+                logerr "More than one EMQX node found running (root dir: ${RUNNER_ROOT_DIR})"
+                logerr "Running nodes: $tmp_nodenames"
+                logerr "Make sure environment variable EMQX_NODE__NAME is set to indicate for which node this command is intended."
+                exit 1
+            fi
+        fi
+        ## We have no choiece but to read the bootstrap config (with environment overrides available in the current shell)
         [ -f "$EMQX_ETC_DIR"/emqx.conf ] || die "emqx.conf is not found in $EMQX_ETC_DIR" 1
         maybe_use_portable_dynlibs
         EMQX_BOOT_CONFIGS="$(call_hocon -s "$SCHEMA_MOD" -c "$EMQX_ETC_DIR"/emqx.conf multi_get "${CONF_KEYS[@]}")"
     fi
 fi
-[ "$DEBUG" -eq 1 ] && echo "EMQX_BOOT_CONFIGS: $EMQX_BOOT_CONFIGS"
+logdebug "EMQX_BOOT_CONFIGS: $EMQX_BOOT_CONFIGS"
 [ "$DEBUG" -eq 1 ] && set -x
 
 get_boot_config() {
@@ -877,11 +902,6 @@ maybe_log_to_console() {
     fi
 }
 
-## To be backward compatible, read and then unset EMQX_NODE_NAME
-if [ -n "${EMQX_NODE_NAME:-}" ]; then
-    export EMQX_NODE__NAME="${EMQX_NODE_NAME}"
-    unset EMQX_NODE_NAME
-fi
 ## Possible ways to configure emqx node name:
 ## 1. configure node.name in emqx.conf
 ## 2. override with environment variable EMQX_NODE__NAME

From 8fcce53a0d4d8ac11188e6b38af00daaea5c5e80 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 16 Feb 2023 11:17:04 +0100
Subject: [PATCH 041/144] test: add script to run two nodes in host from the
 same build

---
 .../{ => test}/start-two-nodes-in-docker.sh   |  0
 scripts/test/start-two-nodes-in-host.sh       | 63 +++++++++++++++++++
 2 files changed, 63 insertions(+)
 rename scripts/{ => test}/start-two-nodes-in-docker.sh (100%)
 create mode 100755 scripts/test/start-two-nodes-in-host.sh

diff --git a/scripts/start-two-nodes-in-docker.sh b/scripts/test/start-two-nodes-in-docker.sh
similarity index 100%
rename from scripts/start-two-nodes-in-docker.sh
rename to scripts/test/start-two-nodes-in-docker.sh
diff --git a/scripts/test/start-two-nodes-in-host.sh b/scripts/test/start-two-nodes-in-host.sh
new file mode 100755
index 000000000..7dbd6009e
--- /dev/null
+++ b/scripts/test/start-two-nodes-in-host.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+## This starts two nodes on the same host (not in docker).
+## The listener ports are shifted with an offset to avoid clashing.
+## The data and log directories are configured to use ./tmp/
+
+## By default, the boot script is ./_build/emqx/rel/emqx
+## it can be overriden with arg1 and arg2 for the two nodes respectfully
+
+# ensure dir
+cd -P -- "$(dirname -- "$0")/../../"
+
+DEFAULT_BOOT='./_build/emqx/rel/emqx/bin/emqx'
+BOOT1="${1:-$DEFAULT_BOOT}"
+BOOT2="${2:-$BOOT1}"
+
+DATA1="$(pwd)/tmp/emqx1/data"
+LOG1="$(pwd)/tmp/emqx1/log"
+DATA2="$(pwd)/tmp/emqx2/data"
+LOG2="$(pwd)/tmp/emqx2/log"
+
+mkdir -p "$DATA1" "$DATA2" "$LOG1" "$LOG2"
+
+echo "Stopping emqx1"
+env EMQX_NODE_NAME='emqx1@127.0.0.1' \
+    ./_build/emqx/rel/emqx/bin/emqx stop || true
+
+echo "Stopping emqx2"
+env EMQX_NODE_NAME='emqx2@127.0.0.1' \
+    ./_build/emqx/rel/emqx/bin/emqx stop || true
+
+## Fork-start node1, otherwise it'll keep waiting for node2 because we are using static cluster
+env DEBUG="${DEBUG:-0}" \
+    EMQX_CLUSTER__STATIC__SEEDS='["emqx1@127.0.0.1","emqx2@127.0.0.1"]' \
+    EMQX_CLUSTER__DISCOVERY_STRATEGY=static \
+    EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL="${EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL:-debug}" \
+    EMQX_LOG__FILE_HANDLERS__DEFAULT__FILE="$LOG1/emqx.log" \
+    EMQX_NODE_NAME='emqx1@127.0.0.1' \
+    EMQX_LOG_DIR="$LOG1" \
+    EMQX_NODE__DATA_DIR="$DATA1" \
+    EMQX_LISTENERS__TCP__DEFAULT__BIND='0.0.0.0:1881' \
+    EMQX_LISTENERS__SSL__DEFAULT__BIND='0.0.0.0:8881' \
+    EMQX_LISTENERS__WS__DEFAULT__BIND='0.0.0.0:8081' \
+    EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8084' \
+    EMQX_DASHBOARD__LISTENERS__HTTP__BIND='0.0.0.0:18081' \
+    "$BOOT1" start &
+
+env DEBUG="${DEBUG:-0}" \
+    EMQX_CLUSTER__STATIC__SEEDS='["emqx1@127.0.0.1","emqx2@127.0.0.1"]' \
+    EMQX_CLUSTER__DISCOVERY_STRATEGY=static \
+    EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL="${EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL:-debug}" \
+    EMQX_LOG__FILE_HANDLERS__DEFAULT__FILE="$LOG2/emqx.log" \
+    EMQX_NODE_NAME='emqx2@127.0.0.1' \
+    EMQX_LOG_DIR="$LOG2" \
+    EMQX_NODE__DATA_DIR="$DATA2" \
+    EMQX_LISTENERS__TCP__DEFAULT__BIND='0.0.0.0:1882' \
+    EMQX_LISTENERS__SSL__DEFAULT__BIND='0.0.0.0:8882' \
+    EMQX_LISTENERS__WS__DEFAULT__BIND='0.0.0.0:8082' \
+    EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8085' \
+    EMQX_DASHBOARD__LISTENERS__HTTP__BIND='0.0.0.0:18082' \
+    "$BOOT2" start

From 056bc71af26d486b220cb5780151e1d8ecef3f76 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Thu, 16 Feb 2023 15:05:38 +0100
Subject: [PATCH 042/144] chore: bump VSN version

---
 apps/emqx_resource/src/emqx_resource.app.src | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src
index 4618e94a6..cb26c7f09 100644
--- a/apps/emqx_resource/src/emqx_resource.app.src
+++ b/apps/emqx_resource/src/emqx_resource.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_resource, [
     {description, "Manager for all external resources"},
-    {vsn, "0.1.7"},
+    {vsn, "0.1.8"},
     {registered, []},
     {mod, {emqx_resource_app, []}},
     {applications, [

From 24825da2a813a5ecb8722874627feda9f62e15e6 Mon Sep 17 00:00:00 2001
From: Adrian Deaconu <adrian@hackerville.co.uk>
Date: Fri, 17 Feb 2023 01:10:52 +0000
Subject: [PATCH 043/144] feat: Add MQTT ingress and remove mgmt references

---
 deploy/charts/emqx/README.md              | 24 ++++++-----
 deploy/charts/emqx/templates/ingress.yaml | 50 +++++++++++++++++++++++
 deploy/charts/emqx/values.yaml            | 14 +++++++
 3 files changed, 77 insertions(+), 11 deletions(-)

diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index 6ee3617ce..352de3740 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -68,28 +68,30 @@ The following table lists the configurable parameters of the emqx chart and thei
 | `service.dashboard` | Port for dashboard and API. | 18083 |
 | `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil |
 | `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil |
-| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil |
 | `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | nil |
 | `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | nil |
 | `service.nodePorts.dashboard` | Kubernetes node port for dashboard. | nil |
 | `service.loadBalancerIP` | loadBalancerIP for Service | nil |
 | `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] |
 | `service.externalIPs` | ExternalIPs for the service | [] |
-`service.externalTrafficPolicy` |	External Traffic Policy for the service |	`Cluster`
+| `service.externalTrafficPolicy` |	External Traffic Policy for the service |	`Cluster`
 | `service.annotations` | Service annotations | {}(evaluated as a template) |
 | `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false |
 | `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | |
 | `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / |
 | `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` |
-| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local |
-| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] |
-| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} |
-| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false |
-| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | |
-| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / |
-| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local |
-| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] |
-| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} |
+| `ingress.dashboard.hosts` | Ingress hosts for EMQX Dashboard | dashboard.emqx.local |
+| `ingress.dashboard.tls` | Ingress tls for EMQX Dashboard | [] |
+| `ingress.dashboard.annotations` | Ingress annotations for EMQX Dashboard | {} |
+| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | |
+| `ingress.mqtt.enabled` | Enable ingress for MQTT | false |
+| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | |
+| `ingress.mqtt.path` | Ingress path for MQTT | / |
+| `ingress.mqtt.pathType` | Ingress pathType for MQTT | `ImplementationSpecific` |
+| `ingress.mqtt.hosts` | Ingress hosts for MQTT | mqtt.emqx.local |
+| `ingress.mqtt.tls` | Ingress tls for MQTT | [] |
+| `ingress.mqtt.annotations` | Ingress annotations for MQTT | {} |
+| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | |
 | `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false |
 | `metrics.type` | Now we only supported "prometheus" | "prometheus" |
 | `ssl.enabled` | Enable SSL support | false |
diff --git a/deploy/charts/emqx/templates/ingress.yaml b/deploy/charts/emqx/templates/ingress.yaml
index b6f496d88..29bac213d 100644
--- a/deploy/charts/emqx/templates/ingress.yaml
+++ b/deploy/charts/emqx/templates/ingress.yaml
@@ -48,3 +48,53 @@ spec:
   {{- end }}
 ---
 {{- end }}
+{{- if .Values.ingress.mqtt.enabled -}}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ printf "%s-%s" (include "emqx.fullname" .) "mqtt" }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- if .Values.ingress.mqtt.annotations }}
+  annotations:
+    {{- toYaml .Values.ingress.mqtt.annotations | nindent 4 }}
+  {{- end }}
+spec:
+{{- if and .Values.ingress.mqtt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingress.mqtt.ingressClassName }}
+{{- end }}
+  rules:
+  {{- range $host := .Values.ingress.mqtt.hosts }}
+  - host: {{ $host }}
+    http:
+      paths:
+      - path: {{ $.Values.ingress.mqtt.path | default "/" }}
+        {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+        pathType: {{ $.Values.ingress.mqtt.pathType | default "ImplementationSpecific" }}
+        {{- end }}
+        backend:
+          {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ include "emqx.fullname" $ }}
+            port:
+              number: {{ $.Values.service.mqtt }}
+          {{- else }}
+          serviceName: {{ include "emqx.fullname" $ }}
+          servicePort: {{ $.Values.service.mqtt }}
+          {{- end }}
+  {{- end -}}
+  {{- if .Values.ingress.mqtt.tls }}
+  tls:
+    {{- toYaml .Values.ingress.mqtt.tls | nindent 4 }}
+  {{- end }}
+---
+{{- end }}
diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index c737c8808..e3cb4fc25 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -189,6 +189,20 @@ ingress:
     hosts:
     - dashboard.emqx.local
     tls: []
+  ## ingress for MQTT
+  mqtt:
+    enabled: false
+    # ingressClassName: haproxy
+    annotations: {}
+      # kubernetes.io/ingress.class: haproxy
+      # kubernetes.io/tls-acme: "true"
+      # haproxy-ingress.github.io/tcp-service-port: "8883"
+      # haproxy-ingress.github.io/proxy-protocol: "v2"
+    path: /
+    pathType: ImplementationSpecific
+    hosts:
+    - mqtt.emqx.local
+    tls: []
 
 podSecurityContext:
   enabled: true

From c7f535abc7b1eb3c90bdc6afd0338b2d0ad56df0 Mon Sep 17 00:00:00 2001
From: Andrew Mayorov <encube.ul@gmail.com>
Date: Fri, 17 Feb 2023 11:04:24 +0300
Subject: [PATCH 044/144] chore: bump OTP-24 image version to 24.3.4.2-2

---
 .ci/docker-compose-file/docker-compose-kafka.yaml   |  2 +-
 .ci/docker-compose-file/docker-compose.yaml         |  2 +-
 .github/actions/package-macos/action.yaml           |  2 +-
 .github/workflows/build_and_push_docker_images.yaml |  6 +++---
 .github/workflows/build_packages.yaml               | 12 ++++++------
 .github/workflows/build_slim_packages.yaml          | 10 +++++-----
 .github/workflows/check_deps_integrity.yaml         |  2 +-
 .github/workflows/code_style_check.yaml             |  2 +-
 .github/workflows/elixir_apps_check.yaml            |  2 +-
 .github/workflows/elixir_deps_check.yaml            |  2 +-
 .github/workflows/elixir_release.yml                |  2 +-
 .github/workflows/release.yaml                      |  2 +-
 .github/workflows/run_emqx_app_tests.yaml           |  4 ++--
 .github/workflows/run_fvt_tests.yaml                | 10 +++++-----
 .github/workflows/run_relup_tests.yaml              |  2 +-
 .github/workflows/run_test_cases.yaml               | 10 +++++-----
 .tool-versions                                      |  2 +-
 Makefile                                            |  2 +-
 deploy/docker/Dockerfile                            |  2 +-
 scripts/buildx.sh                                   |  4 ++--
 scripts/relup-test/run-relup-lux.sh                 |  4 ++--
 scripts/relup-test/start-relup-test-cluster.sh      |  2 +-
 22 files changed, 44 insertions(+), 44 deletions(-)

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
index 9662b174d..e3ade50a1 100644
--- a/.ci/docker-compose-file/docker-compose-kafka.yaml
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -19,7 +19,7 @@ services:
     command: /bin/generate-certs.sh
   kdc:
     hostname: kdc.emqx.net
-    image:  ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04
+    image:  ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04
     container_name: kdc.emqx.net
     networks:
       emqx_bridge:
diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml
index b55b3196f..ff330872d 100644
--- a/.ci/docker-compose-file/docker-compose.yaml
+++ b/.ci/docker-compose-file/docker-compose.yaml
@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04}
+    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04}
     env_file:
       - conf.env
     environment:
diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml
index 95915dd7d..49d9b3dbf 100644
--- a/.github/actions/package-macos/action.yaml
+++ b/.github/actions/package-macos/action.yaml
@@ -3,7 +3,7 @@ inputs:
   profile: # emqx, emqx-enterprise
     required: true
     type: string
-  otp: # 25.1.2-2, 24.3.4.2-1
+  otp: # 25.1.2-2, 24.3.4.2-2
     required: true
     type: string
   os:
diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml
index b21683dc7..c612d2d5f 100644
--- a/.github/workflows/build_and_push_docker_images.yaml
+++ b/.github/workflows/build_and_push_docker_images.yaml
@@ -25,7 +25,7 @@ jobs:
   prepare:
     runs-on: ubuntu-20.04
     # prepare source with any OTP version, no need for a matrix
-    container: "ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04"
 
     outputs:
       PROFILE: ${{ steps.get_profile.outputs.PROFILE }}
@@ -125,9 +125,9 @@ jobs:
         # NOTE: 'otp' and 'elixir' are to configure emqx-builder image
         #       only support latest otp and elixir, not a matrix
         builder:
-          - 5.0-27 # update to latest
+          - 5.0-28 # update to latest
         otp:
-          - 24.3.4.2-1 # switch to 25 once ready to release 5.1
+          - 24.3.4.2-2 # switch to 25 once ready to release 5.1
         elixir:
           - 'no_elixir'
           - '1.13.4' # update to latest
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 6abb36d86..b23e2c604 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -23,7 +23,7 @@ on:
 jobs:
   prepare:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04
     outputs:
       BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
       IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
@@ -150,7 +150,7 @@ jobs:
         profile:
           - ${{ needs.prepare.outputs.BUILD_PROFILE }}
         otp:
-          - 24.3.4.2-1
+          - 24.3.4.2-2
         os:
           - macos-11
           - macos-12-arm64
@@ -201,7 +201,7 @@ jobs:
         profile:
           - ${{ needs.prepare.outputs.BUILD_PROFILE }}
         otp:
-          - 24.3.4.2-1
+          - 24.3.4.2-2
         arch:
           - amd64
           - arm64
@@ -218,7 +218,7 @@ jobs:
           - aws-arm64
           - ubuntu-20.04
         builder:
-          - 5.0-27
+          - 5.0-28
         elixir:
           - 1.13.4
         exclude:
@@ -232,7 +232,7 @@ jobs:
             arch: amd64
             os: ubuntu22.04
             build_machine: ubuntu-22.04
-            builder: 5.0-27
+            builder: 5.0-28
             elixir: 1.13.4
             release_with: elixir
           - profile: emqx
@@ -240,7 +240,7 @@ jobs:
             arch: amd64
             os: amzn2
             build_machine: ubuntu-22.04
-            builder: 5.0-27
+            builder: 5.0-28
             elixir: 1.13.4
             release_with: elixir
 
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index c7b5b0c83..434d1e790 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -29,13 +29,13 @@ jobs:
       fail-fast: false
       matrix:
         profile:
-        - ["emqx", "24.3.4.2-1", "el7"]
-        - ["emqx", "24.3.4.2-1", "ubuntu20.04"]
+        - ["emqx", "24.3.4.2-2", "el7"]
+        - ["emqx", "24.3.4.2-2", "ubuntu20.04"]
         - ["emqx", "25.1.2-2", "ubuntu22.04"]
-        - ["emqx-enterprise", "24.3.4.2-1", "ubuntu20.04"]
+        - ["emqx-enterprise", "24.3.4.2-2", "ubuntu20.04"]
         - ["emqx-enterprise", "25.1.2-2", "ubuntu22.04"]
         builder:
-        - 5.0-27
+        - 5.0-28
         elixir:
         - 1.13.4
 
@@ -128,7 +128,7 @@ jobs:
         - emqx
         - emqx-enterprise
         otp:
-        - 24.3.4.2-1
+        - 24.3.4.2-2
         os:
         - macos-11
         - macos-12-arm64
diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml
index 6d4f53dbc..ff41a4e86 100644
--- a/.github/workflows/check_deps_integrity.yaml
+++ b/.github/workflows/check_deps_integrity.yaml
@@ -5,7 +5,7 @@ on: [pull_request, push]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-25.1.2-2-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v3
diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml
index 38e3a5b8b..393da4dbd 100644
--- a/.github/workflows/code_style_check.yaml
+++ b/.github/workflows/code_style_check.yaml
@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   code_style_check:
     runs-on: ubuntu-20.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-25.1.2-2-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04"
     steps:
       - uses: actions/checkout@v3
         with:
diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml
index 892cd3a05..744618680 100644
--- a/.github/workflows/elixir_apps_check.yaml
+++ b/.github/workflows/elixir_apps_check.yaml
@@ -8,7 +8,7 @@ jobs:
   elixir_apps_check:
     runs-on: ubuntu-latest
     # just use the latest builder
-    container: "ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-25.1.2-2-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04"
 
     strategy:
       fail-fast: false
diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml
index 40d70a902..5f5450cab 100644
--- a/.github/workflows/elixir_deps_check.yaml
+++ b/.github/workflows/elixir_deps_check.yaml
@@ -7,7 +7,7 @@ on: [pull_request, push]
 jobs:
   elixir_deps_check:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-25.1.2-2-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04
 
     steps:
       - name: Checkout
diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml
index b93b3d63c..40bb83636 100644
--- a/.github/workflows/elixir_release.yml
+++ b/.github/workflows/elixir_release.yml
@@ -17,7 +17,7 @@ jobs:
         profile:
           - emqx
           - emqx-enterprise
-    container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-25.1.2-2-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index df26778b4..fe21545ca 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -54,7 +54,7 @@ jobs:
           OUTPUT_DIR=${{ steps.profile.outputs.s3dir }}
           aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages
           cd packages
-          DEFAULT_BEAM_PLATFORM='otp24.3.4.2-1'
+          DEFAULT_BEAM_PLATFORM='otp24.3.4.2-2'
           # all packages including full-name and default-name are uploaded to s3
           # but we only upload default-name packages (and elixir) as github artifacts
           # so we rename (overwrite) non-default packages before uploading
diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml
index 0b3f21b4a..cf6e1bdff 100644
--- a/.github/workflows/run_emqx_app_tests.yaml
+++ b/.github/workflows/run_emqx_app_tests.yaml
@@ -12,9 +12,9 @@ jobs:
     strategy:
       matrix:
         builder:
-          - 5.0-27
+          - 5.0-28
         otp:
-          - 24.3.4.2-1
+          - 24.3.4.2-2
           - 25.1.2-2
         # no need to use more than 1 version of Elixir, since tests
         # run using only Erlang code.  This is needed just to specify
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 994326d23..4ef634d91 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -16,7 +16,7 @@ jobs:
   prepare:
     runs-on: ubuntu-20.04
     # prepare source with any OTP version, no need for a matrix
-    container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-debian11
+    container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11
 
     steps:
       - uses: actions/checkout@v3
@@ -49,9 +49,9 @@ jobs:
         os:
           - ["debian11", "debian:11-slim"]
         builder:
-          - 5.0-27
+          - 5.0-28
         otp:
-          - 24.3.4.2-1
+          - 24.3.4.2-2
         elixir:
           - 1.13.4
         arch:
@@ -122,9 +122,9 @@ jobs:
         os:
         - ["debian11", "debian:11-slim"]
         builder:
-        - 5.0-27
+        - 5.0-28
         otp:
-        - 24.3.4.2-1
+        - 24.3.4.2-2
         elixir:
         - 1.13.4
         arch:
diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml
index d8d64d909..4d03878de 100644
--- a/.github/workflows/run_relup_tests.yaml
+++ b/.github/workflows/run_relup_tests.yaml
@@ -15,7 +15,7 @@ concurrency:
 jobs:
   relup_test_plan:
     runs-on: ubuntu-20.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04"
     outputs:
       CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
       OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }}
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 0abca58c3..cdbef9a8b 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -30,13 +30,13 @@ jobs:
               MATRIX="$(echo "${APPS}" | jq -c '
                 [
                   (.[] | select(.profile == "emqx") | . + {
-                    builder: "5.0-27",
+                    builder: "5.0-28",
                     otp: "25.1.2-2",
                     elixir: "1.13.4"
                   }),
                   (.[] | select(.profile == "emqx-enterprise") | . + {
-                    builder: "5.0-27",
-                    otp: ["24.3.4.2-1", "25.1.2-2"][],
+                    builder: "5.0-28",
+                    otp: ["24.3.4.2-2", "25.1.2-2"][],
                     elixir: "1.13.4"
                   })
                 ]
@@ -224,12 +224,12 @@ jobs:
         - ct
         - ct_docker
       runs-on: ubuntu-20.04
-      container: "ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-ubuntu20.04"
+      container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04"
       steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: source-emqx-enterprise-24.3.4.2-1
+          name: source-emqx-enterprise-24.3.4.2-2
           path: .
       - name: unzip source code
         run: unzip -q source.zip
diff --git a/.tool-versions b/.tool-versions
index 0f7c9b32e..dcf5945a8 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1,2 +1,2 @@
-erlang 24.3.4.2-1
+erlang 24.3.4.2-2
 elixir 1.13.4-otp-24
diff --git a/Makefile b/Makefile
index 2bfdffdfb..5e9307105 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@ REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
 export EMQX_RELUP ?= true
-export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11
+export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 4a00c68fb..d424fb027 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -1,4 +1,4 @@
-ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11
+ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11
 ARG RUN_FROM=debian:11-slim
 FROM ${BUILD_FROM} AS builder
 
diff --git a/scripts/buildx.sh b/scripts/buildx.sh
index ca7f812f6..4f12e0abc 100755
--- a/scripts/buildx.sh
+++ b/scripts/buildx.sh
@@ -9,7 +9,7 @@
 
 ## example:
 ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
-##     --builder ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11
+##     --builder ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11
 
 set -euo pipefail
 
@@ -24,7 +24,7 @@ help() {
     echo "--arch amd64|arm64:        Target arch to build the EMQX package for"
     echo "--src_dir <SRC_DIR>:       EMQX source code in this dir, default to PWD"
     echo "--builder <BUILDER>:       Builder image to pull"
-    echo "                           E.g. ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11"
+    echo "                           E.g. ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11"
 }
 
 while [ "$#" -gt 0 ]; do
diff --git a/scripts/relup-test/run-relup-lux.sh b/scripts/relup-test/run-relup-lux.sh
index cf30db850..570e58340 100755
--- a/scripts/relup-test/run-relup-lux.sh
+++ b/scripts/relup-test/run-relup-lux.sh
@@ -45,8 +45,8 @@ fi
 # From now on, no need for the v|e prefix
 OLD_VSN="${old_vsn#[e|v]}"
 
-OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-1-ubuntu20.04-amd64.tar.gz"
-CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-1-ubuntu20.04-amd64.tar.gz"
+OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-2-ubuntu20.04-amd64.tar.gz"
+CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-2-ubuntu20.04-amd64.tar.gz"
 
 if [ ! -f "$OLD_PKG" ]; then
     echo "$OLD_PKG not found"
diff --git a/scripts/relup-test/start-relup-test-cluster.sh b/scripts/relup-test/start-relup-test-cluster.sh
index c9e3edbd9..385137dc7 100755
--- a/scripts/relup-test/start-relup-test-cluster.sh
+++ b/scripts/relup-test/start-relup-test-cluster.sh
@@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET"
 BENCH="bench.$NET"
 COOKIE='this-is-a-secret'
 ## Erlang image is needed to run webhook server and emqtt-bench
-ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04"
+ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04"
 # builder has emqtt-bench installed
 BENCH_IMAGE="$ERLANG_IMAGE"
 

From 794bedef9f1ece56e17218e4fd9415af05e2f0c8 Mon Sep 17 00:00:00 2001
From: ieQu1 <99872536+ieQu1@users.noreply.github.com>
Date: Wed, 8 Feb 2023 09:58:39 +0100
Subject: [PATCH 045/144] fix(emqx_ctl): Start CLI before ekka

---
 apps/emqx/src/emqx_kernel_sup.erl             |  1 -
 apps/emqx_conf/src/emqx_conf.app.src          |  2 +-
 apps/emqx_ctl/README.md                       |  4 +++
 apps/emqx_ctl/rebar.config                    |  2 ++
 apps/emqx_ctl/src/emqx_ctl.app.src            | 15 +++++++++
 apps/{emqx => emqx_ctl}/src/emqx_ctl.erl      | 27 +++++++++------
 apps/emqx_ctl/src/emqx_ctl_app.erl            | 18 ++++++++++
 apps/emqx_ctl/src/emqx_ctl_sup.erl            | 33 +++++++++++++++++++
 .../test/emqx_ctl_SUITE.erl                   |  6 ++--
 .../emqx_dashboard/src/emqx_dashboard.app.src |  2 +-
 apps/emqx_gateway/src/emqx_gateway.app.src    |  4 +--
 apps/emqx_machine/src/emqx_machine.app.src    |  4 +--
 apps/emqx_machine/src/emqx_machine.erl        |  1 +
 .../src/emqx_management.app.src               |  2 +-
 apps/emqx_management/src/emqx_mgmt_app.erl    |  4 +--
 apps/emqx_modules/src/emqx_modules.app.src    |  4 +--
 apps/emqx_retainer/src/emqx_retainer.app.src  |  2 +-
 .../src/emqx_rule_engine.app.src              |  2 +-
 lib-ee/emqx_license/src/emqx_license.app.src  |  2 +-
 19 files changed, 105 insertions(+), 30 deletions(-)
 create mode 100644 apps/emqx_ctl/README.md
 create mode 100644 apps/emqx_ctl/rebar.config
 create mode 100644 apps/emqx_ctl/src/emqx_ctl.app.src
 rename apps/{emqx => emqx_ctl}/src/emqx_ctl.erl (92%)
 create mode 100644 apps/emqx_ctl/src/emqx_ctl_app.erl
 create mode 100644 apps/emqx_ctl/src/emqx_ctl_sup.erl
 rename apps/{emqx => emqx_ctl}/test/emqx_ctl_SUITE.erl (95%)

diff --git a/apps/emqx/src/emqx_kernel_sup.erl b/apps/emqx/src/emqx_kernel_sup.erl
index 21ed8576a..a69674de8 100644
--- a/apps/emqx/src/emqx_kernel_sup.erl
+++ b/apps/emqx/src/emqx_kernel_sup.erl
@@ -35,7 +35,6 @@ init([]) ->
             child_spec(emqx_hooks, worker),
             child_spec(emqx_stats, worker),
             child_spec(emqx_metrics, worker),
-            child_spec(emqx_ctl, worker),
             child_spec(emqx_authn_authz_metrics_sup, supervisor)
         ]
     }}.
diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src
index beac051a1..a53509593 100644
--- a/apps/emqx_conf/src/emqx_conf.app.src
+++ b/apps/emqx_conf/src/emqx_conf.app.src
@@ -3,7 +3,7 @@
     {vsn, "0.1.12"},
     {registered, []},
     {mod, {emqx_conf_app, []}},
-    {applications, [kernel, stdlib]},
+    {applications, [kernel, stdlib, emqx_ctl]},
     {env, []},
     {modules, []}
 ]}.
diff --git a/apps/emqx_ctl/README.md b/apps/emqx_ctl/README.md
new file mode 100644
index 000000000..a91342606
--- /dev/null
+++ b/apps/emqx_ctl/README.md
@@ -0,0 +1,4 @@
+emqx_ctl
+=====
+
+Backend module for `emqx_ctl` command.
diff --git a/apps/emqx_ctl/rebar.config b/apps/emqx_ctl/rebar.config
new file mode 100644
index 000000000..2656fd554
--- /dev/null
+++ b/apps/emqx_ctl/rebar.config
@@ -0,0 +1,2 @@
+{erl_opts, [debug_info]}.
+{deps, []}.
diff --git a/apps/emqx_ctl/src/emqx_ctl.app.src b/apps/emqx_ctl/src/emqx_ctl.app.src
new file mode 100644
index 000000000..9de598a89
--- /dev/null
+++ b/apps/emqx_ctl/src/emqx_ctl.app.src
@@ -0,0 +1,15 @@
+{application, emqx_ctl, [
+    {description, "Backend for emqx_ctl script"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {mod, {emqx_ctl_app, []}},
+    {applications, [
+        kernel,
+        stdlib
+    ]},
+    {env, []},
+    {modules, []},
+
+    {licenses, ["Apache-2.0"]},
+    {links, []}
+]}.
diff --git a/apps/emqx/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl
similarity index 92%
rename from apps/emqx/src/emqx_ctl.erl
rename to apps/emqx_ctl/src/emqx_ctl.erl
index 53eb5b888..a9aad0259 100644
--- a/apps/emqx/src/emqx_ctl.erl
+++ b/apps/emqx_ctl/src/emqx_ctl.erl
@@ -18,8 +18,7 @@
 
 -behaviour(gen_server).
 
--include("types.hrl").
--include("logger.hrl").
+-include_lib("kernel/include/logger.hrl").
 
 -export([start_link/0, stop/0]).
 
@@ -70,7 +69,7 @@
 -define(SERVER, ?MODULE).
 -define(CMD_TAB, emqx_command).
 
--spec start_link() -> startlink_ret().
+-spec start_link() -> {ok, pid()}.
 start_link() ->
     gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
 
@@ -103,7 +102,7 @@ cast(Msg) -> gen_server:cast(?SERVER, Msg).
 run_command([]) ->
     run_command(help, []);
 run_command([Cmd | Args]) ->
-    case emqx_misc:safe_to_existing_atom(Cmd) of
+    case safe_to_existing_atom(Cmd) of
         {ok, Cmd1} ->
             run_command(Cmd1, Args);
         _ ->
@@ -122,7 +121,7 @@ run_command(Cmd, Args) when is_atom(Cmd) ->
                 ok
             catch
                 _:Reason:Stacktrace ->
-                    ?SLOG(error, #{
+                    ?LOG_ERROR(#{
                         msg => "ctl_command_crashed",
                         stacktrace => Stacktrace,
                         reason => Reason
@@ -220,7 +219,7 @@ format_usage(CmdParams, Desc, Width) ->
 %%--------------------------------------------------------------------
 
 init([]) ->
-    ok = emqx_tables:new(?CMD_TAB, [protected, ordered_set]),
+    _ = ets:new(?CMD_TAB, [named_table, protected, ordered_set]),
     {ok, #state{seq = 0}}.
 
 handle_call({register_command, Cmd, MF, Opts}, _From, State = #state{seq = Seq}) ->
@@ -229,23 +228,23 @@ handle_call({register_command, Cmd, MF, Opts}, _From, State = #state{seq = Seq})
             ets:insert(?CMD_TAB, {{Seq, Cmd}, MF, Opts}),
             {reply, ok, next_seq(State)};
         [[OriginSeq] | _] ->
-            ?SLOG(warning, #{msg => "CMD_overidden", cmd => Cmd, mf => MF}),
+            ?LOG_WARNING(#{msg => "CMD_overidden", cmd => Cmd, mf => MF}),
             true = ets:insert(?CMD_TAB, {{OriginSeq, Cmd}, MF, Opts}),
             {reply, ok, State}
     end;
 handle_call(Req, _From, State) ->
-    ?SLOG(error, #{msg => "unexpected_call", call => Req}),
+    ?LOG_ERROR(#{msg => "unexpected_call", call => Req}),
     {reply, ignored, State}.
 
 handle_cast({unregister_command, Cmd}, State) ->
     ets:match_delete(?CMD_TAB, {{'_', Cmd}, '_', '_'}),
     noreply(State);
 handle_cast(Msg, State) ->
-    ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
+    ?LOG_ERROR(#{msg => "unexpected_cast", cast => Msg}),
     noreply(State).
 
 handle_info(Info, State) ->
-    ?SLOG(error, #{msg => "unexpected_info", info => Info}),
+    ?LOG_ERROR(#{msg => "unexpected_info", info => Info}),
     noreply(State).
 
 terminate(_Reason, _State) ->
@@ -272,3 +271,11 @@ zip_cmd([X | Xs], [Y | Ys]) -> [{X, Y} | zip_cmd(Xs, Ys)];
 zip_cmd([X | Xs], []) -> [{X, ""} | zip_cmd(Xs, [])];
 zip_cmd([], [Y | Ys]) -> [{"", Y} | zip_cmd([], Ys)];
 zip_cmd([], []) -> [].
+
+safe_to_existing_atom(Str) ->
+    try
+        {ok, list_to_existing_atom(Str)}
+    catch
+        _:badarg ->
+            undefined
+    end.
diff --git a/apps/emqx_ctl/src/emqx_ctl_app.erl b/apps/emqx_ctl/src/emqx_ctl_app.erl
new file mode 100644
index 000000000..803ba90d3
--- /dev/null
+++ b/apps/emqx_ctl/src/emqx_ctl_app.erl
@@ -0,0 +1,18 @@
+%%%-------------------------------------------------------------------
+%% @doc emqx_ctl public API
+%% @end
+%%%-------------------------------------------------------------------
+
+-module(emqx_ctl_app).
+
+-behaviour(application).
+
+-export([start/2, stop/1]).
+
+start(_StartType, _StartArgs) ->
+    emqx_ctl_sup:start_link().
+
+stop(_State) ->
+    ok.
+
+%% internal functions
diff --git a/apps/emqx_ctl/src/emqx_ctl_sup.erl b/apps/emqx_ctl/src/emqx_ctl_sup.erl
new file mode 100644
index 000000000..21086e424
--- /dev/null
+++ b/apps/emqx_ctl/src/emqx_ctl_sup.erl
@@ -0,0 +1,33 @@
+%%%-------------------------------------------------------------------
+%% @doc emqx_ctl top level supervisor.
+%% @end
+%%%-------------------------------------------------------------------
+
+-module(emqx_ctl_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/0]).
+
+-export([init/1]).
+
+-define(SERVER, ?MODULE).
+
+start_link() ->
+    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
+
+init([]) ->
+    SupFlags = #{
+        strategy => one_for_all,
+        intensity => 0,
+        period => 1
+    },
+    ChildSpecs = [
+        #{
+            id => emqx_ctl,
+            start => {emqx_ctl, start_link, []},
+            type => worker,
+            restart => permanent
+        }
+    ],
+    {ok, {SupFlags, ChildSpecs}}.
diff --git a/apps/emqx/test/emqx_ctl_SUITE.erl b/apps/emqx_ctl/test/emqx_ctl_SUITE.erl
similarity index 95%
rename from apps/emqx/test/emqx_ctl_SUITE.erl
rename to apps/emqx_ctl/test/emqx_ctl_SUITE.erl
index 03f7b2148..46d9008e8 100644
--- a/apps/emqx/test/emqx_ctl_SUITE.erl
+++ b/apps/emqx_ctl/test/emqx_ctl_SUITE.erl
@@ -22,12 +22,10 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
-all() -> emqx_common_test_helpers:all(?MODULE).
+all() -> [t_reg_unreg_command, t_run_commands, t_print, t_usage, t_unexpected].
 
 init_per_suite(Config) ->
-    %% ensure stopped, this suite tests emqx_ctl process independently
-    application:stop(emqx),
-    ok = emqx_logger:set_log_level(emergency),
+    application:stop(emqx_ctl),
     Config.
 
 end_per_suite(_Config) ->
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src
index b6c95ca97..56fc16286 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src
@@ -5,7 +5,7 @@
     {vsn, "5.0.13"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
-    {applications, [kernel, stdlib, mnesia, minirest, emqx]},
+    {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]},
     {mod, {emqx_dashboard_app, []}},
     {env, []},
     {licenses, ["Apache-2.0"]},
diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src
index c5dd76f19..787af7429 100644
--- a/apps/emqx_gateway/src/emqx_gateway.app.src
+++ b/apps/emqx_gateway/src/emqx_gateway.app.src
@@ -1,10 +1,10 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway, [
     {description, "The Gateway management application"},
-    {vsn, "0.1.11"},
+    {vsn, "0.1.12"},
     {registered, []},
     {mod, {emqx_gateway_app, []}},
-    {applications, [kernel, stdlib, grpc, emqx, emqx_authn]},
+    {applications, [kernel, stdlib, grpc, emqx, emqx_authn, emqx_ctl]},
     {env, []},
     {modules, []},
     {licenses, ["Apache 2.0"]},
diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src
index c805fdd25..5aef4f2bc 100644
--- a/apps/emqx_machine/src/emqx_machine.app.src
+++ b/apps/emqx_machine/src/emqx_machine.app.src
@@ -3,10 +3,10 @@
     {id, "emqx_machine"},
     {description, "The EMQX Machine"},
     % strict semver, bump manually!
-    {vsn, "0.1.3"},
+    {vsn, "0.1.4"},
     {modules, []},
     {registered, []},
-    {applications, [kernel, stdlib]},
+    {applications, [kernel, stdlib, emqx_ctl]},
     {mod, {emqx_machine_app, []}},
     {env, []},
     {licenses, ["Apache-2.0"]},
diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl
index ec0aff55b..9dc3fdc54 100644
--- a/apps/emqx_machine/src/emqx_machine.erl
+++ b/apps/emqx_machine/src/emqx_machine.erl
@@ -29,6 +29,7 @@
 
 %% @doc EMQX boot entrypoint.
 start() ->
+    emqx_mgmt_cli:load(),
     case os:type() of
         {win32, nt} ->
             ok;
diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index 8a10f1b6b..08de7b670 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -5,7 +5,7 @@
     {vsn, "5.0.15"},
     {modules, []},
     {registered, [emqx_management_sup]},
-    {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},
+    {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]},
     {mod, {emqx_mgmt_app, []}},
     {env, []},
     {licenses, ["Apache-2.0"]},
diff --git a/apps/emqx_management/src/emqx_mgmt_app.erl b/apps/emqx_management/src/emqx_mgmt_app.erl
index 137f4502c..b4cf9091a 100644
--- a/apps/emqx_management/src/emqx_mgmt_app.erl
+++ b/apps/emqx_management/src/emqx_mgmt_app.erl
@@ -31,9 +31,7 @@ start(_Type, _Args) ->
     ok = mria_rlog:wait_for_shards([?MANAGEMENT_SHARD], infinity),
     case emqx_mgmt_auth:init_bootstrap_file() of
         ok ->
-            {ok, Sup} = emqx_mgmt_sup:start_link(),
-            ok = emqx_mgmt_cli:load(),
-            {ok, Sup};
+            emqx_mgmt_sup:start_link();
         {error, Reason} ->
             {error, Reason}
     end.
diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src
index 20f8a76fc..60d36d673 100644
--- a/apps/emqx_modules/src/emqx_modules.app.src
+++ b/apps/emqx_modules/src/emqx_modules.app.src
@@ -1,9 +1,9 @@
 %% -*- mode: erlang -*-
 {application, emqx_modules, [
     {description, "EMQX Modules"},
-    {vsn, "5.0.9"},
+    {vsn, "5.0.10"},
     {modules, []},
-    {applications, [kernel, stdlib, emqx]},
+    {applications, [kernel, stdlib, emqx, emqx_ctl]},
     {mod, {emqx_modules_app, []}},
     {registered, [emqx_modules_sup]},
     {env, []}
diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src
index d151ad4e7..fb2a991a7 100644
--- a/apps/emqx_retainer/src/emqx_retainer.app.src
+++ b/apps/emqx_retainer/src/emqx_retainer.app.src
@@ -5,7 +5,7 @@
     {vsn, "5.0.9"},
     {modules, []},
     {registered, [emqx_retainer_sup]},
-    {applications, [kernel, stdlib, emqx]},
+    {applications, [kernel, stdlib, emqx, emqx_ctl]},
     {mod, {emqx_retainer_app, []}},
     {env, []},
     {licenses, ["Apache-2.0"]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index ee1544223..4c3d67bc9 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -5,7 +5,7 @@
     {vsn, "5.0.8"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
-    {applications, [kernel, stdlib, rulesql, getopt]},
+    {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]},
     {mod, {emqx_rule_engine_app, []}},
     {env, []},
     {licenses, ["Apache-2.0"]},
diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src
index 93ae665d5..2dc4cd89e 100644
--- a/lib-ee/emqx_license/src/emqx_license.app.src
+++ b/lib-ee/emqx_license/src/emqx_license.app.src
@@ -3,6 +3,6 @@
     {vsn, "5.0.5"},
     {modules, []},
     {registered, [emqx_license_sup]},
-    {applications, [kernel, stdlib]},
+    {applications, [kernel, stdlib, emqx_ctl]},
     {mod, {emqx_license_app, []}}
 ]}.

From d9554c36ace515cd411ba293b80e01af9d373be4 Mon Sep 17 00:00:00 2001
From: ieQu1 <99872536+ieQu1@users.noreply.github.com>
Date: Thu, 16 Feb 2023 17:19:58 +0100
Subject: [PATCH 046/144] chore: Bump umbrealla app versions

---
 apps/emqx_conf/src/emqx_conf.app.src               | 2 +-
 apps/emqx_dashboard/src/emqx_dashboard.app.src     | 2 +-
 apps/emqx_retainer/src/emqx_retainer.app.src       | 2 +-
 apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +-
 lib-ee/emqx_license/src/emqx_license.app.src       | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src
index a53509593..1d04dc362 100644
--- a/apps/emqx_conf/src/emqx_conf.app.src
+++ b/apps/emqx_conf/src/emqx_conf.app.src
@@ -1,6 +1,6 @@
 {application, emqx_conf, [
     {description, "EMQX configuration management"},
-    {vsn, "0.1.12"},
+    {vsn, "0.1.13"},
     {registered, []},
     {mod, {emqx_conf_app, []}},
     {applications, [kernel, stdlib, emqx_ctl]},
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src
index 56fc16286..44260cbe1 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src
@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.13"},
+    {vsn, "5.0.14"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]},
diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src
index fb2a991a7..8bdae6d7f 100644
--- a/apps/emqx_retainer/src/emqx_retainer.app.src
+++ b/apps/emqx_retainer/src/emqx_retainer.app.src
@@ -2,7 +2,7 @@
 {application, emqx_retainer, [
     {description, "EMQX Retainer"},
     % strict semver, bump manually!
-    {vsn, "5.0.9"},
+    {vsn, "5.0.10"},
     {modules, []},
     {registered, [emqx_retainer_sup]},
     {applications, [kernel, stdlib, emqx, emqx_ctl]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index 4c3d67bc9..06ed059a4 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -2,7 +2,7 @@
 {application, emqx_rule_engine, [
     {description, "EMQX Rule Engine"},
     % strict semver, bump manually!
-    {vsn, "5.0.8"},
+    {vsn, "5.0.9"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
     {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]},
diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src
index 2dc4cd89e..fdc701369 100644
--- a/lib-ee/emqx_license/src/emqx_license.app.src
+++ b/lib-ee/emqx_license/src/emqx_license.app.src
@@ -1,6 +1,6 @@
 {application, emqx_license, [
     {description, "EMQX License"},
-    {vsn, "5.0.5"},
+    {vsn, "5.0.6"},
     {modules, []},
     {registered, [emqx_license_sup]},
     {applications, [kernel, stdlib, emqx_ctl]},

From 843ed464d57878f1871ef4b1c0069bb124e67c53 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Fri, 17 Feb 2023 10:47:57 +0100
Subject: [PATCH 047/144] fix: remove all mentions of UDP support for InfluxDB
 EE bridge

The InfluxDB EE bridge doesn't support the UDP protocol, but it is
defined in the schema. This commit removes all such traces in the
schema.
---
 .../src/emqx_ee_bridge_influxdb.erl           | 22 +------------------
 .../i18n/emqx_ee_connector_influxdb.conf      | 14 ++----------
 .../src/emqx_ee_connector_influxdb.erl        | 16 +-------------
 3 files changed, 4 insertions(+), 48 deletions(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
index 6d96e3883..14f53b5e7 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl
@@ -31,12 +31,6 @@
 
 conn_bridge_examples(Method) ->
     [
-        #{
-            <<"influxdb_udp">> => #{
-                summary => <<"InfluxDB UDP Bridge">>,
-                value => values("influxdb_udp", Method)
-            }
-        },
         #{
             <<"influxdb_api_v1">> => #{
                 summary => <<"InfluxDB HTTP API V1 Bridge">>,
@@ -71,12 +65,6 @@ values("influxdb_api_v1", post) ->
         server => <<"127.0.0.1:8086">>
     },
     values(common, "influxdb_api_v1", SupportUint, TypeOpts);
-values("influxdb_udp", post) ->
-    SupportUint = <<>>,
-    TypeOpts = #{
-        server => <<"127.0.0.1:8089">>
-    },
-    values(common, "influxdb_udp", SupportUint, TypeOpts);
 values(Protocol, put) ->
     values(Protocol, post).
 
@@ -106,26 +94,20 @@ namespace() -> "bridge_influxdb".
 
 roots() -> [].
 
-fields("post_udp") ->
-    method_fileds(post, influxdb_udp);
 fields("post_api_v1") ->
     method_fileds(post, influxdb_api_v1);
 fields("post_api_v2") ->
     method_fileds(post, influxdb_api_v2);
-fields("put_udp") ->
-    method_fileds(put, influxdb_udp);
 fields("put_api_v1") ->
     method_fileds(put, influxdb_api_v1);
 fields("put_api_v2") ->
     method_fileds(put, influxdb_api_v2);
-fields("get_udp") ->
-    method_fileds(get, influxdb_udp);
 fields("get_api_v1") ->
     method_fileds(get, influxdb_api_v1);
 fields("get_api_v2") ->
     method_fileds(get, influxdb_api_v2);
 fields(Type) when
-    Type == influxdb_udp orelse Type == influxdb_api_v1 orelse Type == influxdb_api_v2
+    Type == influxdb_api_v1 orelse Type == influxdb_api_v2
 ->
     influxdb_bridge_common_fields() ++
         connector_fields(Type).
@@ -164,8 +146,6 @@ desc("config") ->
     ?DESC("desc_config");
 desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
     ["Configuration for InfluxDB using `", string:to_upper(Method), "` method."];
-desc(influxdb_udp) ->
-    ?DESC(emqx_ee_connector_influxdb, "influxdb_udp");
 desc(influxdb_api_v1) ->
     ?DESC(emqx_ee_connector_influxdb, "influxdb_api_v1");
 desc(influxdb_api_v2) ->
diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
index 81ea39d49..03246c07d 100644
--- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
+++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf
@@ -28,24 +28,14 @@ The InfluxDB default port 8086 is used if `[:Port]` is not specified.
     }
     protocol {
         desc {
-                en: """InfluxDB's protocol. UDP or HTTP API or HTTP API V2."""
-                zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2。"""
+                en: """InfluxDB's protocol. HTTP API or HTTP API V2."""
+                zh: """InfluxDB 协议。HTTP API 或 HTTP API V2。"""
         }
         label {
                 en: """Protocol"""
                 zh: """协议"""
         }
     }
-    influxdb_udp {
-        desc {
-                en: """InfluxDB's UDP protocol."""
-                zh: """InfluxDB UDP 协议。"""
-        }
-        label {
-                en: """UDP Protocol"""
-                zh: """UDP 协议"""
-            }
-    }
     influxdb_api_v1 {
         desc {
                 en: """InfluxDB's protocol. Support InfluxDB v1.8 and before."""
diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
index f056f4af2..785ec5d07 100644
--- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
+++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl
@@ -145,7 +145,6 @@ roots() ->
         {config, #{
             type => hoconsc:union(
                 [
-                    hoconsc:ref(?MODULE, influxdb_udp),
                     hoconsc:ref(?MODULE, influxdb_api_v1),
                     hoconsc:ref(?MODULE, influxdb_api_v2)
                 ]
@@ -165,8 +164,6 @@ fields(common) ->
                 required => false, default => ms, desc => ?DESC("precision")
             })}
     ];
-fields(influxdb_udp) ->
-    fields(common);
 fields(influxdb_api_v1) ->
     fields(common) ++
         [
@@ -199,8 +196,6 @@ server() ->
 
 desc(common) ->
     ?DESC("common");
-desc(influxdb_udp) ->
-    ?DESC("influxdb_udp");
 desc(influxdb_api_v1) ->
     ?DESC("influxdb_api_v1");
 desc(influxdb_api_v2) ->
@@ -326,12 +321,7 @@ protocol_config(#{
         {bucket, str(Bucket)},
         {org, str(Org)},
         {token, Token}
-    ] ++ ssl_config(SSL);
-%% udp config
-protocol_config(_) ->
-    [
-        {protocol, udp}
-    ].
+    ] ++ ssl_config(SSL).
 
 ssl_config(#{enable := false}) ->
     [
@@ -659,10 +649,6 @@ desc_test_() ->
             {desc, _, _},
             desc(common)
         ),
-        ?_assertMatch(
-            {desc, _, _},
-            desc(influxdb_udp)
-        ),
         ?_assertMatch(
             {desc, _, _},
             desc(influxdb_api_v1)

From 4b8b3e1d98e94b60db343d55bf5ab873566a68fc Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 17 Feb 2023 11:26:28 +0100
Subject: [PATCH 048/144] chore: upgrade dashboards

For Opensource edition:
https://github.com/emqx/emqx-dashboard-web-new/releases/tag/v1.1.8

For Entperprise edition:
https://github.com/emqx/emqx-dashboard-web-new/releases/tag/e1.0.4-beta.3
---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 5e9307105..2cabaebcf 100644
--- a/Makefile
+++ b/Makefile
@@ -6,8 +6,8 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v1.1.7
-export EMQX_EE_DASHBOARD_VERSION ?= e1.0.3
+export EMQX_DASHBOARD_VERSION ?= v1.1.8
+export EMQX_EE_DASHBOARD_VERSION ?= e1.0.4-beta.3
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
 ifeq ($(OS),Windows_NT)

From fea310cf932cd25364f7b161e749db67652d8143 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 17 Feb 2023 13:15:22 +0100
Subject: [PATCH 049/144] test: use different loop back address in script which
 starts two nodes

---
 scripts/test/start-two-nodes-in-host.sh | 86 +++++++++++++------------
 1 file changed, 46 insertions(+), 40 deletions(-)

diff --git a/scripts/test/start-two-nodes-in-host.sh b/scripts/test/start-two-nodes-in-host.sh
index 7dbd6009e..417df54e5 100755
--- a/scripts/test/start-two-nodes-in-host.sh
+++ b/scripts/test/start-two-nodes-in-host.sh
@@ -13,51 +13,57 @@ set -euo pipefail
 cd -P -- "$(dirname -- "$0")/../../"
 
 DEFAULT_BOOT='./_build/emqx/rel/emqx/bin/emqx'
+
 BOOT1="${1:-$DEFAULT_BOOT}"
 BOOT2="${2:-$BOOT1}"
 
-DATA1="$(pwd)/tmp/emqx1/data"
-LOG1="$(pwd)/tmp/emqx1/log"
-DATA2="$(pwd)/tmp/emqx2/data"
-LOG2="$(pwd)/tmp/emqx2/log"
+export IP1='127.0.0.1'
+export IP2='127.0.0.2'
 
-mkdir -p "$DATA1" "$DATA2" "$LOG1" "$LOG2"
+# cannot use the same node name even IPs are different because Erlang distribution listens on 0.0.0.0
+NODE1="emqx1@$IP1"
+NODE2="emqx2@$IP2"
 
-echo "Stopping emqx1"
-env EMQX_NODE_NAME='emqx1@127.0.0.1' \
-    ./_build/emqx/rel/emqx/bin/emqx stop || true
+start_cmd() {
+    local index="$1"
+    local nodehome
+    nodehome="$(pwd)/tmp/emqx${index}"
+    [ "$index" -eq 1 ] && BOOT_SCRIPT="$BOOT1"
+    [ "$index" -eq 2 ] && BOOT_SCRIPT="$BOOT2"
+    mkdir -p "${nodehome}/data" "${nodehome}/log"
+    cat <<-EOF
+env DEBUG="${DEBUG:-0}" \
+EMQX_CLUSTER__STATIC__SEEDS="[\"$NODE1\",\"$NODE2\"]" \
+EMQX_CLUSTER__DISCOVERY_STRATEGY=static \
+EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL="${EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL:-debug}" \
+EMQX_LOG__FILE_HANDLERS__DEFAULT__FILE="${nodehome}/log/emqx.log" \
+EMQX_NODE_NAME="emqx${index}@\$IP${index}" \
+EMQX_NODE__COOKIE="${EMQX_NODE__COOKIE:-cookie1}" \
+EMQX_LOG_DIR="${nodehome}/log" \
+EMQX_NODE__DATA_DIR="${nodehome}/data" \
+EMQX_LISTENERS__TCP__DEFAULT__BIND="\$IP${index}:1883" \
+EMQX_LISTENERS__SSL__DEFAULT__BIND="\$IP${index}:8883" \
+EMQX_LISTENERS__WS__DEFAULT__BIND="\$IP${index}:8083" \
+EMQX_LISTENERS__WSS__DEFAULT__BIND="\$IP${index}:8084" \
+EMQX_DASHBOARD__LISTENERS__HTTP__BIND="\$IP${index}:18083" \
+$BOOT_SCRIPT start
+EOF
+}
 
-echo "Stopping emqx2"
-env EMQX_NODE_NAME='emqx2@127.0.0.1' \
-    ./_build/emqx/rel/emqx/bin/emqx stop || true
+echo "Stopping $NODE1"
+env EMQX_NODE_NAME="$NODE1" ./_build/emqx/rel/emqx/bin/emqx stop || true
+
+echo "Stopping $NODE2"
+env EMQX_NODE_NAME="$NODE2" ./_build/emqx/rel/emqx/bin/emqx stop || true
+
+start_one_node() {
+    local index="$1"
+    local cmd
+    cmd="$(start_cmd "$index" | envsubst)"
+    echo "$cmd"
+    eval "$cmd"
+}
 
 ## Fork-start node1, otherwise it'll keep waiting for node2 because we are using static cluster
-env DEBUG="${DEBUG:-0}" \
-    EMQX_CLUSTER__STATIC__SEEDS='["emqx1@127.0.0.1","emqx2@127.0.0.1"]' \
-    EMQX_CLUSTER__DISCOVERY_STRATEGY=static \
-    EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL="${EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL:-debug}" \
-    EMQX_LOG__FILE_HANDLERS__DEFAULT__FILE="$LOG1/emqx.log" \
-    EMQX_NODE_NAME='emqx1@127.0.0.1' \
-    EMQX_LOG_DIR="$LOG1" \
-    EMQX_NODE__DATA_DIR="$DATA1" \
-    EMQX_LISTENERS__TCP__DEFAULT__BIND='0.0.0.0:1881' \
-    EMQX_LISTENERS__SSL__DEFAULT__BIND='0.0.0.0:8881' \
-    EMQX_LISTENERS__WS__DEFAULT__BIND='0.0.0.0:8081' \
-    EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8084' \
-    EMQX_DASHBOARD__LISTENERS__HTTP__BIND='0.0.0.0:18081' \
-    "$BOOT1" start &
-
-env DEBUG="${DEBUG:-0}" \
-    EMQX_CLUSTER__STATIC__SEEDS='["emqx1@127.0.0.1","emqx2@127.0.0.1"]' \
-    EMQX_CLUSTER__DISCOVERY_STRATEGY=static \
-    EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL="${EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL:-debug}" \
-    EMQX_LOG__FILE_HANDLERS__DEFAULT__FILE="$LOG2/emqx.log" \
-    EMQX_NODE_NAME='emqx2@127.0.0.1' \
-    EMQX_LOG_DIR="$LOG2" \
-    EMQX_NODE__DATA_DIR="$DATA2" \
-    EMQX_LISTENERS__TCP__DEFAULT__BIND='0.0.0.0:1882' \
-    EMQX_LISTENERS__SSL__DEFAULT__BIND='0.0.0.0:8882' \
-    EMQX_LISTENERS__WS__DEFAULT__BIND='0.0.0.0:8082' \
-    EMQX_LISTENERS__WSS__DEFAULT__BIND='0.0.0.0:8085' \
-    EMQX_DASHBOARD__LISTENERS__HTTP__BIND='0.0.0.0:18082' \
-    "$BOOT2" start
+start_one_node 1 &
+start_one_node 2

From 2e6aa6388ff3dd93415faaed990535c5eab35180 Mon Sep 17 00:00:00 2001
From: firest <blankalupo@163.com>
Date: Fri, 17 Feb 2023 10:47:18 +0800
Subject: [PATCH 050/144] chore: update changelogs

---
 changes/{v5.0.18 => ce}/fix-9958.en.md                  | 0
 changes/{v5.0.18 => ce}/fix-9958.zh.md                  | 0
 changes/{v5.0.18 => ce}/fix-9961.en.md                  | 0
 changes/{v5.0.18 => ce}/fix-9961.zh.md                  | 0
 changes/{v5.0.18 => ce}/fix-9974.en.md                  | 0
 changes/{v5.0.18 => ce}/fix-9974.zh.md                  | 0
 changes/{v5.0.18 => ce}/fix-9978.en.md                  | 0
 changes/{v5.0.18 => ce}/fix-9978.zh.md                  | 0
 changes/{v5.0.18/perf-9967-en.md => ce/perf-9967.en.md} | 0
 changes/{v5.0.18/perf-9967-zh.md => ce/perf-9967.zh.md} | 0
 changes/ee/feat-9932-en.md                              | 1 +
 changes/ee/feat-9932-zh.md                              | 1 +
 12 files changed, 2 insertions(+)
 rename changes/{v5.0.18 => ce}/fix-9958.en.md (100%)
 rename changes/{v5.0.18 => ce}/fix-9958.zh.md (100%)
 rename changes/{v5.0.18 => ce}/fix-9961.en.md (100%)
 rename changes/{v5.0.18 => ce}/fix-9961.zh.md (100%)
 rename changes/{v5.0.18 => ce}/fix-9974.en.md (100%)
 rename changes/{v5.0.18 => ce}/fix-9974.zh.md (100%)
 rename changes/{v5.0.18 => ce}/fix-9978.en.md (100%)
 rename changes/{v5.0.18 => ce}/fix-9978.zh.md (100%)
 rename changes/{v5.0.18/perf-9967-en.md => ce/perf-9967.en.md} (100%)
 rename changes/{v5.0.18/perf-9967-zh.md => ce/perf-9967.zh.md} (100%)
 create mode 100644 changes/ee/feat-9932-en.md
 create mode 100644 changes/ee/feat-9932-zh.md

diff --git a/changes/v5.0.18/fix-9958.en.md b/changes/ce/fix-9958.en.md
similarity index 100%
rename from changes/v5.0.18/fix-9958.en.md
rename to changes/ce/fix-9958.en.md
diff --git a/changes/v5.0.18/fix-9958.zh.md b/changes/ce/fix-9958.zh.md
similarity index 100%
rename from changes/v5.0.18/fix-9958.zh.md
rename to changes/ce/fix-9958.zh.md
diff --git a/changes/v5.0.18/fix-9961.en.md b/changes/ce/fix-9961.en.md
similarity index 100%
rename from changes/v5.0.18/fix-9961.en.md
rename to changes/ce/fix-9961.en.md
diff --git a/changes/v5.0.18/fix-9961.zh.md b/changes/ce/fix-9961.zh.md
similarity index 100%
rename from changes/v5.0.18/fix-9961.zh.md
rename to changes/ce/fix-9961.zh.md
diff --git a/changes/v5.0.18/fix-9974.en.md b/changes/ce/fix-9974.en.md
similarity index 100%
rename from changes/v5.0.18/fix-9974.en.md
rename to changes/ce/fix-9974.en.md
diff --git a/changes/v5.0.18/fix-9974.zh.md b/changes/ce/fix-9974.zh.md
similarity index 100%
rename from changes/v5.0.18/fix-9974.zh.md
rename to changes/ce/fix-9974.zh.md
diff --git a/changes/v5.0.18/fix-9978.en.md b/changes/ce/fix-9978.en.md
similarity index 100%
rename from changes/v5.0.18/fix-9978.en.md
rename to changes/ce/fix-9978.en.md
diff --git a/changes/v5.0.18/fix-9978.zh.md b/changes/ce/fix-9978.zh.md
similarity index 100%
rename from changes/v5.0.18/fix-9978.zh.md
rename to changes/ce/fix-9978.zh.md
diff --git a/changes/v5.0.18/perf-9967-en.md b/changes/ce/perf-9967.en.md
similarity index 100%
rename from changes/v5.0.18/perf-9967-en.md
rename to changes/ce/perf-9967.en.md
diff --git a/changes/v5.0.18/perf-9967-zh.md b/changes/ce/perf-9967.zh.md
similarity index 100%
rename from changes/v5.0.18/perf-9967-zh.md
rename to changes/ce/perf-9967.zh.md
diff --git a/changes/ee/feat-9932-en.md b/changes/ee/feat-9932-en.md
new file mode 100644
index 000000000..f4f9ce40d
--- /dev/null
+++ b/changes/ee/feat-9932-en.md
@@ -0,0 +1 @@
+Integrate `TDengine` into `bridges` as a new backend.
diff --git a/changes/ee/feat-9932-zh.md b/changes/ee/feat-9932-zh.md
new file mode 100644
index 000000000..1fbf7bf34
--- /dev/null
+++ b/changes/ee/feat-9932-zh.md
@@ -0,0 +1 @@
+在 `桥接` 中集成 `TDengine`。

From caacc08ff9a8a326a093b96604cdbfcc6f5c0113 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Fri, 17 Feb 2023 15:28:27 +0100
Subject: [PATCH 051/144] chore: bump app VSN

---
 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
index b2a3c80c6..c30c927f2 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src
@@ -1,6 +1,6 @@
 {application, emqx_ee_bridge, [
     {description, "EMQX Enterprise data bridges"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {applications, [
         kernel,

From ca28be4387214f0eb86ff76e7b1e8230b72a57b9 Mon Sep 17 00:00:00 2001
From: Erik Timan <erik.timan@emqx.io>
Date: Fri, 17 Feb 2023 15:28:49 +0100
Subject: [PATCH 052/144] test(emqx_ee_connector): review fixes of influxdb
 tests

---
 .../test/emqx_ee_connector_influxdb_SUITE.erl | 47 +++++++++----------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
index a2e6c7c8f..f5e43c0bb 100644
--- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
+++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl
@@ -24,7 +24,8 @@ init_per_suite(Config) ->
     InfluxDBTCPPort = list_to_integer(os:getenv("INFLUXDB_APIV2_TCP_PORT", "8086")),
     InfluxDBTLSHost = os:getenv("INFLUXDB_APIV2_TLS_HOST", "toxiproxy"),
     InfluxDBTLSPort = list_to_integer(os:getenv("INFLUXDB_APIV2_TLS_PORT", "8087")),
-    case emqx_common_test_helpers:is_tcp_server_available(InfluxDBTCPHost, InfluxDBTCPPort) of
+    Servers = [{InfluxDBTCPHost, InfluxDBTCPPort}, {InfluxDBTLSHost, InfluxDBTLSPort}],
+    case emqx_common_test_helpers:is_all_tcp_servers_available(Servers) of
         true ->
             ok = emqx_common_test_helpers:start_apps([emqx_conf]),
             ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
@@ -37,7 +38,12 @@ init_per_suite(Config) ->
                 | Config
             ];
         false ->
-            {skip, no_influxdb}
+            case os:getenv("IS_CI") of
+                "yes" ->
+                    throw(no_influxdb);
+                _ ->
+                    {skip, no_influxdb}
+            end
     end.
 
 end_per_suite(_Config) ->
@@ -60,7 +66,7 @@ t_lifecycle(Config) ->
     Port = ?config(influxdb_tcp_port, Config),
     perform_lifecycle_check(
         <<"emqx_ee_connector_influxdb_SUITE">>,
-        influxdb_config(Host, Port, false, "verify_none")
+        influxdb_config(Host, Port, false, <<"verify_none">>)
     ).
 
 perform_lifecycle_check(PoolName, InitialConfig) ->
@@ -121,7 +127,7 @@ t_tls_verify_none(Config) ->
     PoolName = <<"emqx_ee_connector_influxdb_SUITE">>,
     Host = ?config(influxdb_tls_host, Config),
     Port = ?config(influxdb_tls_port, Config),
-    InitialConfig = influxdb_config(Host, Port, true, "verify_none"),
+    InitialConfig = influxdb_config(Host, Port, true, <<"verify_none">>),
     ValidStatus = perform_tls_opts_check(PoolName, InitialConfig, valid),
     ?assertEqual(connected, ValidStatus),
     InvalidStatus = perform_tls_opts_check(PoolName, InitialConfig, fail),
@@ -132,7 +138,8 @@ t_tls_verify_peer(Config) ->
     PoolName = <<"emqx_ee_connector_influxdb_SUITE">>,
     Host = ?config(influxdb_tls_host, Config),
     Port = ?config(influxdb_tls_port, Config),
-    InitialConfig = influxdb_config(Host, Port, true, "verify_peer"),
+    InitialConfig = influxdb_config(Host, Port, true, <<"verify_peer">>),
+    %% This works without a CA-cert & friends since we are using a mock
     ValidStatus = perform_tls_opts_check(PoolName, InitialConfig, valid),
     ?assertEqual(connected, ValidStatus),
     InvalidStatus = perform_tls_opts_check(PoolName, InitialConfig, fail),
@@ -181,25 +188,17 @@ perform_tls_opts_check(PoolName, InitialConfig, VerifyReturn) ->
 % %%------------------------------------------------------------------------------
 
 influxdb_config(Host, Port, SslEnabled, Verify) ->
-    RawConfig = list_to_binary(
-        io_lib:format(
-            ""
-            "\n"
-            "  bucket = mqtt\n"
-            "  org = emqx\n"
-            "  token = abcdefg\n"
-            "  server = \"~s:~b\"\n"
-            "  ssl {\n"
-            "    enable = ~s\n"
-            "    verify = ~s\n"
-            "  }\n"
-            "  "
-            "",
-            [Host, Port, SslEnabled, Verify]
-        )
-    ),
-
-    {ok, ResourceConfig} = hocon:binary(RawConfig),
+    Server = list_to_binary(io_lib:format("~s:~b", [Host, Port])),
+    ResourceConfig = #{
+        <<"bucket">> => <<"mqtt">>,
+        <<"org">> => <<"emqx">>,
+        <<"token">> => <<"abcdefg">>,
+        <<"server">> => Server,
+        <<"ssl">> => #{
+            <<"enable">> => SslEnabled,
+            <<"verify">> => Verify
+        }
+    },
     #{<<"config">> => ResourceConfig}.
 
 custom_verify() ->

From b90d3718360e513730b28ecfc24ba3a09288d30f Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Wed, 15 Feb 2023 15:18:52 +0100
Subject: [PATCH 053/144] ci: export docker image in build-slim-packages

---
 .github/workflows/build_slim_packages.yaml | 41 ++++++++++++++++++++++
 build                                      | 18 +++++-----
 deploy/docker/Dockerfile                   |  2 --
 3 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 434d1e790..83d7588ef 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -154,6 +154,47 @@ jobs:
         name: ${{ matrix.os }}
         path: _packages/**/*
 
+  docker:
+    runs-on: ubuntu-20.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        profile:
+          - emqx
+          - emqx-enterprise
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: prepare
+      run: |
+        echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
+    - name: build and test docker image
+      run: |
+        set -x
+        make $EMQX_NAME-docker
+        PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)}"
+        EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$EMQX_NAME:$PKG_VSN}"
+        CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
+        HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
+        IDLE_TIME=0
+        while ! curl http://127.0.0.1:$HTTP_PORT/status >/dev/null 2>&1; do
+            if [ $IDLE_TIME -gt 5 ]
+            then
+                echo "emqx running error"
+                docker rm -f $CID
+                exit 1
+            fi
+            sleep 5
+            IDLE_TIME=$((IDLE_TIME+1))
+        done
+        docker stop $CID
+        docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz
+    - uses: actions/upload-artifact@v3
+      with:
+        name: "${{ matrix.profile }}-docker"
+        path: "*.tar.gz"
+
   spellcheck:
     needs: linux
     strategy:
diff --git a/build b/build
index 195464612..82946eb7f 100755
--- a/build
+++ b/build
@@ -1,18 +1,12 @@
 #!/usr/bin/env bash
 
 # This script helps to build release artifacts.
-# arg1: profile, e.g. emqx | emqx-pkg
+# arg1: profile, e.g. emqx | emqx-enterprise
 # arg2: artifact, e.g. rel | relup | tgz | pkg
 
-if [[ -n "$DEBUG" ]]; then
-    set -x
-fi
 set -euo pipefail
 
-DEBUG="${DEBUG:-0}"
-if [ "$DEBUG" -eq 1 ]; then
-    set -x
-fi
+[ ${DEBUG:-0} -eq 1 ] && set -x
 
 PROFILE_ARG="$1"
 ARTIFACT="$2"
@@ -318,6 +312,12 @@ make_tgz() {
     log "Archive sha256sum: $(cat "${target}.sha256")"
 }
 
+trap docker_cleanup EXIT
+
+docker_cleanup() {
+    rm -f ./.dockerignore >/dev/null
+}
+
 ## This function builds the default docker image based on debian 11
 make_docker() {
     EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
@@ -329,6 +329,7 @@ make_docker() {
     local default_tag="emqx/${PROFILE%%-elixir}:${PKG_VSN}"
     EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-$default_tag}"
 
+    echo '_build' >> ./.dockerignore
     set -x
     docker build --no-cache --pull \
        --build-arg BUILD_FROM="${EMQX_BUILDER}" \
@@ -336,6 +337,7 @@ make_docker() {
        --build-arg EMQX_NAME="$PROFILE" \
        --tag "${EMQX_IMAGE_TAG}" \
        -f "${EMQX_DOCKERFILE}" .
+    [[ "${DEBUG:-}" -eq 1 ]] || set +x
 }
 
 function join {
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index d424fb027..f26926bce 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -10,11 +10,9 @@ ENV EMQX_RELUP=false
 RUN export PROFILE=${EMQX_NAME%%-elixir} \
     && export EMQX_NAME1=$EMQX_NAME \
     && export EMQX_NAME=$PROFILE \
-    && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \
     && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \
     && export EMQX_REL_FORM='docker' \
     && cd /emqx \
-    && rm -rf $EMQX_LIB_PATH \
     && make $EMQX_NAME1 \
     && rm -f $EMQX_REL_PATH/*.tar.gz \
     && mkdir -p /emqx-rel \

From 739455d8443a3428ee36e6b3a70d913e02a215b4 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Fri, 17 Feb 2023 16:18:20 +0100
Subject: [PATCH 054/144] ci: test docker images and packages in the same way

---
 .github/workflows/build_slim_packages.yaml | 41 ++++++++++++----------
 build                                      |  2 +-
 scripts/pkg-tests.sh                       | 22 ++----------
 scripts/test/emqx-smoke-test.sh            | 19 ++++++++++
 4 files changed, 44 insertions(+), 40 deletions(-)
 create mode 100755 scripts/test/emqx-smoke-test.sh

diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 83d7588ef..692e4a987 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -155,7 +155,7 @@ jobs:
         path: _packages/**/*
 
   docker:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false
@@ -168,32 +168,35 @@ jobs:
     - uses: actions/checkout@v3
     - name: prepare
       run: |
-        echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
-    - name: build and test docker image
+        EMQX_NAME=${{ matrix.profile }}
+        PKG_VSN=${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)}
+        EMQX_IMAGE_TAG=emqx/$EMQX_NAME:test
+        echo "EMQX_NAME=$EMQX_NAME" >> $GITHUB_ENV
+        echo "PKG_VSN=$PKG_VSN" >> $GITHUB_ENV
+        echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV
+    - uses: docker/setup-buildx-action@v2
+    - name: build and export to Docker
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        file: ./deploy/docker/Dockerfile
+        load: true
+        tags: ${{ env.EMQX_IMAGE_TAG }}
+        build-args: |
+          EMQX_NAME=${{ env.EMQX_NAME }}
+    - name: test docker image
       run: |
-        set -x
-        make $EMQX_NAME-docker
-        PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)}"
-        EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$EMQX_NAME:$PKG_VSN}"
         CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
         HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
-        IDLE_TIME=0
-        while ! curl http://127.0.0.1:$HTTP_PORT/status >/dev/null 2>&1; do
-            if [ $IDLE_TIME -gt 5 ]
-            then
-                echo "emqx running error"
-                docker rm -f $CID
-                exit 1
-            fi
-            sleep 5
-            IDLE_TIME=$((IDLE_TIME+1))
-        done
+        ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
         docker stop $CID
+    - name: export docker image
+      run: |
         docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz
     - uses: actions/upload-artifact@v3
       with:
         name: "${{ matrix.profile }}-docker"
-        path: "*.tar.gz"
+        path: "${{ env.EMQX_NAME }}-${{ env.PKG_VSN }}.tar.gz"
 
   spellcheck:
     needs: linux
diff --git a/build b/build
index 82946eb7f..120fc5eec 100755
--- a/build
+++ b/build
@@ -6,7 +6,7 @@
 
 set -euo pipefail
 
-[ ${DEBUG:-0} -eq 1 ] && set -x
+[ "${DEBUG:-0}" -eq 1 ] && set -x
 
 PROFILE_ARG="$1"
 ARTIFACT="$2"
diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh
index 768a152c0..c17c47ad2 100755
--- a/scripts/pkg-tests.sh
+++ b/scripts/pkg-tests.sh
@@ -103,16 +103,7 @@ emqx_test(){
                 cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true
                 exit 1
             fi
-            IDLE_TIME=0
-            while ! curl http://127.0.0.1:18083/status >/dev/null 2>&1; do
-                if [ $IDLE_TIME -gt 10 ]
-                then
-                    echo "emqx running error"
-                    exit 1
-                fi
-                sleep 10
-                IDLE_TIME=$((IDLE_TIME+1))
-            done
+            "$SCRIPTS/test/emqx-smoke-test.sh" 127.0.0.1 18083
             pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
             if ! "${PACKAGE_PATH}"/emqx/bin/emqx stop; then
                 cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true
@@ -208,16 +199,7 @@ EOF
         cat /var/log/emqx/emqx.log.1 || true
         exit 1
     fi
-    IDLE_TIME=0
-    while ! curl http://127.0.0.1:18083/status >/dev/null 2>&1; do
-        if [ $IDLE_TIME -gt 10 ]
-        then
-            echo "emqx running error"
-            exit 1
-        fi
-        sleep 10
-        IDLE_TIME=$((IDLE_TIME+1))
-    done
+    "$SCRIPTS/test/emqx-smoke-test.sh" 127.0.0.1 18083
     pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
     # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
     ps -ef | grep -E '\-progname\s.+emqx\s'
diff --git a/scripts/test/emqx-smoke-test.sh b/scripts/test/emqx-smoke-test.sh
new file mode 100755
index 000000000..361137bc0
--- /dev/null
+++ b/scripts/test/emqx-smoke-test.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+[ $# -ne 2 ] && { echo "Usage: $0 ip port"; exit 1; }
+
+IP=$1
+PORT=$2
+URL="http://$IP:$PORT/status"
+
+ATTEMPTS=10
+while ! curl "$URL" >/dev/null 2>&1; do
+    if [ $ATTEMPTS -eq 0 ]; then
+        echo "emqx is not responding on $URL"
+        exit 1
+    fi
+    sleep 5
+    ATTEMPTS=$((ATTEMPTS-1))
+done

From 00b59b493928daa001fe9300f19ee18325580cbb Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 1 Nov 2022 23:03:22 +0100
Subject: [PATCH 055/144] feat(quic): WIP multi-stream

---
 apps/emqx/src/emqx_connection.erl             |  21 ++--
 apps/emqx/src/emqx_listeners.erl              |   3 +-
 apps/emqx/src/emqx_quic_connection.erl        | 104 +++++++++++++++---
 apps/emqx/src/emqx_quic_stream.erl            |   2 +-
 .../emqx/test/emqx_mqtt_protocol_v5_SUITE.erl |   8 ++
 5 files changed, 111 insertions(+), 27 deletions(-)

diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 5b783f2fe..6c88b87cf 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -525,11 +525,10 @@ handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl ->
     inc_counter(incoming_bytes, Oct),
     ok = emqx_metrics:inc('bytes.received', Oct),
     when_bytes_in(Oct, Data, State);
-handle_msg({quic, Data, _Sock, _, _, _}, State) ->
-    Oct = iolist_size(Data),
-    inc_counter(incoming_bytes, Oct),
-    ok = emqx_metrics:inc('bytes.received', Oct),
-    when_bytes_in(Oct, Data, State);
+handle_msg({quic, Data, _Stream, #{len := Len}}, State) when is_binary(Data) ->
+    inc_counter(incoming_bytes, Len),
+    ok = emqx_metrics:inc('bytes.received', Len),
+    when_bytes_in(Len, Data, State);
 handle_msg(check_cache, #state{limiter_buffer = Cache} = State) ->
     case queue:peek(Cache) of
         empty ->
@@ -893,12 +892,12 @@ handle_info({sock_error, Reason}, State) ->
         false -> ok
     end,
     handle_info({sock_closed, Reason}, close_socket(State));
-handle_info({quic, peer_send_shutdown, _Stream}, State) ->
-    handle_info({sock_closed, force}, close_socket(State));
-handle_info({quic, closed, _Channel, ReasonFlag}, State) ->
-    handle_info({sock_closed, ReasonFlag}, State);
-handle_info({quic, closed, _Stream}, State) ->
-    handle_info({sock_closed, force}, State);
+%% handle_info({quic, peer_send_shutdown, _Stream}, State) ->
+%%     handle_info({sock_closed, force}, close_socket(State));
+%% handle_info({quic, closed, _Channel, ReasonFlag}, State) ->
+%%     handle_info({sock_closed, ReasonFlag}, State);
+%% handle_info({quic, closed, _Stream}, State) ->
+%%     handle_info({sock_closed, force}, State);
 handle_info(Info, State) ->
     with_channel(handle_info, [Info], State).
 
diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index 003c8785e..45f3b2cfd 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -375,7 +375,8 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) ->
                 {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)},
                 {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)},
                 {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)},
-                {server_resumption_level, 2}
+                {server_resumption_level, 2},
+                {verify, none}
             ],
             ConnectionOpts = #{
                 conn_callback => emqx_quic_connection,
diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index 9a2589a3a..6da9ec9a8 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -22,24 +22,42 @@
 -define(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0).
 -endif.
 
-%% Callbacks
+-behavior(quicer_connection).
+
 -export([
     init/1,
-    new_conn/2,
-    connected/2,
-    shutdown/2
+    new_conn/3,
+    connected/3,
+    transport_shutdown/3,
+    shutdown/3,
+    closed/3,
+    local_address_changed/3,
+    peer_address_changed/3,
+    streams_available/3,
+    peer_needs_streams/3,
+    resumed/3,
+    nst_received/3,
+    new_stream/3
 ]).
 
 -type cb_state() :: map() | proplists:proplist().
+-type cb_ret() :: ok.
 
--spec init(cb_state()) -> cb_state().
 init(ConnOpts) when is_list(ConnOpts) ->
     init(maps:from_list(ConnOpts));
+init(#{stream_opts := SOpts} = S) when is_list(SOpts) ->
+    init(S#{stream_opts := maps:from_list(SOpts)});
 init(ConnOpts) when is_map(ConnOpts) ->
-    ConnOpts.
+    {ok, ConnOpts}.
 
--spec new_conn(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}.
-new_conn(Conn, #{zone := Zone} = S) ->
+closed(_Conn, #{is_peer_acked := true}, S) ->
+    {stop, normal, S};
+closed(_Conn, #{is_peer_acked := false}, S) ->
+    {stop, abnorml, S}.
+
+-spec new_conn(quicer:connection_handler(), quicer:new_conn_props(), cb_state()) ->
+    {ok, cb_state()} | {error, any()}.
+new_conn(Conn, #{version := _Vsn}, #{zone := Zone} = S) ->
     process_flag(trap_exit, true),
     case emqx_olp:is_overloaded() andalso is_zone_olp_enabled(Zone) of
         false ->
@@ -47,7 +65,7 @@ new_conn(Conn, #{zone := Zone} = S) ->
             receive
                 {Pid, stream_acceptor_ready} ->
                     ok = quicer:async_handshake(Conn),
-                    {ok, S};
+                    {ok, S#{conn => Conn}};
                 {'EXIT', Pid, _Reason} ->
                     {error, stream_accept_error}
             end;
@@ -56,18 +74,76 @@ new_conn(Conn, #{zone := Zone} = S) ->
             {error, overloaded}
     end.
 
--spec connected(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}.
-connected(Conn, #{slow_start := false} = S) ->
+-spec connected(quicer:connection_handler(), quicer:connected_props(), cb_state()) ->
+    {ok, cb_state()} | {error, any()}.
+connected(Conn, _Props, #{slow_start := false} = S) ->
     {ok, _Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S),
     {ok, S};
-connected(_Conn, S) ->
+connected(_Conn, _Props, S) ->
     {ok, S}.
 
--spec shutdown(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}.
-shutdown(Conn, S) ->
+-spec resumed(quicer:connection_handle(), SessionData :: binary() | false, cb_state()) -> cb_ret().
+resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when
+    is_function(ResumeFun)
+->
+    ResumeFun(Conn, Data, S);
+resumed(_Conn, _Data, S) ->
+    {ok, S}.
+
+-spec nst_received(quicer:connection_handle(), TicketBin :: binary(), cb_state()) -> cb_ret().
+nst_received(_Conn, _Data, S) ->
+    {stop, no_nst_for_server, S}.
+
+-spec new_stream(quicer:stream_handle(), quicer:new_stream_props(), cb_state()) -> cb_ret().
+new_stream(
+    Stream,
+    #{is_orphan := true} = Props,
+    #{
+        conn := Conn,
+        streams := Streams,
+        stream_opts := SOpts
+    } = CBState
+) ->
+    %% Spawn new stream
+    case quicer_stream:start_link(emqx_quic_stream, Stream, Conn, SOpts, Props) of
+        {ok, StreamOwner} ->
+            quicer_connection:handoff_stream(Stream, StreamOwner),
+            {ok, CBState#{streams := [{StreamOwner, Stream} | Streams]}};
+        Other ->
+            Other
+    end.
+-spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret().
+shutdown(Conn, _ErrorCode, S) ->
     quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
     {ok, S}.
 
+-spec transport_shutdown(quicer:connection_handle(), quicer:transport_shutdown_props(), cb_state()) ->
+    cb_ret().
+transport_shutdown(_C, _DownInfo, S) ->
+    {ok, S}.
+
+-spec peer_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state) -> cb_ret().
+peer_address_changed(_C, _NewAddr, S) ->
+    {ok, S}.
+
+-spec local_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state()) ->
+    cb_ret().
+local_address_changed(_C, _NewAddr, S) ->
+    {ok, S}.
+
+-spec streams_available(
+    quicer:connection_handle(),
+    {BidirStreams :: non_neg_integer(), UnidirStreams :: non_neg_integer()},
+    cb_state()
+) -> cb_ret().
+streams_available(_C, {_BidirCnt, _UnidirCnt}, S) ->
+    {ok, S}.
+
+-spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret().
+%% for https://github.com/microsoft/msquic/issues/3120
+peer_needs_streams(_C, undefined, S) ->
+    {ok, S}.
+
 -spec is_zone_olp_enabled(emqx_types:zone()) -> boolean().
 is_zone_olp_enabled(Zone) ->
     case emqx_config:get_zone_conf(Zone, [overload_protection]) of
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index 567488862..fe6ff692c 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -37,7 +37,7 @@ wait({ConnOwner, Conn}) ->
     ConnOwner ! {self(), stream_acceptor_ready},
     receive
         %% from msquic
-        {quic, new_stream, Stream} ->
+        {quic, new_stream, Stream, _Props} ->
             {ok, {quic, Conn, Stream}};
         {'EXIT', ConnOwner, _Reason} ->
             {error, enotconn}
diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
index 7e97c5bf4..07299bd42 100644
--- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
+++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
@@ -78,6 +78,14 @@ end_per_group(_Group, _Config) ->
 
 init_per_suite(Config) ->
     %% Start Apps
+    %% dbg:tracer(process, {fun dbg:dhandler/2,group_leader()}),
+    %% dbg:p(all,c),
+    %% dbg:tp(emqx_quic_connection,cx),
+    %% dbg:tp(emqx_quic_stream,cx),
+    %% dbg:tp(emqtt_quic,cx),
+    %% dbg:tp(emqtt,cx),
+    %% dbg:tp(emqtt_quic_stream,cx),
+    %% dbg:tp(emqtt_quic_connection,cx),
     emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
     Config.

From 2d09a054e328ff824b28354c80d917549341805e Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 2 Nov 2022 09:46:48 +0100
Subject: [PATCH 056/144] chore: add some typing

---
 apps/emqx/rebar.config.script          | 2 +-
 apps/emqx/src/emqx_quic_connection.erl | 6 +++++-
 rebar.config                           | 2 +-
 rebar.config.erl                       | 3 ++-
 4 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index 75f748017..0ecd21715 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}.
+Quicer = {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/multi-streams"}}}. %% @TODO revert
 
 ExtraDeps = fun(C) ->
     {deps, Deps0} = lists:keyfind(deps, 1, C),
diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index 6da9ec9a8..22d068237 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -40,9 +40,11 @@
     new_stream/3
 ]).
 
--type cb_state() :: map() | proplists:proplist().
+-type cb_state() :: map().
 -type cb_ret() :: ok.
 
+-spec init(map() | list()) -> cb_state().
+
 init(ConnOpts) when is_list(ConnOpts) ->
     init(maps:from_list(ConnOpts));
 init(#{stream_opts := SOpts} = S) when is_list(SOpts) ->
@@ -50,6 +52,8 @@ init(#{stream_opts := SOpts} = S) when is_list(SOpts) ->
 init(ConnOpts) when is_map(ConnOpts) ->
     {ok, ConnOpts}.
 
+-spec closed(quicer:conneciton_hanlder(), quicer:conn_closed_props(), cb_state()) ->
+    {ok, cb_state()} | {error, any()}.
 closed(_Conn, #{is_peer_acked := true}, S) ->
     {stop, normal, S};
 closed(_Conn, #{is_peer_acked := false}, S) ->
diff --git a/rebar.config b/rebar.config
index ffdb7407a..76402897b 100644
--- a/rebar.config
+++ b/rebar.config
@@ -62,7 +62,7 @@
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}}
     , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
-    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0"}}}
+    , {emqtt, {git, "https://github.com/qzhuyan/emqtt", {branch, "dev/william/multi-streams"}}} %% @TODO revert
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
     , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
diff --git a/rebar.config.erl b/rebar.config.erl
index 4ff94bd78..9da71355b 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,7 +39,8 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}.
+    %% @TODO revert
+    {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/multi-streams"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From a51c8869086358e9e4387727bdc3b8f593448f46 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 2 Nov 2022 14:20:17 +0100
Subject: [PATCH 057/144] fix: prepare for multi stream

---
 apps/emqx/src/emqx_connection.erl      |  27 ++--
 apps/emqx/src/emqx_quic_connection.erl |  72 ++++++---
 apps/emqx/src/emqx_quic_stream.erl     | 201 +++++++++++++++++++++++--
 3 files changed, 263 insertions(+), 37 deletions(-)

diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 6c88b87cf..1c8b85808 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% MQTT/TCP|TLS Connection
+%% MQTT/TCP|TLS Connection|QUIC Stream
 -module(emqx_connection).
 
 -include("emqx.hrl").
@@ -189,12 +189,16 @@
     ]}
 ).
 
--spec start_link(
-    esockd:transport(),
-    esockd:socket() | {pid(), quicer:connection_handler()},
-    emqx_channel:opts()
-) ->
-    {ok, pid()}.
+-spec start_link
+    (esockd:transport(), esockd:socket(), emqx_channel:opts()) ->
+        {ok, pid()};
+    (
+        emqx_quic_stream,
+        {ConnOwner :: pid(), quicer:connection_handler(), quicer:new_conn_props()},
+        emqx_quic_connection:cb_state()
+    ) ->
+        {ok, pid()}.
+
 start_link(Transport, Socket, Options) ->
     Args = [self(), Transport, Socket, Options],
     CPid = proc_lib:spawn_link(?MODULE, init, Args),
@@ -324,6 +328,7 @@ init_state(
     Limiter = emqx_limiter_container:get_limiter_by_types(Listener, LimiterTypes, LimiterCfg),
 
     FrameOpts = #{
+        %% @TODO:q what is strict_mode?
         strict_mode => emqx_config:get_zone_conf(Zone, [mqtt, strict_mode]),
         max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size])
     },
@@ -476,7 +481,9 @@ process_msg([Msg | More], State) ->
             {ok, Msgs, NState} ->
                 process_msg(append_msg(More, Msgs), NState);
             {stop, Reason, NState} ->
-                {stop, Reason, NState}
+                {stop, Reason, NState};
+            {stop, Reason} ->
+                {stop, Reason, State}
         end
     catch
         exit:normal ->
@@ -507,7 +514,6 @@ append_msg(Q, Msg) ->
 
 %%--------------------------------------------------------------------
 %% Handle a Msg
-
 handle_msg({'$gen_call', From, Req}, State) ->
     case handle_call(From, Req, State) of
         {reply, Reply, NState} ->
@@ -747,6 +753,7 @@ when_bytes_in(Oct, Data, State) ->
         NState
     ).
 
+%% @doc: return a reversed Msg list
 -compile({inline, [next_incoming_msgs/3]}).
 next_incoming_msgs([Packet], Msgs, State) ->
     {ok, [{incoming, Packet} | Msgs], State};
@@ -892,6 +899,8 @@ handle_info({sock_error, Reason}, State) ->
         false -> ok
     end,
     handle_info({sock_closed, Reason}, close_socket(State));
+handle_info({quic, Event, Handle, Prop}, State) ->
+    emqx_quic_stream:Event(Handle, Prop, State);
 %% handle_info({quic, peer_send_shutdown, _Stream}, State) ->
 %%     handle_info({sock_closed, force}, close_socket(State));
 %% handle_info({quic, closed, _Channel, ReasonFlag}, State) ->
diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index 22d068237..a5af3d4b3 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -16,6 +16,7 @@
 
 -module(emqx_quic_connection).
 
+-include("logger.hrl").
 -ifndef(BUILD_WITHOUT_QUIC).
 -include_lib("quicer/include/quicer.hrl").
 -else.
@@ -40,37 +41,50 @@
     new_stream/3
 ]).
 
--type cb_state() :: map().
--type cb_ret() :: ok.
-
--spec init(map() | list()) -> cb_state().
+-type cb_state() :: #{
+    ctrl_pid := undefined | pid(),
+    conn := undefined | quicer:conneciton_hanlder(),
+    stream_opts := map(),
+    is_resumed => boolean(),
+    _ => _
+}.
+-type cb_ret() :: quicer_lib:cb_ret().
 
+-spec init(map() | list()) -> {ok, cb_state()}.
 init(ConnOpts) when is_list(ConnOpts) ->
     init(maps:from_list(ConnOpts));
 init(#{stream_opts := SOpts} = S) when is_list(SOpts) ->
     init(S#{stream_opts := maps:from_list(SOpts)});
 init(ConnOpts) when is_map(ConnOpts) ->
-    {ok, ConnOpts}.
+    {ok, init_cb_state(ConnOpts)}.
 
 -spec closed(quicer:conneciton_hanlder(), quicer:conn_closed_props(), cb_state()) ->
-    {ok, cb_state()} | {error, any()}.
-closed(_Conn, #{is_peer_acked := true}, S) ->
-    {stop, normal, S};
-closed(_Conn, #{is_peer_acked := false}, S) ->
-    {stop, abnorml, S}.
+    {stop, normal, cb_state()}.
+closed(_Conn, #{is_peer_acked := _} = Prop, S) ->
+    ?SLOG(debug, Prop),
+    {stop, normal, S}.
 
 -spec new_conn(quicer:connection_handler(), quicer:new_conn_props(), cb_state()) ->
     {ok, cb_state()} | {error, any()}.
-new_conn(Conn, #{version := _Vsn}, #{zone := Zone} = S) ->
+new_conn(
+    Conn,
+    #{version := _Vsn} = ConnInfo,
+    #{zone := Zone, conn := undefined, ctrl_pid := undefined} = S
+) ->
     process_flag(trap_exit, true),
+    ?SLOG(debug, ConnInfo),
     case emqx_olp:is_overloaded() andalso is_zone_olp_enabled(Zone) of
         false ->
-            {ok, Pid} = emqx_connection:start_link(emqx_quic_stream, {self(), Conn}, S),
+            {ok, Pid} = emqx_connection:start_link(
+                emqx_quic_stream,
+                {self(), Conn, maps:without([crypto_buffer], ConnInfo)},
+                S
+            ),
             receive
                 {Pid, stream_acceptor_ready} ->
                     ok = quicer:async_handshake(Conn),
-                    {ok, S#{conn => Conn}};
-                {'EXIT', Pid, _Reason} ->
+                    {ok, S#{conn := Conn, ctrl_pid := Pid}};
+                {'EXIT', _Pid, _Reason} ->
                     {error, stream_accept_error}
             end;
         true ->
@@ -80,10 +94,12 @@ new_conn(Conn, #{version := _Vsn}, #{zone := Zone} = S) ->
 
 -spec connected(quicer:connection_handler(), quicer:connected_props(), cb_state()) ->
     {ok, cb_state()} | {error, any()}.
-connected(Conn, _Props, #{slow_start := false} = S) ->
+connected(Conn, Props, #{slow_start := false} = S) ->
+    ?SLOG(debug, Props),
     {ok, _Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S),
     {ok, S};
-connected(_Conn, _Props, S) ->
+connected(_Conn, Props, S) ->
+    ?SLOG(debug, Props),
     {ok, S}.
 
 -spec resumed(quicer:connection_handle(), SessionData :: binary() | false, cb_state()) -> cb_ret().
@@ -92,10 +108,11 @@ resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when
 ->
     ResumeFun(Conn, Data, S);
 resumed(_Conn, _Data, S) ->
-    {ok, S}.
+    {ok, S#{is_resumed := true}}.
 
 -spec nst_received(quicer:connection_handle(), TicketBin :: binary(), cb_state()) -> cb_ret().
 nst_received(_Conn, _Data, S) ->
+    %% As server we should not recv NST!
     {stop, no_nst_for_server, S}.
 
 -spec new_stream(quicer:stream_handle(), quicer:new_stream_props(), cb_state()) -> cb_ret().
@@ -116,14 +133,17 @@ new_stream(
         Other ->
             Other
     end.
+
 -spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret().
 shutdown(Conn, _ErrorCode, S) ->
+    %% @TODO check spec what to do with the ErrorCode?
     quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
     {ok, S}.
 
 -spec transport_shutdown(quicer:connection_handle(), quicer:transport_shutdown_props(), cb_state()) ->
     cb_ret().
 transport_shutdown(_C, _DownInfo, S) ->
+    %% @TODO some counter
     {ok, S}.
 
 -spec peer_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state) -> cb_ret().
@@ -140,14 +160,21 @@ local_address_changed(_C, _NewAddr, S) ->
     {BidirStreams :: non_neg_integer(), UnidirStreams :: non_neg_integer()},
     cb_state()
 ) -> cb_ret().
-streams_available(_C, {_BidirCnt, _UnidirCnt}, S) ->
-    {ok, S}.
+streams_available(_C, {BidirCnt, UnidirCnt}, S) ->
+    {ok, S#{
+        peer_bidi_stream_count => BidirCnt,
+        peer_unidi_stream_count => UnidirCnt
+    }}.
 
 -spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret().
+%% @TODO this is not going to get triggered.
 %% for https://github.com/microsoft/msquic/issues/3120
 peer_needs_streams(_C, undefined, S) ->
     {ok, S}.
 
+%%%
+%%%  Internals
+%%%
 -spec is_zone_olp_enabled(emqx_types:zone()) -> boolean().
 is_zone_olp_enabled(Zone) ->
     case emqx_config:get_zone_conf(Zone, [overload_protection]) of
@@ -156,3 +183,10 @@ is_zone_olp_enabled(Zone) ->
         _ ->
             false
     end.
+
+-spec init_cb_state(map()) -> cb_state().
+init_cb_state(Map) ->
+    Map#{
+        ctrl_pid => undefined,
+        conn => undefined
+    }.
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index fe6ff692c..d9c080c0d 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -17,6 +17,8 @@
 %% MQTT/QUIC Stream
 -module(emqx_quic_stream).
 
+-behaviour(quicer_stream).
+
 %% emqx transport Callbacks
 -export([
     type/1,
@@ -32,13 +34,71 @@
     peercert/1
 ]).
 
-wait({ConnOwner, Conn}) ->
+-include("logger.hrl").
+-ifndef(BUILD_WITHOUT_QUIC).
+-include_lib("quicer/include/quicer.hrl").
+-else.
+%% STREAM SHUTDOWN FLAGS
+-define(QUIC_STREAM_SHUTDOWN_FLAG_NONE, 0).
+% Cleanly closes the send path.
+-define(QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 1).
+% Abruptly closes the send path.
+-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND, 2).
+% Abruptly closes the receive path.
+-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, 4).
+% Abruptly closes both send and receive paths.
+-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT, 6).
+-define(QUIC_STREAM_SHUTDOWN_FLAG_IMMEDIATE, 8).
+-endif.
+
+-type cb_ret() :: gen_statem:event_handler_result().
+-type cb_data() :: emqtt_quic:cb_data().
+-type connection_handle() :: quicer:connection_handle().
+-type stream_handle() :: quicer:stream_handle().
+
+-export([
+    init_handoff/4,
+    new_stream/3,
+    start_completed/3,
+    send_complete/3,
+    peer_send_shutdown/3,
+    peer_send_aborted/3,
+    peer_receive_aborted/3,
+    send_shutdown_complete/3,
+    stream_closed/3,
+    peer_accepted/3,
+    passive/3,
+    handle_call/4
+]).
+
+-export_type([socket/0]).
+
+-opaque socket() :: {quic, connection_handle(), stream_handle(), socket_info()}.
+
+-type socket_info() :: #{
+    is_orphan => boolean(),
+    ctrl_stream_start_flags => quicer:stream_open_flags(),
+    %% quicer:new_conn_props
+    _ => _
+}.
+
+-spec wait({pid(), quicer:connection_handle(), socket_info()}) ->
+    {ok, socket()} | {error, enotconn}.
+wait({ConnOwner, Conn, ConnInfo}) ->
     {ok, Conn} = quicer:async_accept_stream(Conn, []),
     ConnOwner ! {self(), stream_acceptor_ready},
     receive
-        %% from msquic
-        {quic, new_stream, Stream, _Props} ->
-            {ok, {quic, Conn, Stream}};
+        %% New incoming stream, this is a *ctrl* stream
+        {quic, new_stream, Stream, #{is_orphan := IsOrphan, flags := StartFlags}} ->
+            SocketInfo = ConnInfo#{
+                is_orphan => IsOrphan,
+                ctrl_stream_start_flags => StartFlags
+            },
+            {ok, socket(Conn, Stream, SocketInfo)};
+        %% connection closed event for stream acceptor
+        {quic, closed, undefined, undefined} ->
+            {error, enotconn};
+        %% Connection owner process down
         {'EXIT', ConnOwner, _Reason} ->
             {error, enotconn}
     end.
@@ -46,17 +106,17 @@ wait({ConnOwner, Conn}) ->
 type(_) ->
     quic.
 
-peername({quic, Conn, _Stream}) ->
+peername({quic, Conn, _Stream, _Info}) ->
     quicer:peername(Conn).
 
-sockname({quic, Conn, _Stream}) ->
+sockname({quic, Conn, _Stream, _Info}) ->
     quicer:sockname(Conn).
 
 peercert(_S) ->
     %% @todo but unsupported by msquic
     nossl.
 
-getstat({quic, Conn, _Stream}, Stats) ->
+getstat({quic, Conn, _Stream, _Info}, Stats) ->
     case quicer:getstat(Conn, Stats) of
         {error, _} -> {error, closed};
         Res -> Res
@@ -84,7 +144,7 @@ getopts(_Socket, _Opts) ->
         {buffer, 80000}
     ]}.
 
-fast_close({quic, _Conn, Stream}) ->
+fast_close({quic, _Conn, Stream, _Info}) ->
     %% Flush send buffer, gracefully shutdown
     quicer:async_shutdown_stream(Stream),
     ok.
@@ -102,8 +162,131 @@ ensure_ok_or_exit(Fun, Args = [Sock | _]) when is_atom(Fun), is_list(Args) ->
             Result
     end.
 
-async_send({quic, _Conn, Stream}, Data, _Options) ->
+async_send({quic, _Conn, Stream, _Info}, Data, _Options) ->
     case quicer:send(Stream, Data) of
         {ok, _Len} -> ok;
         Other -> Other
     end.
+
+%%%
+%%% quicer stream callbacks
+%%%
+
+-spec init_handoff(stream_handle(), #{}, quicer:connection_handle(), #{}) -> cb_ret().
+init_handoff(_Stream, _StreamOpts, _Conn, _Flags) ->
+    %% stream owner already set while starts.
+    {stop, unimpl}.
+
+-spec new_stream(stream_handle(), quicer:new_stream_props(), cb_data()) -> cb_ret().
+new_stream(_Stream, #{flags := _Flags, is_orphan := _IsOrphan}, _Conn) ->
+    {stop, unimpl}.
+
+-spec peer_accepted(stream_handle(), undefined, cb_data()) -> cb_ret().
+peer_accepted(_Stream, undefined, S) ->
+    %% We just ignore it
+    {ok, S}.
+
+-spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret().
+peer_receive_aborted(Stream, ErrorCode, #{is_unidir := false} = S) ->
+    %% we abort send with same reason
+    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
+    {ok, S};
+peer_receive_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := true} = S) ->
+    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
+    {ok, S}.
+
+-spec peer_send_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret().
+peer_send_aborted(Stream, ErrorCode, #{is_unidir := false} = S) ->
+    %% we abort receive with same reason
+    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode),
+    {ok, S};
+peer_send_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := false} = S) ->
+    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode),
+    {ok, S}.
+
+-spec peer_send_shutdown(stream_handle(), undefined, cb_data()) -> cb_ret().
+peer_send_shutdown(Stream, undefined, S) ->
+    ok = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0),
+    {ok, S}.
+
+-spec send_complete(stream_handle(), boolean(), cb_data()) -> cb_ret().
+send_complete(_Stream, false, S) ->
+    {ok, S};
+send_complete(_Stream, true = _IsCancelled, S) ->
+    ?SLOG(error, #{message => "send cancelled"}),
+    {ok, S}.
+
+-spec send_shutdown_complete(stream_handle(), boolean(), cb_data()) -> cb_ret().
+send_shutdown_complete(_Stream, _IsGraceful, S) ->
+    {ok, S}.
+
+-spec start_completed(stream_handle(), quicer:stream_start_completed_props(), cb_data()) ->
+    cb_ret().
+start_completed(_Stream, #{status := success, stream_id := StreamId} = Prop, S) ->
+    ?SLOG(debug, Prop),
+    {ok, S#{stream_id => StreamId}};
+start_completed(_Stream, #{status := stream_limit_reached, stream_id := _StreamId} = Prop, _S) ->
+    ?SLOG(error, #{message => start_completed}, Prop),
+    {stop, stream_limit_reached};
+start_completed(_Stream, #{status := Other} = Prop, S) ->
+    ?SLOG(error, Prop),
+    %% or we could retry?
+    {stop, {start_fail, Other}, S}.
+
+%% Local stream, Unidir
+%% -spec handle_stream_data(stream_handle(), binary(), quicer:recv_data_props(), cb_data())
+%%                         -> cb_ret().
+%% handle_stream_data(Stream, Bin, Flags, #{ is_local := true
+%%                                         , parse_state := PS} = S) ->
+%%     ?SLOG(debug, #{data => Bin}, Flags),
+%%     case parse(Bin, PS, []) of
+%%         {keep_state, NewPS, Packets} ->
+%%             quicer:setopt(Stream, active, once),
+%%             {keep_state, S#{parse_state := NewPS},
+%%              [{next_event, cast, P } || P <- lists:reverse(Packets)]};
+%%         {stop, _} = Stop ->
+%%             Stop
+%%     end;
+%% %% Remote stream
+%% handle_stream_data(_Stream, _Bin, _Flags,
+%%                    #{is_local := false, is_unidir := true, conn := _Conn} = _S) ->
+%%     {stop, unimpl}.
+
+-spec passive(stream_handle(), undefined, cb_data()) -> cb_ret().
+passive(_Stream, undefined, _S) ->
+    {stop, unimpl}.
+
+-spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) -> cb_ret().
+stream_closed(
+    _Stream,
+    #{
+        is_conn_shutdown := IsConnShutdown,
+        is_app_closing := IsAppClosing,
+        is_shutdown_by_app := IsAppShutdown,
+        is_closed_remotely := IsRemote,
+        status := Status,
+        error := Code
+    },
+    S
+) when
+    is_boolean(IsConnShutdown) andalso
+        is_boolean(IsAppClosing) andalso
+        is_boolean(IsAppShutdown) andalso
+        is_boolean(IsRemote) andalso
+        is_atom(Status) andalso
+        is_integer(Code)
+->
+    %% @TODO for now we fake a sock_closed for
+    %% emqx_connection:process_msg to append
+    %% a msg to be processed
+    {ok, {sock_closed, Status}, S}.
+
+handle_call(_Stream, _Request, _Opts, S) ->
+    {error, unimpl, S}.
+
+%%%
+%%%  Internals
+%%%
+-spec socket(connection_handle(), stream_handle(), socket_info()) -> socket().
+socket(Conn, CtrlStream, Info) when is_map(Info) ->
+    {quic, Conn, CtrlStream, Info}.

From 9f696928b6174bb17f4505ee8a1f4e72df028dcb Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 25 Nov 2022 15:15:52 +0100
Subject: [PATCH 058/144] feat(quic): multi streams

---
 apps/emqx/src/emqx_channel.erl                |   1 +
 apps/emqx/src/emqx_connection.erl             |  37 +-
 apps/emqx/src/emqx_quic_connection.erl        | 139 +++++-
 apps/emqx/src/emqx_quic_data_stream.erl       | 466 ++++++++++++++++++
 apps/emqx/src/emqx_quic_stream.erl            |  43 +-
 .../emqx/test/emqx_mqtt_protocol_v5_SUITE.erl |  22 +-
 .../test/emqx_quic_multistreams_SUITE.erl     | 190 +++++++
 7 files changed, 848 insertions(+), 50 deletions(-)
 create mode 100644 apps/emqx/src/emqx_quic_data_stream.erl
 create mode 100644 apps/emqx/test/emqx_quic_multistreams_SUITE.erl

diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index e82adc786..a12df9c64 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -1136,6 +1136,7 @@ do_deliver(Publishes, Channel) when is_list(Publishes) ->
     {Packets, NChannel} =
         lists:foldl(
             fun(Publish, {Acc, Chann}) ->
+                %% @FIXME perf:  list append with copy left list
                 {Packets, NChann} = do_deliver(Publish, Chann),
                 {Packets ++ Acc, NChann}
             end,
diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 1c8b85808..980c41010 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -14,7 +14,12 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% MQTT/TCP|TLS Connection|QUIC Stream
+%% This module interacts with the transport layer of MQTT
+%% Transport:
+%%   - TCP connection
+%%   - TCP/TLS connection
+%%   - WebSocket
+%%   - QUIC Stream
 -module(emqx_connection).
 
 -include("emqx.hrl").
@@ -111,7 +116,13 @@
     limiter_buffer :: queue:queue(pending_req()),
 
     %% limiter timers
-    limiter_timer :: undefined | reference()
+    limiter_timer :: undefined | reference(),
+
+    %% QUIC conn pid if is a pid
+    quic_conn_pid :: maybe(pid()),
+
+    %% QUIC control stream callback state
+    quic_ctrl_state :: map()
 }).
 
 -record(retry, {
@@ -194,7 +205,7 @@
         {ok, pid()};
     (
         emqx_quic_stream,
-        {ConnOwner :: pid(), quicer:connection_handler(), quicer:new_conn_props()},
+        {ConnOwner :: pid(), quicer:connection_handle(), quicer:new_conn_props()},
         emqx_quic_connection:cb_state()
     ) ->
         {ok, pid()}.
@@ -334,6 +345,7 @@ init_state(
     },
     ParseState = emqx_frame:initial_parse_state(FrameOpts),
     Serialize = emqx_frame:serialize_opts(),
+    %% Init Channel
     Channel = emqx_channel:init(ConnInfo, Opts),
     GcState =
         case emqx_config:get_zone_conf(Zone, [force_gc]) of
@@ -364,7 +376,10 @@ init_state(
         zone = Zone,
         listener = Listener,
         limiter_buffer = queue:new(),
-        limiter_timer = undefined
+        limiter_timer = undefined,
+        %% for quic streams to inherit
+        quic_conn_pid = maps:get(conn_pid, Opts, undefined),
+        quic_ctrl_state = #{}
     }.
 
 run_loop(
@@ -600,9 +615,20 @@ handle_msg({inet_reply, _Sock, {error, Reason}}, State) ->
 handle_msg({connack, ConnAck}, State) ->
     handle_outgoing(ConnAck, State);
 handle_msg({close, Reason}, State) ->
+    %% @FIXME here it could be close due to appl error.
     ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}),
     handle_info({sock_closed, Reason}, close_socket(State));
-handle_msg({event, connected}, State = #state{channel = Channel}) ->
+handle_msg(
+    {event, connected},
+    State = #state{
+        channel = Channel,
+        serialize = Serialize,
+        parse_state = PS,
+        quic_conn_pid = QuicConnPid
+    }
+) ->
+    QuicConnPid =/= undefined andalso
+        emqx_quic_connection:activate_data_streams(QuicConnPid, {PS, Serialize, Channel}),
     ClientId = emqx_channel:info(clientid, Channel),
     emqx_cm:insert_channel_info(ClientId, info(State), stats(State));
 handle_msg({event, disconnected}, State = #state{channel = Channel}) ->
@@ -876,6 +902,7 @@ send(IoData, #state{transport = Transport, socket = Socket, channel = Channel})
             ok;
         Error = {error, _Reason} ->
             %% Send an inet_reply to postpone handling the error
+            %% @FIXME: why not just return error?
             self() ! {inet_reply, Socket, Error},
             ok
     end.
diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index a5af3d4b3..de7776429 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -14,6 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
+%% @doc impl. the quic connection owner process.
 -module(emqx_quic_connection).
 
 -include("logger.hrl").
@@ -41,15 +42,46 @@
     new_stream/3
 ]).
 
+-export([activate_data_streams/2]).
+
+-export([
+    handle_call/3,
+    handle_info/2
+]).
+
 -type cb_state() :: #{
+    %% connecion owner pid
+    conn_pid := pid(),
+    %% Pid of ctrl stream
     ctrl_pid := undefined | pid(),
+    %% quic connecion handle
     conn := undefined | quicer:conneciton_hanlder(),
+    %% streams that handoff from this process, excluding control stream
+    %% these streams could die/closed without effecting the connecion/session.
+
+    %@TODO type?
+    streams := [{pid(), quicer:stream_handle()}],
+    %% New stream opts
     stream_opts := map(),
+    %% If conneciton is resumed from session ticket
     is_resumed => boolean(),
+    %% mqtt message serializer config
+    serialize => undefined,
     _ => _
 }.
 -type cb_ret() :: quicer_lib:cb_ret().
 
+%% @doc  Data streams initializions are started in parallel with control streams, data streams are blocked
+%%       for the activation from control stream after it is accepted as a legit conneciton.
+%%       For security, the initial number of allowed data streams from client should be limited by
+%%       'peer_bidi_stream_count` & 'peer_unidi_stream_count`
+-spec activate_data_streams(pid(), {
+    emqx_frame:parse_state(), emqx_frame:serialize_opts(), emqx_channel:channel()
+}) -> ok.
+activate_data_streams(ConnOwner, {PS, Serialize, Channel}) ->
+    gen_server:call(ConnOwner, {activate_data_streams, {PS, Serialize, Channel}}, infinity).
+
+%% @doc conneciton owner init callback
 -spec init(map() | list()) -> {ok, cb_state()}.
 init(ConnOpts) when is_list(ConnOpts) ->
     init(maps:from_list(ConnOpts));
@@ -64,6 +96,7 @@ closed(_Conn, #{is_peer_acked := _} = Prop, S) ->
     ?SLOG(debug, Prop),
     {stop, normal, S}.
 
+%% @doc handle the new incoming connecion as the connecion acceptor.
 -spec new_conn(quicer:connection_handler(), quicer:new_conn_props(), cb_state()) ->
     {ok, cb_state()} | {error, any()}.
 new_conn(
@@ -75,15 +108,17 @@ new_conn(
     ?SLOG(debug, ConnInfo),
     case emqx_olp:is_overloaded() andalso is_zone_olp_enabled(Zone) of
         false ->
-            {ok, Pid} = emqx_connection:start_link(
+            %% Start control stream process
+            StartOption = S,
+            {ok, CtrlPid} = emqx_connection:start_link(
                 emqx_quic_stream,
                 {self(), Conn, maps:without([crypto_buffer], ConnInfo)},
-                S
+                StartOption
             ),
             receive
-                {Pid, stream_acceptor_ready} ->
+                {CtrlPid, stream_acceptor_ready} ->
                     ok = quicer:async_handshake(Conn),
-                    {ok, S#{conn := Conn, ctrl_pid := Pid}};
+                    {ok, S#{conn := Conn, ctrl_pid := CtrlPid}};
                 {'EXIT', _Pid, _Reason} ->
                     {error, stream_accept_error}
             end;
@@ -92,6 +127,7 @@ new_conn(
             {error, overloaded}
     end.
 
+%% @doc callback when connection is connected.
 -spec connected(quicer:connection_handler(), quicer:connected_props(), cb_state()) ->
     {ok, cb_state()} | {error, any()}.
 connected(Conn, Props, #{slow_start := false} = S) ->
@@ -102,6 +138,7 @@ connected(_Conn, Props, S) ->
     ?SLOG(debug, Props),
     {ok, S}.
 
+%% @doc callback when connection is resumed from 0-RTT
 -spec resumed(quicer:connection_handle(), SessionData :: binary() | false, cb_state()) -> cb_ret().
 resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when
     is_function(ResumeFun)
@@ -110,51 +147,77 @@ resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when
 resumed(_Conn, _Data, S) ->
     {ok, S#{is_resumed := true}}.
 
+%% @doc callback for receiving nst, should never happen on server.
 -spec nst_received(quicer:connection_handle(), TicketBin :: binary(), cb_state()) -> cb_ret().
 nst_received(_Conn, _Data, S) ->
     %% As server we should not recv NST!
     {stop, no_nst_for_server, S}.
 
+%% @doc callback for handling orphan data streams
+%%      depends on the connecion state and control stream state.
 -spec new_stream(quicer:stream_handle(), quicer:new_stream_props(), cb_state()) -> cb_ret().
 new_stream(
     Stream,
-    #{is_orphan := true} = Props,
+    #{is_orphan := true, flags := _Flags} = Props,
     #{
         conn := Conn,
         streams := Streams,
-        stream_opts := SOpts
-    } = CBState
+        stream_opts := SOpts,
+        zone := Zone,
+        limiter := Limiter,
+        parse_state := PS,
+        channel := Channel,
+        serialize := Serialize
+    } = S
 ) ->
-    %% Spawn new stream
-    case quicer_stream:start_link(emqx_quic_stream, Stream, Conn, SOpts, Props) of
-        {ok, StreamOwner} ->
-            quicer_connection:handoff_stream(Stream, StreamOwner),
-            {ok, CBState#{streams := [{StreamOwner, Stream} | Streams]}};
-        Other ->
-            Other
-    end.
+    %% Cherry pick options for data streams
+    SOpts1 = SOpts#{
+        is_local => false,
+        zone => Zone,
+        % unused
+        limiter => Limiter,
+        parse_state => PS,
+        channel => Channel,
+        serialize => Serialize
+    },
+    {ok, NewStreamOwner} = quicer_stream:start_link(
+        emqx_quic_data_stream,
+        Stream,
+        Conn,
+        SOpts1,
+        Props
+    ),
+    quicer:handoff_stream(Stream, NewStreamOwner, {PS, Serialize, Channel}),
+    %% @TODO keep them in ``inactive_streams'
+    {ok, S#{streams := [{NewStreamOwner, Stream} | Streams]}}.
 
+%% @doc callback for handling for remote connecion shutdown.
 -spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret().
 shutdown(Conn, _ErrorCode, S) ->
-    %% @TODO check spec what to do with the ErrorCode?
+    %% @TODO check spec what to set for the ErrorCode?
     quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
     {ok, S}.
 
+%% @doc callback for handling for transport error, such as idle timeout
 -spec transport_shutdown(quicer:connection_handle(), quicer:transport_shutdown_props(), cb_state()) ->
     cb_ret().
 transport_shutdown(_C, _DownInfo, S) ->
     %% @TODO some counter
     {ok, S}.
 
+%% @doc callback for handling for peer addr changed.
 -spec peer_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state) -> cb_ret().
 peer_address_changed(_C, _NewAddr, S) ->
+    %% @TODO update session info?
     {ok, S}.
 
+%% @doc callback for handling local addr change, currently unused
 -spec local_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state()) ->
     cb_ret().
 local_address_changed(_C, _NewAddr, S) ->
     {ok, S}.
 
+%% @doc callback for handling remote stream limit updates
 -spec streams_available(
     quicer:connection_handle(),
     {BidirStreams :: non_neg_integer(), UnidirStreams :: non_neg_integer()},
@@ -166,12 +229,43 @@ streams_available(_C, {BidirCnt, UnidirCnt}, S) ->
         peer_unidi_stream_count => UnidirCnt
     }}.
 
--spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret().
-%% @TODO this is not going to get triggered.
+%% @doc callback for handling request when remote wants for more streams
+%%      should cope with rate limiting
+%% @TODO this is not going to get triggered in current version
 %% for https://github.com/microsoft/msquic/issues/3120
+-spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret().
 peer_needs_streams(_C, undefined, S) ->
     {ok, S}.
 
+%% @doc handle API calls
+handle_call(
+    {activate_data_streams, {PS, Serialize, Channel} = ActivateData},
+    _From,
+    #{streams := Streams} = S
+) ->
+    [emqx_quic_data_stream:activate_data(OwnerPid, ActivateData) || {OwnerPid, _Stream} <- Streams],
+    {reply, ok, S#{
+        %streams := [], %% @FIXME what ??????
+        channel := Channel,
+        serialize := Serialize,
+        parse_state := PS
+    }};
+handle_call(_Req, _From, S) ->
+    {reply, {error, unimpl}, S}.
+
+%% @doc handle DOWN messages from streams.
+%% @TODO handle DOWN from supervisor?
+handle_info({'DOWN', _Ref, process, Pid, Reason}, #{streams := Streams} = S) when
+    Reason =:= normal orelse
+        Reason =:= {shutdown, protocol_error}
+->
+    case proplists:is_defined(Pid, Streams) of
+        true ->
+            {ok, S};
+        false ->
+            {stop, unknown_pid_down, S}
+    end.
+
 %%%
 %%%  Internals
 %%%
@@ -185,8 +279,13 @@ is_zone_olp_enabled(Zone) ->
     end.
 
 -spec init_cb_state(map()) -> cb_state().
-init_cb_state(Map) ->
+init_cb_state(#{zone := _Zone} = Map) ->
     Map#{
+        conn_pid => self(),
         ctrl_pid => undefined,
-        conn => undefined
+        conn => undefined,
+        streams => [],
+        parse_state => undefined,
+        channel => undefined,
+        serialize => undefined
     }.
diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl
new file mode 100644
index 000000000..72f0e913f
--- /dev/null
+++ b/apps/emqx/src/emqx_quic_data_stream.erl
@@ -0,0 +1,466 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%%
+%% @doc QUIC data stream
+%% Following the behaviour of emqx_connection:
+%%  The MQTT packets and their side effects are handled *atomically*.
+%%
+
+-module(emqx_quic_data_stream).
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("quicer/include/quicer.hrl").
+-include("emqx_mqtt.hrl").
+-include("logger.hrl").
+-behaviour(quicer_stream).
+
+%% Connection Callbacks
+-export([
+    init_handoff/4,
+    post_handoff/3,
+    new_stream/3,
+    start_completed/3,
+    send_complete/3,
+    peer_send_shutdown/3,
+    peer_send_aborted/3,
+    peer_receive_aborted/3,
+    send_shutdown_complete/3,
+    stream_closed/3,
+    peer_accepted/3,
+    passive/3
+]).
+
+-export([handle_stream_data/4]).
+
+-export([activate_data/2]).
+
+-export([
+    handle_call/3,
+    handle_info/2,
+    handle_continue/2
+]).
+
+%%
+%% @doc Activate the data handling.
+%%      Data handling is disabled before control stream allows the data processing.
+-spec activate_data(pid(), {
+    emqx_frame:parse_state(), emqx_frame:serialize_opts(), emqx_channel:channel()
+}) -> ok.
+activate_data(StreamPid, {PS, Serialize, Channel}) ->
+    gen_server:call(StreamPid, {activate, {PS, Serialize, Channel}}, infinity).
+
+%%
+%% @doc Handoff from previous owner, mostly from the connection owner.
+%% @TODO parse_state doesn't look necessary since we have it in post_handoff
+%% @TODO -spec
+init_handoff(
+    Stream,
+    #{parse_state := PS} = _StreamOpts,
+    Connection,
+    #{is_orphan := true, flags := Flags}
+) ->
+    {ok, init_state(Stream, Connection, Flags, PS)}.
+
+%%
+%% @doc Post handoff data stream
+%%
+%% @TODO -spec
+%%
+post_handoff(Stream, {PS, Serialize, Channel}, S) ->
+    ?tp(debug, ?FUNCTION_NAME, #{channel => Channel, serialize => Serialize}),
+    quicer:setopt(Stream, active, true),
+    {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}.
+
+%%
+%% @doc when this proc is assigned to the owner of new stream
+%%
+new_stream(Stream, #{flags := Flags}, Connection) ->
+    {ok, init_state(Stream, Connection, Flags)}.
+
+%%
+%% @doc for local initiated stream
+%%
+peer_accepted(_Stream, _Flags, S) ->
+    %% we just ignore it
+    {ok, S}.
+
+peer_receive_aborted(Stream, ErrorCode, #{is_unidir := false} = S) ->
+    %% we abort send with same reason
+    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
+    {ok, S};
+peer_receive_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := true} = S) ->
+    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
+    {ok, S}.
+
+peer_send_aborted(Stream, ErrorCode, #{is_unidir := false} = S) ->
+    %% we abort receive with same reason
+    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode),
+    {ok, S};
+peer_send_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := false} = S) ->
+    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode),
+    {ok, S}.
+
+peer_send_shutdown(Stream, _Flags, S) ->
+    ok = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0),
+    {ok, S}.
+
+send_complete(_Stream, false, S) ->
+    {ok, S};
+send_complete(_Stream, true = _IsCanceled, S) ->
+    {ok, S}.
+
+send_shutdown_complete(_Stream, _Flags, S) ->
+    {ok, S}.
+
+start_completed(_Stream, #{status := success, stream_id := StreamId}, S) ->
+    {ok, S#{stream_id => StreamId}};
+start_completed(_Stream, #{status := Other}, S) ->
+    %% or we could retry
+    {stop, {start_fail, Other}, S}.
+
+handle_stream_data(
+    Stream,
+    Bin,
+    _Flags,
+    #{
+        is_unidir := false,
+        channel := undefined,
+        data_queue := Queue,
+        stream := Stream
+    } = State
+) when is_binary(Bin) ->
+    {ok, State#{data_queue := [Bin | Queue]}};
+handle_stream_data(
+    _Stream,
+    Bin,
+    _Flags,
+    #{
+        is_unidir := false,
+        channel := Channel,
+        parse_state := PS,
+        data_queue := QueuedData,
+        task_queue := TQ
+    } = State
+) when
+    Channel =/= undefined
+->
+    {MQTTPackets, NewPS} = parse_incoming(list_to_binary(lists:reverse([Bin | QueuedData])), PS),
+    NewTQ = lists:foldl(
+        fun(Item, Acc) ->
+            queue:in(Item, Acc)
+        end,
+        TQ,
+        [{incoming, P} || P <- lists:reverse(MQTTPackets)]
+    ),
+    {{continue, handle_appl_msg}, State#{parse_state := NewPS, task_queue := NewTQ}}.
+
+%% Reserved for unidi streams
+%% handle_stream_data(Stream, Bin, _Flags, #{is_unidir := true, peer_stream := PeerStream, conn := Conn} = State) ->
+%%     case PeerStream of
+%%         undefined ->
+%%             {ok, StreamProc} = quicer_stream:start_link(?MODULE, Conn,
+%%                                                         [ {open_flag, ?QUIC_STREAM_OPEN_FLAG_UNIDIRECTIONAL}
+%%                                                         , {is_local, true}
+%%                                                         ]),
+%%             {ok, _} = quicer_stream:send(StreamProc, Bin),
+%%             {ok, State#{peer_stream := StreamProc}};
+%%         StreamProc when is_pid(StreamProc) ->
+%%             {ok, _} = quicer_stream:send(StreamProc, Bin),
+%%             {ok, State}
+%%     end.
+
+passive(_Stream, undefined, S) ->
+    {ok, S}.
+
+stream_closed(
+    _Stream,
+    #{
+        is_conn_shutdown := IsConnShutdown,
+        is_app_closing := IsAppClosing,
+        is_shutdown_by_app := IsAppShutdown,
+        is_closed_remotely := IsRemote,
+        status := Status,
+        error := Code
+    },
+    S
+) when
+    is_boolean(IsConnShutdown) andalso
+        is_boolean(IsAppClosing) andalso
+        is_boolean(IsAppShutdown) andalso
+        is_boolean(IsRemote) andalso
+        is_atom(Status) andalso
+        is_integer(Code)
+->
+    {stop, normal, S}.
+
+handle_call(Call, _From, S) ->
+    do_handle_call(Call, S).
+
+handle_continue(handle_appl_msg, #{task_queue := Q} = S) ->
+    case queue:out(Q) of
+        {{value, Item}, Q2} ->
+            do_handle_appl_msg(Item, S#{task_queue := Q2});
+        {empty, Q} ->
+            {ok, S}
+    end.
+
+do_handle_appl_msg(
+    {outgoing, Packets},
+    #{
+        channel := Channel,
+        stream := _Stream,
+        serialize := _Serialize
+    } = S
+) when
+    Channel =/= undefined
+->
+    case handle_outgoing(Packets, S) of
+        {ok, Size} ->
+            ok = emqx_metrics:inc('bytes.sent', Size),
+            {{continue, handle_appl_msg}, S};
+        {error, E1, E2} ->
+            {stop, {E1, E2}, S};
+        {error, E} ->
+            {stop, E, S}
+    end;
+do_handle_appl_msg({incoming, #mqtt_packet{} = Packet}, #{channel := Channel} = S) when
+    Channel =/= undefined
+->
+    with_channel(handle_in, [Packet], S);
+do_handle_appl_msg({close, Reason}, S) ->
+    %% @TODO shall we abort shutdown or graceful shutdown?
+    with_channel(handle_info, [{sock_closed, Reason}], S);
+do_handle_appl_msg({event, updated}, S) ->
+    %% Data stream don't care about connection state changes.
+    {{continue, handle_appl_msg}, S}.
+
+handle_info(Deliver = {deliver, _, _}, S) ->
+    Delivers = [Deliver],
+    with_channel(handle_deliver, [Delivers], S).
+
+with_channel(Fun, Args, #{channel := Channel, task_queue := Q} = S) when
+    Channel =/= undefined
+->
+    case apply(emqx_channel, Fun, Args ++ [Channel]) of
+        ok ->
+            {{continue, handle_appl_msg}, S};
+        {ok, Msgs, NewChannel} when is_list(Msgs) ->
+            {{continue, handle_appl_msg}, S#{
+                task_queue := queue:join(Q, queue:from_list(Msgs)),
+                channel := NewChannel
+            }};
+        {ok, Msg, NewChannel} when is_record(Msg, mqtt_packet) ->
+            {{continue, handle_appl_msg}, S#{
+                task_queue := queue:in({outgoing, Msg}, Q), channel := NewChannel
+            }};
+        %% @FIXME WTH?
+        {ok, {outgoing, _} = Msg, NewChannel} ->
+            {{continue, handle_appl_msg}, S#{task_queue := queue:in(Msg, Q), channel := NewChannel}};
+        {ok, NewChannel} ->
+            {{continue, handle_appl_msg}, S#{channel := NewChannel}};
+        %% @TODO optimisation for shutdown wrap
+        {shutdown, Reason, NewChannel} ->
+            {stop, {shutdown, Reason}, S#{channel := NewChannel}};
+        {shutdown, Reason, Msgs, NewChannel} when is_list(Msgs) ->
+            %% @TODO handle outgoing?
+            {stop, {shutdown, Reason}, S#{
+                channel := NewChannel,
+                task_queue := queue:join(Q, queue:from_list(Msgs))
+            }};
+        {shutdown, Reason, Msg, NewChannel} ->
+            {stop, {shutdown, Reason}, S#{
+                channel := NewChannel,
+                task_queue := queue:in(Msg, Q)
+            }}
+    end.
+
+%%% Internals
+handle_outgoing(#mqtt_packet{} = P, S) ->
+    handle_outgoing([P], S);
+handle_outgoing(Packets, #{serialize := Serialize, stream := Stream, is_unidir := false}) when
+    is_list(Packets)
+->
+    OutBin = [serialize_packet(P, Serialize) || P <- filter_disallowed_out(Packets)],
+    %% @TODO in which case shall we use sync send?
+    Res = quicer:async_send(Stream, OutBin),
+    ?TRACE("MQTT", "mqtt_packet_sent", #{packets => Packets}),
+    [ok = inc_outgoing_stats(P) || P <- Packets],
+    Res.
+
+serialize_packet(Packet, Serialize) ->
+    try emqx_frame:serialize_pkt(Packet, Serialize) of
+        <<>> ->
+            ?SLOG(warning, #{
+                msg => "packet_is_discarded",
+                reason => "frame_is_too_large",
+                packet => emqx_packet:format(Packet, hidden)
+            }),
+            ok = emqx_metrics:inc('delivery.dropped.too_large'),
+            ok = emqx_metrics:inc('delivery.dropped'),
+            ok = inc_outgoing_stats({error, message_too_large}),
+            <<>>;
+        Data ->
+            Data
+    catch
+        %% Maybe Never happen.
+        throw:{?FRAME_SERIALIZE_ERROR, Reason} ->
+            ?SLOG(info, #{
+                reason => Reason,
+                input_packet => Packet
+            }),
+            erlang:error({?FRAME_SERIALIZE_ERROR, Reason});
+        error:Reason:Stacktrace ->
+            ?SLOG(error, #{
+                input_packet => Packet,
+                exception => Reason,
+                stacktrace => Stacktrace
+            }),
+            erlang:error(?FRAME_SERIALIZE_ERROR)
+    end.
+
+-spec init_state(
+    quicer:stream_handle(),
+    quicer:connection_handle(),
+    quicer:new_stream_props()
+) ->
+    % @TODO
+    map().
+init_state(Stream, Connection, OpenFlags) ->
+    init_state(Stream, Connection, OpenFlags, undefined).
+
+init_state(Stream, Connection, OpenFlags, PS) ->
+    %% quic stream handle
+    #{
+        stream => Stream,
+        %% quic connection handle
+        conn => Connection,
+        %% if it is QUIC unidi stream
+        is_unidir => quicer:is_unidirectional(OpenFlags),
+        %% Frame Parse State
+        parse_state => PS,
+        %% Peer Stream handle in a pair for type unidir only
+        peer_stream => undefined,
+        %% if the stream is locally initiated.
+        is_local => false,
+        %% queue binary data when is NOT connected, in reversed order.
+        data_queue => [],
+        %% Channel from connection
+        %% `undefined' means the connection is not connected.
+        channel => undefined,
+        %% serialize opts for connection
+        serialize => undefined,
+        %% Current working queue
+        task_queue => queue:new()
+    }.
+
+-spec do_handle_call(term(), quicer_stream:cb_state()) -> quicer_stream:cb_ret().
+do_handle_call(
+    {activate, {PS, Serialize, Channel}},
+    #{
+        channel := undefined,
+        stream := Stream,
+        serialize := undefined
+    } = S
+) ->
+    NewS = S#{channel := Channel, serialize := Serialize, parse_state := PS},
+    %% We use quic protocol for flow control, and we don't check return val
+    case quicer:setopt(Stream, active, true) of
+        ok ->
+            {ok, NewS};
+        {error, E} ->
+            ?SLOG(error, #{msg => "set stream active failed", error => E}),
+            {stop, E, NewS}
+    end;
+do_handle_call(_Call, S) ->
+    {reply, {error, unimpl}, S}.
+
+%% @doc return reserved order of Packets
+parse_incoming(Data, PS) ->
+    try
+        do_parse_incoming(Data, [], PS)
+    catch
+        throw:{?FRAME_PARSE_ERROR, Reason} ->
+            ?SLOG(info, #{
+                reason => Reason,
+                input_bytes => Data
+            }),
+            {[{frame_error, Reason}], PS};
+        error:Reason:Stacktrace ->
+            ?SLOG(error, #{
+                input_bytes => Data,
+                reason => Reason,
+                stacktrace => Stacktrace
+            }),
+            {[{frame_error, Reason}], PS}
+    end.
+
+do_parse_incoming(<<>>, Packets, ParseState) ->
+    {Packets, ParseState};
+do_parse_incoming(Data, Packets, ParseState) ->
+    case emqx_frame:parse(Data, ParseState) of
+        {more, NParseState} ->
+            {Packets, NParseState};
+        {ok, Packet, Rest, NParseState} ->
+            do_parse_incoming(Rest, [Packet | Packets], NParseState)
+    end.
+
+%% followings are copied from emqx_connection
+-compile({inline, [inc_outgoing_stats/1]}).
+inc_outgoing_stats({error, message_too_large}) ->
+    inc_counter('send_msg.dropped', 1),
+    inc_counter('send_msg.dropped.too_large', 1);
+inc_outgoing_stats(Packet = ?PACKET(Type)) ->
+    inc_counter(send_pkt, 1),
+    case Type of
+        ?PUBLISH ->
+            inc_counter(send_msg, 1),
+            inc_counter(outgoing_pubs, 1),
+            inc_qos_stats(send_msg, Packet);
+        _ ->
+            ok
+    end,
+    emqx_metrics:inc_sent(Packet).
+
+inc_counter(Key, Inc) ->
+    _ = emqx_pd:inc_counter(Key, Inc),
+    ok.
+
+inc_qos_stats(Type, Packet) ->
+    case inc_qos_stats_key(Type, emqx_packet:qos(Packet)) of
+        undefined ->
+            ignore;
+        Key ->
+            inc_counter(Key, 1)
+    end.
+
+inc_qos_stats_key(send_msg, ?QOS_0) -> 'send_msg.qos0';
+inc_qos_stats_key(send_msg, ?QOS_1) -> 'send_msg.qos1';
+inc_qos_stats_key(send_msg, ?QOS_2) -> 'send_msg.qos2';
+inc_qos_stats_key(recv_msg, ?QOS_0) -> 'recv_msg.qos0';
+inc_qos_stats_key(recv_msg, ?QOS_1) -> 'recv_msg.qos1';
+inc_qos_stats_key(recv_msg, ?QOS_2) -> 'recv_msg.qos2';
+%% for bad qos
+inc_qos_stats_key(_, _) -> undefined.
+
+filter_disallowed_out(Packets) ->
+    lists:filter(fun is_datastream_out_pkt/1, Packets).
+
+is_datastream_out_pkt(#mqtt_packet{header = #mqtt_packet_header{type = Type}}) when
+    Type > 2 andalso Type < 12
+->
+    true;
+is_datastream_out_pkt(_) ->
+    false.
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index d9c080c0d..70b01e643 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -78,17 +78,24 @@
 -type socket_info() :: #{
     is_orphan => boolean(),
     ctrl_stream_start_flags => quicer:stream_open_flags(),
-    %% quicer:new_conn_props
+    %% and quicer:new_conn_props()
     _ => _
 }.
 
--spec wait({pid(), quicer:connection_handle(), socket_info()}) ->
-    {ok, socket()} | {error, enotconn}.
+%% for accepting
+-spec wait
+    ({pid(), connection_handle(), socket_info()}) ->
+        {ok, socket()} | {error, enotconn};
+    %% For handover
+    ({pid(), connection_handle(), stream_handle(), socket_info()}) ->
+        {ok, socket()} | {error, any()}.
+
+%%% For Accepting New Remote Stream
 wait({ConnOwner, Conn, ConnInfo}) ->
     {ok, Conn} = quicer:async_accept_stream(Conn, []),
     ConnOwner ! {self(), stream_acceptor_ready},
     receive
-        %% New incoming stream, this is a *ctrl* stream
+        %% New incoming stream, this is a *control* stream
         {quic, new_stream, Stream, #{is_orphan := IsOrphan, flags := StartFlags}} ->
             SocketInfo = ConnInfo#{
                 is_orphan => IsOrphan,
@@ -101,6 +108,14 @@ wait({ConnOwner, Conn, ConnInfo}) ->
         %% Connection owner process down
         {'EXIT', ConnOwner, _Reason} ->
             {error, enotconn}
+    end;
+%% For ownership handover
+wait({PrevOwner, Conn, Stream, SocketInfo}) ->
+    case quicer:wait_for_handoff(PrevOwner, Stream) of
+        ok ->
+            {ok, socket(Conn, Stream, SocketInfo)};
+        owner_down ->
+            {error, owner_down}
     end.
 
 type(_) ->
@@ -144,9 +159,10 @@ getopts(_Socket, _Opts) ->
         {buffer, 80000}
     ]}.
 
-fast_close({quic, _Conn, Stream, _Info}) ->
-    %% Flush send buffer, gracefully shutdown
-    quicer:async_shutdown_stream(Stream),
+fast_close({quic, Conn, _Stream, _Info}) ->
+    %% Since we shutdown the control stream, we shutdown the connection as well
+    %% @TODO supply some App Error Code
+    quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
     ok.
 
 -spec ensure_ok_or_exit(atom(), list(term())) -> term().
@@ -187,21 +203,14 @@ peer_accepted(_Stream, undefined, S) ->
     {ok, S}.
 
 -spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret().
-peer_receive_aborted(Stream, ErrorCode, #{is_unidir := false} = S) ->
-    %% we abort send with same reason
-    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
-    {ok, S};
-peer_receive_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := true} = S) ->
+peer_receive_aborted(Stream, ErrorCode, S) ->
     quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
     {ok, S}.
 
 -spec peer_send_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret().
-peer_send_aborted(Stream, ErrorCode, #{is_unidir := false} = S) ->
+peer_send_aborted(Stream, ErrorCode, S) ->
     %% we abort receive with same reason
-    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode),
-    {ok, S};
-peer_send_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := false} = S) ->
-    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode),
+    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
     {ok, S}.
 
 -spec peer_send_shutdown(stream_handle(), undefined, cb_data()) -> cb_ret().
diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
index 07299bd42..0b493dff6 100644
--- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
+++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
@@ -65,6 +65,7 @@ init_per_group(quic, Config) ->
     UdpPort = 1884,
     emqx_common_test_helpers:start_apps([]),
     emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort),
+    emqx_logger:set_log_level(debug),
     [{port, UdpPort}, {conn_fun, quic_connect} | Config];
 init_per_group(_, Config) ->
     emqx_common_test_helpers:stop_apps([]),
@@ -78,14 +79,19 @@ end_per_group(_Group, _Config) ->
 
 init_per_suite(Config) ->
     %% Start Apps
-    %% dbg:tracer(process, {fun dbg:dhandler/2,group_leader()}),
-    %% dbg:p(all,c),
-    %% dbg:tp(emqx_quic_connection,cx),
-    %% dbg:tp(emqx_quic_stream,cx),
-    %% dbg:tp(emqtt_quic,cx),
-    %% dbg:tp(emqtt,cx),
-    %% dbg:tp(emqtt_quic_stream,cx),
-    %% dbg:tp(emqtt_quic_connection,cx),
+    dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}),
+    dbg:p(all, c),
+    dbg:tp(emqx_quic_connection, cx),
+    dbg:tp(quicer_connection, cx),
+    %% dbg:tp(emqx_quic_stream, cx),
+    %% dbg:tp(emqtt_quic, cx),
+    %% dbg:tp(emqtt, cx),
+    %% dbg:tp(emqtt_quic_stream, cx),
+    %% dbg:tp(emqtt_quic_connection, cx),
+    %% dbg:tp(emqx_cm, open_session, cx),
+    %% dbg:tpl(emqx_cm, lookup_channels, cx),
+    %% dbg:tpl(emqx_cm, register_channel, cx),
+    %% dbg:tpl(emqx_cm, unregister_channel, cx),
     emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
     Config.
diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
new file mode 100644
index 000000000..bb19092f7
--- /dev/null
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -0,0 +1,190 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_quic_multistreams_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-define(TOPICS, [
+    <<"TopicA">>,
+    <<"TopicA/B">>,
+    <<"Topic/C">>,
+    <<"TopicA/C">>,
+    <<"/TopicA">>
+]).
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+    [{timetrap, {seconds, 30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%%     Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+    UdpPort = 1884,
+    emqx_common_test_helpers:boot_modules(all),
+    emqx_common_test_helpers:start_apps([]),
+    emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort),
+    %% @TODO remove
+    emqx_logger:set_log_level(debug),
+
+    dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}),
+    dbg:p(all, c),
+
+    %dbg:tp(emqx_quic_stream, cx),
+    %% dbg:tp(quicer_stream, cx),
+    %% dbg:tp(emqx_quic_data_stream, cx),
+    %% dbg:tp(emqx_channel, cx),
+    %% dbg:tp(emqx_packet,check,cx),
+    %% dbg:tp(emqx_frame,parse,cx),
+    %dbg:tp(emqx_quic_connection, cx),
+    [{port, UdpPort}, {conn_fun, quic_connect} | Config].
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> term() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+    Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%%               term() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%%               term() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%%              repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+    [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+    [
+        tc_data_stream_sub
+    ].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%%               ok | exit() | {skip,Reason} | {comment,Comment} |
+%%               {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+
+%% @doc Test MQTT Subscribe via data_stream
+tc_data_stream_sub(Config) ->
+    Topic = lists:nth(1, ?TOPICS),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [1]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [{Topic, [{qos, qos1}]}]),
+    {ok, _, [2]} = emqtt:subscribe_via(
+        C,
+        {new_data_stream, []},
+        #{},
+        [{lists:nth(2, ?TOPICS), [{qos, qos2}]}]
+    ),
+    {ok, _} = emqtt:publish(C, Topic, <<"qos 2 1">>, 2),
+    {ok, _} = emqtt:publish(C, Topic, <<"qos 2 2">>, 2),
+    {ok, _} = emqtt:publish(C, Topic, <<"qos 2 3">>, 2),
+    Msgs = receive_messages(3),
+    ct:pal("recv msg: ~p", [Msgs]),
+    ?assertEqual(3, length(Msgs)),
+    ok = emqtt:disconnect(C).
+
+receive_messages(Count) ->
+    receive_messages(Count, []).
+
+receive_messages(0, Msgs) ->
+    Msgs;
+receive_messages(Count, Msgs) ->
+    receive
+        {publish, Msg} ->
+            receive_messages(Count - 1, [Msg | Msgs]);
+        _Other ->
+            receive_messages(Count, Msgs)
+    after 1000 ->
+        Msgs
+    end.

From 9b52beaee92b3d22d9402bc948dc41874ca785e2 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 5 Dec 2022 11:04:03 +0100
Subject: [PATCH 059/144] fix(quic): handle fast_close while handshake fail

---
 apps/emqx/src/emqx_quic_stream.erl | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index 70b01e643..2469a2ea7 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -159,9 +159,13 @@ getopts(_Socket, _Opts) ->
         {buffer, 80000}
     ]}.
 
+%% @TODO supply some App Error Code
+fast_close({ConnOwner, Conn, _ConnInfo}) when is_pid(ConnOwner) ->
+    %% handshake aborted.
+    quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
+    ok;
 fast_close({quic, Conn, _Stream, _Info}) ->
     %% Since we shutdown the control stream, we shutdown the connection as well
-    %% @TODO supply some App Error Code
     quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
     ok.
 

From 7d9bd33de9a3452868f7281e33b2c97d64504b9f Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 7 Dec 2022 13:54:07 +0100
Subject: [PATCH 060/144] feat(quic): bump quicer version to 0.0.100

---
 apps/emqx/rebar.config.script | 2 +-
 mix.exs                       | 2 +-
 rebar.config.erl              | 3 +--
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index 0ecd21715..3ac2b8758 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/multi-streams"}}}. %% @TODO revert
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.100"}}}.
 
 ExtraDeps = fun(C) ->
     {deps, Deps0} = lists:keyfind(deps, 1, C),
diff --git a/mix.exs b/mix.exs
index a2df76701..ce798997b 100644
--- a/mix.exs
+++ b/mix.exs
@@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
-      do: [{:quicer, github: "emqx/quic", tag: "0.0.16", override: true}],
+      do: [{:quicer, github: "emqx/quic", tag: "0.0.100", override: true}],
       else: []
   end
 
diff --git a/rebar.config.erl b/rebar.config.erl
index 9da71355b..1d342b403 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,8 +39,7 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    %% @TODO revert
-    {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/multi-streams"}}}.
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.100"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From 04a8a49dbecc63dde7671ca1f8ce80aa04db3f09 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 13 Dec 2022 09:49:02 +0100
Subject: [PATCH 061/144] test: update testcase for new emqtt

---
 apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
index 0b493dff6..5a9abc7f4 100644
--- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
+++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
@@ -919,7 +919,7 @@ t_shared_subscriptions_client_terminates_when_qos_eq_2(Config) ->
         emqtt,
         connected,
         fun
-            (cast, ?PUBLISH_PACKET(?QOS_2, _PacketId), _State) ->
+            (cast, {?PUBLISH_PACKET(?QOS_2, _PacketId), _Via}, _State) ->
                 ok = counters:add(CRef, 1, 1),
                 {stop, {shutdown, for_testing}};
             (Arg1, ARg2, Arg3) ->

From 5bdcb0562d988e0cd5f01d6e188376cb93ea446a Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 14 Dec 2022 23:30:13 +0100
Subject: [PATCH 062/144] feat(quic): workaround to flushing the send buffer
 after conn shutdown

Could not find a way to ensure msquic flush the send buffer after
calling ConnectionShutdown.

So just close the ctrl stream and let conn owner shutdown the conn.
---
 apps/emqx/src/emqx_quic_connection.erl |  9 +++++++++
 apps/emqx/src/emqx_quic_stream.erl     | 10 +++++++---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index de7776429..480f59e3a 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -255,6 +255,15 @@ handle_call(_Req, _From, S) ->
 
 %% @doc handle DOWN messages from streams.
 %% @TODO handle DOWN from supervisor?
+handle_info({'DOWN', _Ref, process, Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) ->
+    case Reason of
+        normal ->
+            quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0);
+        _ ->
+            %% @TODO have some reasons mappings here.
+            quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 1)
+    end,
+    {ok, S};
 handle_info({'DOWN', _Ref, process, Pid, Reason}, #{streams := Streams} = S) when
     Reason =:= normal orelse
         Reason =:= {shutdown, protocol_error}
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index 2469a2ea7..555637d20 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -164,9 +164,13 @@ fast_close({ConnOwner, Conn, _ConnInfo}) when is_pid(ConnOwner) ->
     %% handshake aborted.
     quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
     ok;
-fast_close({quic, Conn, _Stream, _Info}) ->
-    %% Since we shutdown the control stream, we shutdown the connection as well
-    quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
+fast_close({quic, _Conn, Stream, _Info}) ->
+    %% Force flush
+    quicer:async_shutdown_stream(Stream),
+    %% @FIXME Since we shutdown the control stream, we shutdown the connection as well
+    %% *BUT* Msquic does not flush the send buffer if we shutdown the connection after
+    %% gracefully shutdown the stream.
+    % quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
     ok.
 
 -spec ensure_ok_or_exit(atom(), list(term())) -> term().

From 1840a7f9237e75e10ee3f2d4b12e6d267608ec88 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 15 Dec 2022 22:23:10 +0100
Subject: [PATCH 063/144] test(quic): improve coverage

---
 apps/emqx/include/emqx_quic.hrl         |   24 +
 apps/emqx/src/emqx_quic_connection.erl  |   33 +-
 apps/emqx/src/emqx_quic_data_stream.erl |    4 +
 apps/emqx/src/emqx_quic_stream.erl      |   16 +-
 apps/emqx/test/emqtt_quic_SUITE.erl     | 1291 +++++++++++++++++++++++
 5 files changed, 1344 insertions(+), 24 deletions(-)
 create mode 100644 apps/emqx/include/emqx_quic.hrl
 create mode 100644 apps/emqx/test/emqtt_quic_SUITE.erl

diff --git a/apps/emqx/include/emqx_quic.hrl b/apps/emqx/include/emqx_quic.hrl
new file mode 100644
index 000000000..302f2704d
--- /dev/null
+++ b/apps/emqx/include/emqx_quic.hrl
@@ -0,0 +1,24 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-ifndef(EMQX_QUIC_HRL).
+-define(EMQX_QUIC_HRL, true).
+
+%% MQTT Over QUIC Shutdown Error code.
+-define(MQTT_QUIC_CONN_NOERROR, 0).
+-define(MQTT_QUIC_CONN_ERROR_OVERLOADED, 2).
+
+-endif.
diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index 480f59e3a..e632e6b1a 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -20,6 +20,7 @@
 -include("logger.hrl").
 -ifndef(BUILD_WITHOUT_QUIC).
 -include_lib("quicer/include/quicer.hrl").
+-include_lib("emqx/include/emqx_quic.hrl").
 -else.
 -define(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0).
 -endif.
@@ -36,9 +37,9 @@
     local_address_changed/3,
     peer_address_changed/3,
     streams_available/3,
-    peer_needs_streams/3,
+    % @TODO wait for newer quicer
+    %peer_needs_streams/3,
     resumed/3,
-    nst_received/3,
     new_stream/3
 ]).
 
@@ -120,11 +121,16 @@ new_conn(
                     ok = quicer:async_handshake(Conn),
                     {ok, S#{conn := Conn, ctrl_pid := CtrlPid}};
                 {'EXIT', _Pid, _Reason} ->
-                    {error, stream_accept_error}
+                    {stop, stream_accept_error, S}
             end;
         true ->
             emqx_metrics:inc('olp.new_conn'),
-            {error, overloaded}
+            quicer:async_shutdown_connection(
+                Conn,
+                ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE,
+                ?MQTT_QUIC_CONN_ERROR_OVERLOADED
+            ),
+            {stop, normal, S}
     end.
 
 %% @doc callback when connection is connected.
@@ -132,8 +138,8 @@ new_conn(
     {ok, cb_state()} | {error, any()}.
 connected(Conn, Props, #{slow_start := false} = S) ->
     ?SLOG(debug, Props),
-    {ok, _Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S),
-    {ok, S};
+    {ok, Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S),
+    {ok, S#{ctrl_pid => Pid}};
 connected(_Conn, Props, S) ->
     ?SLOG(debug, Props),
     {ok, S}.
@@ -147,12 +153,6 @@ resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when
 resumed(_Conn, _Data, S) ->
     {ok, S#{is_resumed := true}}.
 
-%% @doc callback for receiving nst, should never happen on server.
--spec nst_received(quicer:connection_handle(), TicketBin :: binary(), cb_state()) -> cb_ret().
-nst_received(_Conn, _Data, S) ->
-    %% As server we should not recv NST!
-    {stop, no_nst_for_server, S}.
-
 %% @doc callback for handling orphan data streams
 %%      depends on the connecion state and control stream state.
 -spec new_stream(quicer:stream_handle(), quicer:new_stream_props(), cb_state()) -> cb_ret().
@@ -233,9 +233,9 @@ streams_available(_C, {BidirCnt, UnidirCnt}, S) ->
 %%      should cope with rate limiting
 %% @TODO this is not going to get triggered in current version
 %% for https://github.com/microsoft/msquic/issues/3120
--spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret().
-peer_needs_streams(_C, undefined, S) ->
-    {ok, S}.
+%% -spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret().
+%% peer_needs_streams(_C, undefined, S) ->
+%%     {ok, S}.
 
 %% @doc handle API calls
 handle_call(
@@ -296,5 +296,6 @@ init_cb_state(#{zone := _Zone} = Map) ->
         streams => [],
         parse_state => undefined,
         channel => undefined,
-        serialize => undefined
+        serialize => undefined,
+        is_resumed => false
     }.
diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl
index 72f0e913f..094680b19 100644
--- a/apps/emqx/src/emqx_quic_data_stream.erl
+++ b/apps/emqx/src/emqx_quic_data_stream.erl
@@ -240,6 +240,10 @@ do_handle_appl_msg({incoming, #mqtt_packet{} = Packet}, #{channel := Channel} =
     Channel =/= undefined
 ->
     with_channel(handle_in, [Packet], S);
+do_handle_appl_msg({incoming, {frame_error, _} = FE}, #{channel := Channel} = S) when
+    Channel =/= undefined
+->
+    with_channel(handle_in, [FE], S);
 do_handle_appl_msg({close, Reason}, S) ->
     %% @TODO shall we abort shutdown or graceful shutdown?
     with_channel(handle_info, [{sock_closed, Reason}], S);
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index 555637d20..714ef337f 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -108,15 +108,15 @@ wait({ConnOwner, Conn, ConnInfo}) ->
         %% Connection owner process down
         {'EXIT', ConnOwner, _Reason} ->
             {error, enotconn}
-    end;
-%% For ownership handover
-wait({PrevOwner, Conn, Stream, SocketInfo}) ->
-    case quicer:wait_for_handoff(PrevOwner, Stream) of
-        ok ->
-            {ok, socket(Conn, Stream, SocketInfo)};
-        owner_down ->
-            {error, owner_down}
     end.
+%% UNUSED, for ownership handover,
+%% wait({PrevOwner, Conn, Stream, SocketInfo}) ->
+%%     case quicer:wait_for_handoff(PrevOwner, Stream) of
+%%         ok ->
+%%             {ok, socket(Conn, Stream, SocketInfo)};
+%%         owner_down ->
+%%             {error, owner_down}
+%%     end.
 
 type(_) ->
     quic.
diff --git a/apps/emqx/test/emqtt_quic_SUITE.erl b/apps/emqx/test/emqtt_quic_SUITE.erl
new file mode 100644
index 000000000..28e9bcd7b
--- /dev/null
+++ b/apps/emqx/test/emqtt_quic_SUITE.erl
@@ -0,0 +1,1291 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqtt_quic_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("quicer/include/quicer.hrl").
+
+suite() ->
+    [{timetrap, {seconds, 30}}].
+
+all() ->
+    [
+        {group, mstream},
+        {group, shutdown},
+        {group, misc}
+    ].
+
+groups() ->
+    [
+        {mstream, [], [{group, profiles}]},
+
+        {profiles, [], [
+            {group, profile_low_latency},
+            {group, profile_max_throughput}
+        ]},
+        {profile_low_latency, [], [
+            {group, pub_qos0},
+            {group, pub_qos1},
+            {group, pub_qos2}
+        ]},
+        {profile_max_throughput, [], [
+            {group, pub_qos0},
+            {group, pub_qos1},
+            {group, pub_qos2}
+        ]},
+        {pub_qos0, [], [
+            {group, sub_qos0},
+            {group, sub_qos1},
+            {group, sub_qos2}
+        ]},
+        {pub_qos1, [], [
+            {group, sub_qos0},
+            {group, sub_qos1},
+            {group, sub_qos2}
+        ]},
+        {pub_qos2, [], [
+            {group, sub_qos0},
+            {group, sub_qos1},
+            {group, sub_qos2}
+        ]},
+        {sub_qos0, [{group, qos}]},
+        {sub_qos1, [{group, qos}]},
+        {sub_qos2, [{group, qos}]},
+        {qos, [
+            t_multi_streams_sub,
+            t_multi_streams_pub_parallel,
+            t_multi_streams_sub_pub_async,
+            t_multi_streams_sub_pub_sync,
+            t_multi_streams_unsub,
+            t_multi_streams_corr_topic,
+            t_multi_streams_unsub_via_other,
+            t_multi_streams_shutdown_data_stream_abortive,
+            t_multi_streams_dup_sub,
+            t_multi_streams_packet_boundary,
+            t_multi_streams_packet_malform
+        ]},
+
+        {shutdown, [
+            {group, graceful_shutdown},
+            {group, abort_recv_shutdown},
+            {group, abort_send_shutdown},
+            {group, abort_send_recv_shutdown}
+        ]},
+
+        {graceful_shutdown, [{group, ctrl_stream_shutdown}]},
+        {abort_recv_shutdown, [{group, ctrl_stream_shutdown}]},
+        {abort_send_shutdown, [{group, ctrl_stream_shutdown}]},
+        {abort_send_recv_shutdown, [{group, ctrl_stream_shutdown}]},
+
+        {ctrl_stream_shutdown, [
+            t_multi_streams_shutdown_ctrl_stream,
+            t_multi_streams_shutdown_ctrl_stream_then_reconnect,
+            t_multi_streams_remote_shutdown,
+            t_multi_streams_remote_shutdown_with_reconnect
+        ]},
+        {misc, [
+            t_conn_silent_close,
+            t_client_conn_bump_streams,
+            t_olp_true,
+            t_olp_reject,
+            t_conn_resume,
+            t_conn_without_ctrl_stream
+        ]}
+    ].
+
+init_per_suite(Config) ->
+    emqx_common_test_helpers:start_apps([]),
+    UdpPort = 14567,
+    start_emqx_quic(UdpPort),
+    %% dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}),
+    %% dbg:p(all, c),
+    %% dbg:tp(emqx_quic_connection, cx),
+    %% dbg:tp(emqx_quic_stream, cx),
+    %% dbg:tp(emqtt, cx),
+    %% dbg:tpl(emqtt_quic_stream, cx),
+    %% dbg:tpl(emqx_quic_stream, cx),
+    %% dbg:tpl(emqx_quic_data_stream, cx),
+    %% dbg:tpl(emqtt, cx),
+    [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config].
+
+end_per_suite(_) ->
+    ok.
+
+init_per_group(pub_qos0, Config) ->
+    [{pub_qos, 0} | Config];
+init_per_group(sub_qos0, Config) ->
+    [{sub_qos, 0} | Config];
+init_per_group(pub_qos1, Config) ->
+    [{pub_qos, 1} | Config];
+init_per_group(sub_qos1, Config) ->
+    [{sub_qos, 1} | Config];
+init_per_group(pub_qos2, Config) ->
+    [{pub_qos, 2} | Config];
+init_per_group(sub_qos2, Config) ->
+    [{sub_qos, 2} | Config];
+init_per_group(abort_send_shutdown, Config) ->
+    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND} | Config];
+init_per_group(abort_recv_shutdown, Config) ->
+    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE} | Config];
+init_per_group(abort_send_recv_shutdown, Config) ->
+    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT} | Config];
+init_per_group(graceful_shutdown, Config) ->
+    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL} | Config];
+init_per_group(profile_max_throughput, Config) ->
+    quicer:reg_open(quic_execution_profile_type_max_throughput),
+    Config;
+init_per_group(profile_low_latency, Config) ->
+    quicer:reg_open(quic_execution_profile_low_latency),
+    Config;
+init_per_group(_, Config) ->
+    Config.
+
+end_per_group(_, Config) ->
+    Config.
+
+init_per_testcase(_, Config) ->
+    emqx_common_test_helpers:start_apps([]),
+    Config.
+
+t_quic_sock(Config) ->
+    Port = 4567,
+    SslOpts = [
+        {cert, certfile(Config)},
+        {key, keyfile(Config)},
+        {idle_timeout_ms, 10000},
+        % QUIC_SERVER_RESUME_AND_ZERORTT
+        {server_resumption_level, 2},
+        {peer_bidi_stream_count, 10},
+        {alpn, ["mqtt"]}
+    ],
+    Server = quic_server:start_link(Port, SslOpts),
+    timer:sleep(500),
+    {ok, Sock} = emqtt_quic:connect(
+        "localhost",
+        Port,
+        [{alpn, ["mqtt"]}, {active, false}],
+        3000
+    ),
+    send_and_recv_with(Sock),
+    ok = emqtt_quic:close(Sock),
+    quic_server:stop(Server).
+
+t_quic_sock_fail(_Config) ->
+    Port = 4567,
+    Error1 =
+        {error,
+            {transport_down, #{
+                error => 2,
+                status => connection_refused
+            }}},
+    Error2 = {error, {transport_down, #{error => 1, status => unreachable}}},
+    case
+        emqtt_quic:connect(
+            "localhost",
+            Port,
+            [{alpn, ["mqtt"]}, {active, false}],
+            3000
+        )
+    of
+        Error1 ->
+            ok;
+        Error2 ->
+            ok;
+        Other ->
+            ct:fail("unexpected return ~p", [Other])
+    end.
+
+t_0_rtt(Config) ->
+    Port = 4568,
+    SslOpts = [
+        {cert, certfile(Config)},
+        {key, keyfile(Config)},
+        {idle_timeout_ms, 10000},
+        % QUIC_SERVER_RESUME_AND_ZERORTT
+        {server_resumption_level, 2},
+        {peer_bidi_stream_count, 10},
+        {alpn, ["mqtt"]}
+    ],
+    Server = quic_server:start_link(Port, SslOpts),
+    timer:sleep(500),
+    {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect(
+        "localhost",
+        Port,
+        [
+            {alpn, ["mqtt"]},
+            {active, false},
+            {quic_event_mask, 1}
+        ],
+        3000
+    ),
+    send_and_recv_with(Sock),
+    ok = emqtt_quic:close(Sock),
+    NST =
+        receive
+            {quic, nst_received, Conn, Ticket} ->
+                Ticket
+        end,
+    {ok, Sock2} = emqtt_quic:connect(
+        "localhost",
+        Port,
+        [
+            {alpn, ["mqtt"]},
+            {active, false},
+            {nst, NST}
+        ],
+        3000
+    ),
+    send_and_recv_with(Sock2),
+    ok = emqtt_quic:close(Sock2),
+    quic_server:stop(Server).
+
+t_0_rtt_fail(Config) ->
+    Port = 4569,
+    SslOpts = [
+        {cert, certfile(Config)},
+        {key, keyfile(Config)},
+        {idle_timeout_ms, 10000},
+        % QUIC_SERVER_RESUME_AND_ZERORTT
+        {server_resumption_level, 2},
+        {peer_bidi_stream_count, 10},
+        {alpn, ["mqtt"]}
+    ],
+    Server = quic_server:start_link(Port, SslOpts),
+    timer:sleep(500),
+    {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect(
+        "localhost",
+        Port,
+        [
+            {alpn, ["mqtt"]},
+            {active, false},
+            {quic_event_mask, 1}
+        ],
+        3000
+    ),
+    send_and_recv_with(Sock),
+    ok = emqtt_quic:close(Sock),
+    <<_Head:16, Left/binary>> =
+        receive
+            {quic, nst_received, Conn, Ticket} when is_binary(Ticket) ->
+                Ticket
+        end,
+
+    Error = {error, {not_found, invalid_parameter}},
+    Error = emqtt_quic:connect(
+        "localhost",
+        Port,
+        [
+            {alpn, ["mqtt"]},
+            {active, false},
+            {nst, Left}
+        ],
+        3000
+    ),
+    quic_server:stop(Server).
+
+t_multi_streams_sub(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    case emqtt:publish(C, Topic, <<"qos 2 1">>, PubQos) of
+        ok when PubQos == 0 -> ok;
+        {ok, _} -> ok
+    end,
+    receive
+        {publish, #{
+            client_pid := C,
+            payload := <<"qos 2 1">>,
+            qos := RecQos,
+            topic := Topic
+        }} ->
+            ok;
+        Other ->
+            ct:fail("unexpected recv ~p", [Other])
+    after 100 ->
+        ct:fail("not received")
+    end,
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_pub_parallel(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        <<"stream data 1">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data", _/binary>>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<"stream data", _/binary>>,
+                qos := RecQos,
+                topic := Topic
+            }}
+        ],
+        PubRecvs
+    ),
+    Payloads = [P || {publish, #{payload := P}} <- PubRecvs],
+    ?assert(
+        [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse
+            [<<"stream data 2">>, <<"stream data 1">>] == Payloads
+    ),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_packet_boundary(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+    PktId3 = calc_pkt_id(RecQos, 3),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+
+    %% make quicer to batch job
+    quicer:reg_open(quic_execution_profile_type_max_throughput),
+
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+
+    {ok, PubVia} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 1">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    LargePart3 = binary:copy(<<"stream data3">>, 2000),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        LargePart3,
+        [{qos, PubQos}],
+        undefined
+    ),
+    PubRecvs = recv_pub(3),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 1">>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<"stream data 2">>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId3,
+                payload := LargePart3,
+                qos := RecQos,
+                topic := Topic
+            }}
+        ],
+        PubRecvs
+    ),
+    ok = emqtt:disconnect(C).
+
+%% @doc test that one malformed stream will not close the entire connection
+t_multi_streams_packet_malform(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+    PktId3 = calc_pkt_id(RecQos, 3),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+
+    %% make quicer to batch job
+    quicer:reg_open(quic_execution_profile_type_max_throughput),
+
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+
+    {ok, PubVia} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 1">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+
+    {ok, {quic, _Conn, MalformStream}} = emqtt:start_data_stream(C, []),
+    {ok, _} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>),
+
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    LargePart3 = binary:copy(<<"stream data3">>, 2000),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        LargePart3,
+        [{qos, PubQos}],
+        undefined
+    ),
+    PubRecvs = recv_pub(3),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 1">>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<"stream data 2">>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId3,
+                payload := LargePart3,
+                qos := RecQos,
+                topic := Topic
+            }}
+        ],
+        PubRecvs
+    ),
+
+    case quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>) of
+        {ok, 10} -> ok;
+        {error, cancelled} -> ok;
+        {error, stm_send_error, aborted} -> ok
+    end,
+
+    timer:sleep(200),
+    ?assert(is_list(emqtt:info(C))),
+
+    {error, stm_send_error, aborted} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>),
+    timer:sleep(200),
+    ?assert(is_list(emqtt:info(C))),
+
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_sub_pub_async(Config) ->
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    Topic2 = <<Topic/binary, "_two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        <<"stream data 1">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic2,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data", _/binary>>,
+                qos := RecQos
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data", _/binary>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+    Payloads = [P || {publish, #{payload := P}} <- PubRecvs],
+    ?assert(
+        [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse
+            [<<"stream data 2">>, <<"stream data 1">>] == Payloads
+    ),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_sub_pub_sync(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            Via1 = undefined,
+            ok;
+        {ok, #{reason_code := 0, via := Via1}} ->
+            ok
+    end,
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<"stream data 4">>, [
+            {qos, PubQos}
+        ])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := Via2}} ->
+            ?assert(Via1 =/= Via2),
+            ok
+    end,
+    ct:pal("SVia1: ~p, SVia2: ~p", [SVia1, SVia2]),
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 3">>,
+                qos := RecQos,
+                via := SVia1
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 4">>,
+                qos := RecQos,
+                via := SVia2
+            }}
+        ],
+        lists:sort(PubRecvs)
+    ),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_dup_sub(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+
+    #{data_stream_socks := [{quic, _Conn, SubStream} | _]} = proplists:get_value(
+        extra, emqtt:info(C)
+    ),
+    ?assertEqual(2, length(emqx_broker:subscribers(Topic))),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := 0, via := _Via1}} ->
+            ok
+    end,
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 3">>,
+                qos := RecQos
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 3">>,
+                qos := RecQos
+            }}
+        ],
+        lists:sort(PubRecvs)
+    ),
+
+    RecvVias = [Via || {publish, #{via := Via}} <- PubRecvs],
+
+    ct:pal("~p, ~p, ~n recv from: ~p~n", [SVia1, SVia2, PubRecvs]),
+    %% Can recv in any order
+    ?assert([SVia1, SVia2] == RecvVias orelse [SVia2, SVia1] == RecvVias),
+
+    %% Shutdown one stream
+    quicer:async_shutdown_stream(SubStream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 500),
+    timer:sleep(100),
+
+    ?assertEqual(1, length(emqx_broker:subscribers(Topic))),
+
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_corr_topic(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := 0, via := _Via}} ->
+            ok
+    end,
+
+    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
+    ?assert(PubVia =/= SubVia),
+
+    case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := PubVia}} -> ok
+    end,
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<6, 7, 8, 9>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_unsub(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := 0, via := _PVia}} ->
+            ok
+    end,
+
+    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
+    ?assert(PubVia =/= SubVia),
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    emqtt:unsubscribe_via(C, SubVia, Topic),
+
+    case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := 16, via := PubVia, reason_code_name := no_matching_subscribers}} ->
+            ok
+    end,
+
+    timeout = recv_pub(1),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_unsub_via_other(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
+
+    %% Unsub topic1 via stream2 should fail with error code 17: "No subscription existed"
+    {ok, #{via := SVia2}, [17]} = emqtt:unsubscribe_via(C, SVia2, Topic),
+
+    case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia2}} -> ok
+    end,
+
+    PubRecvs2 = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<6, 7, 8, 9>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs2
+    ),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_shutdown_data_stream_abortive(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia =/= SVia2),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
+    {quic, _Conn, DataStream} = PubVia,
+    quicer:shutdown_stream(DataStream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND, 500, 100),
+    timer:sleep(500),
+    %% Still alive
+    ?assert(is_list(emqtt:info(C))).
+
+t_multi_streams_shutdown_ctrl_stream(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    unlink(C),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
+    quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 1000),
+    timer:sleep(500),
+    %% Client should be closed
+    ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)).
+
+t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) ->
+    erlang:process_flag(trap_exit, true),
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {reconnect, true},
+        %% speedup test
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia2 =/= SVia),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
+    quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 100),
+    timer:sleep(200),
+    %% Client should be closed
+    ?assert(is_list(emqtt:info(C))).
+
+t_multi_streams_remote_shutdown(Config) ->
+    erlang:process_flag(trap_exit, true),
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {reconnect, false},
+        %% speedup test
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia2 =/= SVia),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
+
+    ok = stop_emqx(),
+
+    timer:sleep(200),
+    start_emqx_quic(?config(port, Config)),
+
+    %% Client should be closed
+    ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)).
+
+t_multi_streams_remote_shutdown_with_reconnect(Config) ->
+    erlang:process_flag(trap_exit, true),
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {reconnect, true},
+        %% speedup test
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia2 =/= SVia),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
+
+    ok = stop_emqx(),
+
+    timer:sleep(200),
+
+    start_emqx_quic(?config(port, Config)),
+    %% Client should be closed
+    ?assert(is_list(emqtt:info(C))).
+
+t_conn_silent_close(Config) ->
+    erlang:process_flag(trap_exit, true),
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    %% quic idle timeout + 1s
+    timer:sleep(16000),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    ?assertException(
+        exit,
+        noproc,
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, 1}])
+    ).
+
+t_client_conn_bump_streams(Config) ->
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    {quic, Conn, _Stream} = proplists:get_value(socket, emqtt:info(C)),
+    ok = quicer:setopt(Conn, param_conn_settings, #{peer_unidi_stream_count => 20}).
+
+t_olp_true(Config) ->
+    meck:new(emqx_olp, [passthrough, no_history]),
+    ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end),
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    ok = meck:unload(emqx_olp).
+
+t_olp_reject(Config) ->
+    erlang:process_flag(trap_exit, true),
+    emqx_config:put_zone_conf(default, [overload_protection, enable], true),
+    meck:new(emqx_olp, [passthrough, no_history]),
+    ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end),
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+    ?assertEqual(
+        {error,
+            {transport_down, #{
+                error => 346,
+                status =>
+                    user_canceled
+            }}},
+        emqtt:quic_connect(C)
+    ),
+    ok = meck:unload(emqx_olp),
+    emqx_config:put_zone_conf(default, [overload_protection, enable], false).
+
+t_conn_resume(Config) ->
+    erlang:process_flag(trap_exit, true),
+    {ok, C0} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+
+    {ok, _} = emqtt:quic_connect(C0),
+    #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)),
+    emqtt:disconnect(C0),
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5},
+        {nst, NST}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C).
+
+t_conn_without_ctrl_stream(Config) ->
+    erlang:process_flag(trap_exit, true),
+    {ok, Conn} = quicer:connect(
+        {127, 0, 0, 1},
+        ?config(port, Config),
+        [{alpn, ["mqtt"]}, {verify, none}],
+        3000
+    ),
+    receive
+        {quic, transport_shutdown, Conn, _} -> ok
+    end.
+
+%%--------------------------------------------------------------------
+%% Helper functions
+%%--------------------------------------------------------------------
+send_and_recv_with(Sock) ->
+    {ok, {IP, _}} = emqtt_quic:sockname(Sock),
+    ?assert(lists:member(tuple_size(IP), [4, 8])),
+    ok = emqtt_quic:send(Sock, <<"ping">>),
+    emqtt_quic:setopts(Sock, [{active, false}]),
+    {ok, <<"pong">>} = emqtt_quic:recv(Sock, 0),
+    ok = emqtt_quic:setopts(Sock, [{active, 100}]),
+    {ok, Stats} = emqtt_quic:getstat(Sock, [send_cnt, recv_cnt]),
+    %% connection level counters, not stream level
+    [{send_cnt, _}, {recv_cnt, _}] = Stats.
+
+certfile(Config) ->
+    filename:join([test_dir(Config), "certs", "test.crt"]).
+
+keyfile(Config) ->
+    filename:join([test_dir(Config), "certs", "test.key"]).
+
+test_dir(Config) ->
+    filename:dirname(filename:dirname(proplists:get_value(data_dir, Config))).
+
+recv_pub(Count) ->
+    recv_pub(Count, []).
+
+recv_pub(0, Acc) ->
+    lists:reverse(Acc);
+recv_pub(Count, Acc) ->
+    receive
+        {publish, _Prop} = Pub ->
+            recv_pub(Count - 1, [Pub | Acc])
+    after 100 ->
+        timeout
+    end.
+
+all_tc() ->
+    code:add_patha(filename:join(code:lib_dir(emqx), "ebin/")),
+    emqx_common_test_helpers:all(?MODULE).
+
+-spec calc_qos(0 | 1 | 2, 0 | 1 | 2) -> 0 | 1 | 2.
+calc_qos(PubQos, SubQos) ->
+    if
+        PubQos > SubQos ->
+            SubQos;
+        SubQos > PubQos ->
+            PubQos;
+        true ->
+            PubQos
+    end.
+-spec calc_pkt_id(0 | 1 | 2, non_neg_integer()) -> undefined | non_neg_integer().
+calc_pkt_id(0, _Id) ->
+    undefined;
+calc_pkt_id(1, Id) ->
+    Id;
+calc_pkt_id(2, Id) ->
+    Id.
+
+-spec start_emqx_quic(inet:port_number()) -> ok.
+start_emqx_quic(UdpPort) ->
+    emqx_common_test_helpers:start_apps([]),
+    application:ensure_all_started(quicer),
+    emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort).
+
+-spec stop_emqx() -> ok.
+stop_emqx() ->
+    emqx_common_test_helpers:stop_apps([]).

From ceac5a0ec7b054c7db1ff618a57ea8b9bbb7c00f Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 20 Dec 2022 20:35:59 +0100
Subject: [PATCH 064/144] feat(quic): bump to quicer 0.0.101

---
 apps/emqx/rebar.config.script | 2 +-
 mix.exs                       | 2 +-
 rebar.config.erl              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index 3ac2b8758..873b599cd 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.100"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.101"}}}.
 
 ExtraDeps = fun(C) ->
     {deps, Deps0} = lists:keyfind(deps, 1, C),
diff --git a/mix.exs b/mix.exs
index ce798997b..088b06728 100644
--- a/mix.exs
+++ b/mix.exs
@@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
-      do: [{:quicer, github: "emqx/quic", tag: "0.0.100", override: true}],
+      do: [{:quicer, github: "emqx/quic", tag: "0.0.101", override: true}],
       else: []
   end
 
diff --git a/rebar.config.erl b/rebar.config.erl
index 1d342b403..e1b94a954 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,7 +39,7 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.100"}}}.
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.101"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From 0544a3ca0ce95424e70fbd5bc71ec3022c75c736 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 22 Dec 2022 09:49:20 +0100
Subject: [PATCH 065/144] fix(quic): setops on stream and handle peer needs
 stream

- setopts should go for stream
- handle peer_needs_streams for none msquic clients
---
 apps/emqx/src/emqx_quic_connection.erl | 25 ++++++++++++++-----------
 apps/emqx/src/emqx_quic_stream.erl     |  6 +++---
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index e632e6b1a..8cdd9d5e6 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -37,8 +37,7 @@
     local_address_changed/3,
     peer_address_changed/3,
     streams_available/3,
-    % @TODO wait for newer quicer
-    %peer_needs_streams/3,
+    peer_needs_streams/3,
     resumed/3,
     new_stream/3
 ]).
@@ -233,9 +232,12 @@ streams_available(_C, {BidirCnt, UnidirCnt}, S) ->
 %%      should cope with rate limiting
 %% @TODO this is not going to get triggered in current version
 %% for https://github.com/microsoft/msquic/issues/3120
-%% -spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret().
-%% peer_needs_streams(_C, undefined, S) ->
-%%     {ok, S}.
+-spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret().
+peer_needs_streams(_C, undefined, S) ->
+    ?SLOG(info, #{
+        msg => "ignore: peer need more streames", info => maps:with([conn_pid, ctrl_pid], S)
+    }),
+    {ok, S}.
 
 %% @doc handle API calls
 handle_call(
@@ -255,7 +257,7 @@ handle_call(_Req, _From, S) ->
 
 %% @doc handle DOWN messages from streams.
 %% @TODO handle DOWN from supervisor?
-handle_info({'DOWN', _Ref, process, Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) ->
+handle_info({'EXIT', Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) ->
     case Reason of
         normal ->
             quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0);
@@ -264,12 +266,13 @@ handle_info({'DOWN', _Ref, process, Pid, Reason}, #{ctrl_pid := Pid, conn := Con
             quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 1)
     end,
     {ok, S};
-handle_info({'DOWN', _Ref, process, Pid, Reason}, #{streams := Streams} = S) when
-    Reason =:= normal orelse
-        Reason =:= {shutdown, protocol_error}
-->
+handle_info({'EXIT', Pid, Reason}, #{streams := Streams} = S) ->
     case proplists:is_defined(Pid, Streams) of
-        true ->
+        true when
+            Reason =:= normal orelse
+                Reason =:= {shutdown, protocol_error} orelse
+                Reason =:= killed
+        ->
             {ok, S};
         false ->
             {stop, unknown_pid_down, S}
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index 714ef337f..6fb7b0816 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -137,13 +137,13 @@ getstat({quic, Conn, _Stream, _Info}, Stats) ->
         Res -> Res
     end.
 
-setopts(Socket, Opts) ->
+setopts({quic, _Conn, Stream, _Info}, Opts) ->
     lists:foreach(
         fun
             ({Opt, V}) when is_atom(Opt) ->
-                quicer:setopt(Socket, Opt, V);
+                quicer:setopt(Stream, Opt, V);
             (Opt) when is_atom(Opt) ->
-                quicer:setopt(Socket, Opt, true)
+                quicer:setopt(Stream, Opt, true)
         end,
         Opts
     ),

From 0173121a309adea318e06799bd71508d4643aa60 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 22 Dec 2022 22:20:29 +0100
Subject: [PATCH 066/144] feat(quic): improve coverage and remove unused code

---
 apps/emqx/src/emqx_quic_connection.erl  |  19 ++-
 apps/emqx/src/emqx_quic_data_stream.erl |  18 ++-
 apps/emqx/test/emqtt_quic_SUITE.erl     | 151 +++++++++++++++++++++++-
 3 files changed, 173 insertions(+), 15 deletions(-)

diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index 8cdd9d5e6..588648483 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -135,20 +135,17 @@ new_conn(
 %% @doc callback when connection is connected.
 -spec connected(quicer:connection_handler(), quicer:connected_props(), cb_state()) ->
     {ok, cb_state()} | {error, any()}.
-connected(Conn, Props, #{slow_start := false} = S) ->
-    ?SLOG(debug, Props),
-    {ok, Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S),
-    {ok, S#{ctrl_pid => Pid}};
 connected(_Conn, Props, S) ->
     ?SLOG(debug, Props),
     {ok, S}.
 
 %% @doc callback when connection is resumed from 0-RTT
 -spec resumed(quicer:connection_handle(), SessionData :: binary() | false, cb_state()) -> cb_ret().
-resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when
-    is_function(ResumeFun)
-->
-    ResumeFun(Conn, Data, S);
+%% reserve resume conn with callback.
+%% resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when
+%%     is_function(ResumeFun)
+%% ->
+%%     ResumeFun(Conn, Data, S);
 resumed(_Conn, _Data, S) ->
     {ok, S#{is_resumed := true}}.
 
@@ -245,9 +242,11 @@ handle_call(
     _From,
     #{streams := Streams} = S
 ) ->
-    [emqx_quic_data_stream:activate_data(OwnerPid, ActivateData) || {OwnerPid, _Stream} <- Streams],
+    [
+        catch emqx_quic_data_stream:activate_data(OwnerPid, ActivateData)
+     || {OwnerPid, _Stream} <- Streams
+    ],
     {reply, ok, S#{
-        %streams := [], %% @FIXME what ??????
         channel := Channel,
         serialize := Serialize,
         parse_state := PS
diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl
index 094680b19..24dd71c29 100644
--- a/apps/emqx/src/emqx_quic_data_stream.erl
+++ b/apps/emqx/src/emqx_quic_data_stream.erl
@@ -68,11 +68,11 @@ activate_data(StreamPid, {PS, Serialize, Channel}) ->
 %% @TODO -spec
 init_handoff(
     Stream,
-    #{parse_state := PS} = _StreamOpts,
+    _StreamOpts,
     Connection,
     #{is_orphan := true, flags := Flags}
 ) ->
-    {ok, init_state(Stream, Connection, Flags, PS)}.
+    {ok, init_state(Stream, Connection, Flags)}.
 
 %%
 %% @doc Post handoff data stream
@@ -239,6 +239,7 @@ do_handle_appl_msg(
 do_handle_appl_msg({incoming, #mqtt_packet{} = Packet}, #{channel := Channel} = S) when
     Channel =/= undefined
 ->
+    ok = inc_incoming_stats(Packet),
     with_channel(handle_in, [Packet], S);
 do_handle_appl_msg({incoming, {frame_error, _} = FE}, #{channel := Channel} = S) when
     Channel =/= undefined
@@ -422,6 +423,19 @@ do_parse_incoming(Data, Packets, ParseState) ->
     end.
 
 %% followings are copied from emqx_connection
+-compile({inline, [inc_incoming_stats/1]}).
+inc_incoming_stats(Packet = ?PACKET(Type)) ->
+    inc_counter(recv_pkt, 1),
+    case Type =:= ?PUBLISH of
+        true ->
+            inc_counter(recv_msg, 1),
+            inc_qos_stats(recv_msg, Packet),
+            inc_counter(incoming_pubs, 1);
+        false ->
+            ok
+    end,
+    emqx_metrics:inc_recv(Packet).
+
 -compile({inline, [inc_outgoing_stats/1]}).
 inc_outgoing_stats({error, message_too_large}) ->
     inc_counter('send_msg.dropped', 1),
diff --git a/apps/emqx/test/emqtt_quic_SUITE.erl b/apps/emqx/test/emqtt_quic_SUITE.erl
index 28e9bcd7b..6c19ecdad 100644
--- a/apps/emqx/test/emqtt_quic_SUITE.erl
+++ b/apps/emqx/test/emqtt_quic_SUITE.erl
@@ -21,6 +21,7 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("quicer/include/quicer.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
 suite() ->
     [{timetrap, {seconds, 30}}].
@@ -79,7 +80,10 @@ groups() ->
             t_multi_streams_shutdown_data_stream_abortive,
             t_multi_streams_dup_sub,
             t_multi_streams_packet_boundary,
-            t_multi_streams_packet_malform
+            t_multi_streams_packet_malform,
+            t_multi_streams_kill_sub_stream,
+            t_multi_streams_packet_too_large,
+            t_conn_change_client_addr
         ]},
 
         {shutdown, [
@@ -537,12 +541,84 @@ t_multi_streams_packet_malform(Config) ->
     timer:sleep(200),
     ?assert(is_list(emqtt:info(C))),
 
-    {error, stm_send_error, aborted} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>),
+    {error, stm_send_error, aborted} = quicer:send(MalformStream, <<1, 2, 3, 4, 5, 6, 7, 8, 9, 0>>),
+
     timer:sleep(200),
     ?assert(is_list(emqtt:info(C))),
 
     ok = emqtt:disconnect(C).
 
+t_multi_streams_packet_too_large(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    meck:new(emqx_frame, [passthrough, no_history]),
+    ok = meck:expect(
+        emqx_frame,
+        serialize_opts,
+        fun(#mqtt_packet_connect{proto_ver = ProtoVer}) ->
+            #{version => ProtoVer, max_size => 1024}
+        end
+    ),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+
+    {ok, PubVia} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        binary:copy(<<"stream data 1">>, 1024),
+        [{qos, PubQos}],
+        undefined
+    ),
+    timeout = recv_pub(1),
+    ?assert(is_list(emqtt:info(C))),
+    ok = meck:unload(emqx_frame),
+    ok = emqtt:disconnect(C).
+
+t_conn_change_client_addr(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+
+    {ok, {quic, Conn, _} = PubVia} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 1">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := _PktId1,
+                payload := <<"stream data 1">>,
+                qos := RecQos
+            }}
+        ],
+        recv_pub(1)
+    ),
+    NewPort = select_port(),
+    {ok, OldAddr} = quicer:sockname(Conn),
+    ?assertEqual(
+        ok, quicer:setopt(Conn, param_conn_local_address, "127.0.0.1:" ++ integer_to_list(NewPort))
+    ),
+    {ok, NewAddr} = quicer:sockname(Conn),
+    ct:pal("NewAddr: ~p, Old Addr: ~p", [NewAddr, OldAddr]),
+    ?assertNotEqual(OldAddr, NewAddr),
+    ?assert(is_list(emqtt:info(C))),
+    ok = emqtt:disconnect(C).
+
 t_multi_streams_sub_pub_async(Config) ->
     Topic = atom_to_binary(?FUNCTION_NAME),
     PubQos = ?config(pub_qos, Config),
@@ -815,6 +891,57 @@ t_multi_streams_unsub(Config) ->
     timeout = recv_pub(1),
     ok = emqtt:disconnect(C).
 
+t_multi_streams_kill_sub_stream(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+    [TopicStreamOwner] = emqx_broker:subscribers(Topic),
+    exit(TopicStreamOwner, kill),
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := Code, via := _PVia}} when Code == 0 orelse Code == 16 ->
+            ok
+    end,
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := 0, via := _PVia2}} ->
+            ok
+    end,
+
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                topic := Topic2,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        recv_pub(1)
+    ),
+    ?assertEqual(timeout, recv_pub(1)),
+    ok.
+
 t_multi_streams_unsub_via_other(Config) ->
     PubQos = ?config(pub_qos, Config),
     SubQos = ?config(sub_qos, Config),
@@ -1208,7 +1335,9 @@ t_conn_resume(Config) ->
         {nst, NST}
         | Config
     ]),
-    {ok, _} = emqtt:quic_connect(C).
+    {ok, _} = emqtt:quic_connect(C),
+    Cid = proplists:get_value(clientid, emqtt:info(C)),
+    ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]).
 
 t_conn_without_ctrl_stream(Config) ->
     erlang:process_flag(trap_exit, true),
@@ -1289,3 +1418,19 @@ start_emqx_quic(UdpPort) ->
 -spec stop_emqx() -> ok.
 stop_emqx() ->
     emqx_common_test_helpers:stop_apps([]).
+
+%% select a random port picked by OS
+-spec select_port() -> inet:port_number().
+select_port() ->
+    {ok, S} = gen_udp:open(0, [{reuseaddr, true}]),
+    {ok, {_, Port}} = inet:sockname(S),
+    gen_udp:close(S),
+    case os:type() of
+        {unix, darwin} ->
+            %% in MacOS, still get address_in_use after close port
+            timer:sleep(500);
+        _ ->
+            skip
+    end,
+    ct:pal("select port: ~p", [Port]),
+    Port.

From 71d3148544ac98c7d2ea6bd3be3d32b276354607 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 5 Jan 2023 15:15:34 +0100
Subject: [PATCH 067/144] feat(quic): stream use active_n 10

---
 apps/emqx/src/emqx_quic_data_stream.erl | 3 ++-
 apps/emqx/src/emqx_quic_stream.erl      | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl
index 24dd71c29..61c13bdee 100644
--- a/apps/emqx/src/emqx_quic_data_stream.erl
+++ b/apps/emqx/src/emqx_quic_data_stream.erl
@@ -182,7 +182,8 @@ handle_stream_data(
 %%             {ok, State}
 %%     end.
 
-passive(_Stream, undefined, S) ->
+passive(Stream, undefined, S) ->
+    quicer:setopt(Stream, active, 10),
     {ok, S}.
 
 stream_closed(
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index 6fb7b0816..667ddb2b0 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -270,8 +270,9 @@ start_completed(_Stream, #{status := Other} = Prop, S) ->
 %%     {stop, unimpl}.
 
 -spec passive(stream_handle(), undefined, cb_data()) -> cb_ret().
-passive(_Stream, undefined, _S) ->
-    {stop, unimpl}.
+passive(Stream, undefined, S) ->
+    quicer:setopt(Stream, active, 10),
+    {ok, S}.
 
 -spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) -> cb_ret().
 stream_closed(

From 1e8b2e247e9d31c754412622ce8cfcf4eb0f4bea Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 6 Jan 2023 13:09:25 +0100
Subject: [PATCH 068/144] feat(quic): 0-RTT multi-streams data

---
 apps/emqx/src/emqx_quic_connection.erl  |  3 +-
 apps/emqx/src/emqx_quic_data_stream.erl | 37 ++++++-----
 apps/emqx/src/emqx_quic_stream.erl      | 35 +---------
 apps/emqx/test/emqtt_quic_SUITE.erl     | 88 +++++++++++++++++++++++--
 4 files changed, 104 insertions(+), 59 deletions(-)

diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index 588648483..ef0d9b2e3 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -174,7 +174,8 @@ new_stream(
         limiter => Limiter,
         parse_state => PS,
         channel => Channel,
-        serialize => Serialize
+        serialize => Serialize,
+        quic_event_mask => ?QUICER_STREAM_EVENT_MASK_START_COMPLETE
     },
     {ok, NewStreamOwner} = quicer_stream:start_link(
         emqx_quic_data_stream,
diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl
index 61c13bdee..bea0d37e1 100644
--- a/apps/emqx/src/emqx_quic_data_stream.erl
+++ b/apps/emqx/src/emqx_quic_data_stream.erl
@@ -25,14 +25,12 @@
 -include_lib("quicer/include/quicer.hrl").
 -include("emqx_mqtt.hrl").
 -include("logger.hrl").
--behaviour(quicer_stream).
+-behaviour(quicer_remote_stream).
 
 %% Connection Callbacks
 -export([
     init_handoff/4,
     post_handoff/3,
-    new_stream/3,
-    start_completed/3,
     send_complete/3,
     peer_send_shutdown/3,
     peer_send_aborted/3,
@@ -79,17 +77,15 @@ init_handoff(
 %%
 %% @TODO -spec
 %%
+post_handoff(_Stream, {undefined = _PS, undefined = _Serialize, undefined = _Channel}, S) ->
+    %% Channel isn't ready yet.
+    %% Data stream should wait for activate call with ?MODULE:activate_data/2
+    {ok, S};
 post_handoff(Stream, {PS, Serialize, Channel}, S) ->
     ?tp(debug, ?FUNCTION_NAME, #{channel => Channel, serialize => Serialize}),
     quicer:setopt(Stream, active, true),
     {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}.
 
-%%
-%% @doc when this proc is assigned to the owner of new stream
-%%
-new_stream(Stream, #{flags := Flags}, Connection) ->
-    {ok, init_state(Stream, Connection, Flags)}.
-
 %%
 %% @doc for local initiated stream
 %%
@@ -125,12 +121,6 @@ send_complete(_Stream, true = _IsCanceled, S) ->
 send_shutdown_complete(_Stream, _Flags, S) ->
     {ok, S}.
 
-start_completed(_Stream, #{status := success, stream_id := StreamId}, S) ->
-    {ok, S#{stream_id => StreamId}};
-start_completed(_Stream, #{status := Other}, S) ->
-    %% or we could retry
-    {stop, {start_fail, Other}, S}.
-
 handle_stream_data(
     Stream,
     Bin,
@@ -208,7 +198,18 @@ stream_closed(
     {stop, normal, S}.
 
 handle_call(Call, _From, S) ->
-    do_handle_call(Call, S).
+    case do_handle_call(Call, S) of
+        {ok, NewS} ->
+            {reply, ok, NewS};
+        {error, Reason, NewS} ->
+            {reply, {error, Reason}, NewS};
+        {{continue, _} = Cont, NewS} ->
+            {reply, ok, NewS, Cont};
+        {hibernate, NewS} ->
+            {reply, ok, NewS, hibernate};
+        {stop, Reason, NewS} ->
+            {stop, Reason, {stopped, Reason}, NewS}
+    end.
 
 handle_continue(handle_appl_msg, #{task_queue := Q} = S) ->
     case queue:out(Q) of
@@ -390,8 +391,8 @@ do_handle_call(
             ?SLOG(error, #{msg => "set stream active failed", error => E}),
             {stop, E, NewS}
     end;
-do_handle_call(_Call, S) ->
-    {reply, {error, unimpl}, S}.
+do_handle_call(_Call, _S) ->
+    {error, unimpl}.
 
 %% @doc return reserved order of Packets
 parse_incoming(Data, PS) ->
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index 667ddb2b0..ee764cdc5 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -17,7 +17,7 @@
 %% MQTT/QUIC Stream
 -module(emqx_quic_stream).
 
--behaviour(quicer_stream).
+-behaviour(quicer_remote_stream).
 
 %% emqx transport Callbacks
 -export([
@@ -57,18 +57,14 @@
 -type stream_handle() :: quicer:stream_handle().
 
 -export([
-    init_handoff/4,
     new_stream/3,
-    start_completed/3,
     send_complete/3,
     peer_send_shutdown/3,
     peer_send_aborted/3,
     peer_receive_aborted/3,
     send_shutdown_complete/3,
     stream_closed/3,
-    peer_accepted/3,
-    passive/3,
-    handle_call/4
+    passive/3
 ]).
 
 -export_type([socket/0]).
@@ -195,21 +191,10 @@ async_send({quic, _Conn, Stream, _Info}, Data, _Options) ->
 %%%
 %%% quicer stream callbacks
 %%%
-
--spec init_handoff(stream_handle(), #{}, quicer:connection_handle(), #{}) -> cb_ret().
-init_handoff(_Stream, _StreamOpts, _Conn, _Flags) ->
-    %% stream owner already set while starts.
-    {stop, unimpl}.
-
 -spec new_stream(stream_handle(), quicer:new_stream_props(), cb_data()) -> cb_ret().
 new_stream(_Stream, #{flags := _Flags, is_orphan := _IsOrphan}, _Conn) ->
     {stop, unimpl}.
 
--spec peer_accepted(stream_handle(), undefined, cb_data()) -> cb_ret().
-peer_accepted(_Stream, undefined, S) ->
-    %% We just ignore it
-    {ok, S}.
-
 -spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret().
 peer_receive_aborted(Stream, ErrorCode, S) ->
     quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
@@ -237,19 +222,6 @@ send_complete(_Stream, true = _IsCancelled, S) ->
 send_shutdown_complete(_Stream, _IsGraceful, S) ->
     {ok, S}.
 
--spec start_completed(stream_handle(), quicer:stream_start_completed_props(), cb_data()) ->
-    cb_ret().
-start_completed(_Stream, #{status := success, stream_id := StreamId} = Prop, S) ->
-    ?SLOG(debug, Prop),
-    {ok, S#{stream_id => StreamId}};
-start_completed(_Stream, #{status := stream_limit_reached, stream_id := _StreamId} = Prop, _S) ->
-    ?SLOG(error, #{message => start_completed}, Prop),
-    {stop, stream_limit_reached};
-start_completed(_Stream, #{status := Other} = Prop, S) ->
-    ?SLOG(error, Prop),
-    %% or we could retry?
-    {stop, {start_fail, Other}, S}.
-
 %% Local stream, Unidir
 %% -spec handle_stream_data(stream_handle(), binary(), quicer:recv_data_props(), cb_data())
 %%                         -> cb_ret().
@@ -299,9 +271,6 @@ stream_closed(
     %% a msg to be processed
     {ok, {sock_closed, Status}, S}.
 
-handle_call(_Stream, _Request, _Opts, S) ->
-    {error, unimpl, S}.
-
 %%%
 %%%  Internals
 %%%
diff --git a/apps/emqx/test/emqtt_quic_SUITE.erl b/apps/emqx/test/emqtt_quic_SUITE.erl
index 6c19ecdad..cfb9d4ae4 100644
--- a/apps/emqx/test/emqtt_quic_SUITE.erl
+++ b/apps/emqx/test/emqtt_quic_SUITE.erl
@@ -77,12 +77,12 @@ groups() ->
             t_multi_streams_unsub,
             t_multi_streams_corr_topic,
             t_multi_streams_unsub_via_other,
-            t_multi_streams_shutdown_data_stream_abortive,
             t_multi_streams_dup_sub,
             t_multi_streams_packet_boundary,
             t_multi_streams_packet_malform,
             t_multi_streams_kill_sub_stream,
             t_multi_streams_packet_too_large,
+            t_multi_streams_sub_0_rtt,
             t_conn_change_client_addr
         ]},
 
@@ -93,10 +93,22 @@ groups() ->
             {group, abort_send_recv_shutdown}
         ]},
 
-        {graceful_shutdown, [{group, ctrl_stream_shutdown}]},
-        {abort_recv_shutdown, [{group, ctrl_stream_shutdown}]},
-        {abort_send_shutdown, [{group, ctrl_stream_shutdown}]},
-        {abort_send_recv_shutdown, [{group, ctrl_stream_shutdown}]},
+        {graceful_shutdown, [
+            {group, ctrl_stream_shutdown},
+            {group, data_stream_shutdown}
+        ]},
+        {abort_recv_shutdown, [
+            {group, ctrl_stream_shutdown},
+            {group, data_stream_shutdown}
+        ]},
+        {abort_send_shutdown, [
+            {group, ctrl_stream_shutdown},
+            {group, data_stream_shutdown}
+        ]},
+        {abort_send_recv_shutdown, [
+            {group, ctrl_stream_shutdown},
+            {group, data_stream_shutdown}
+        ]},
 
         {ctrl_stream_shutdown, [
             t_multi_streams_shutdown_ctrl_stream,
@@ -104,6 +116,8 @@ groups() ->
             t_multi_streams_remote_shutdown,
             t_multi_streams_remote_shutdown_with_reconnect
         ]},
+
+        {data_stream_shutdown, [t_multi_streams_shutdown_data_stream]},
         {misc, [
             t_conn_silent_close,
             t_client_conn_bump_streams,
@@ -1004,7 +1018,7 @@ t_multi_streams_unsub_via_other(Config) ->
     ),
     ok = emqtt:disconnect(C).
 
-t_multi_streams_shutdown_data_stream_abortive(Config) ->
+t_multi_streams_shutdown_data_stream(Config) ->
     PubQos = ?config(pub_qos, Config),
     SubQos = ?config(sub_qos, Config),
     RecQos = calc_qos(PubQos, SubQos),
@@ -1045,7 +1059,7 @@ t_multi_streams_shutdown_data_stream_abortive(Config) ->
 
     #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
     {quic, _Conn, DataStream} = PubVia,
-    quicer:shutdown_stream(DataStream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND, 500, 100),
+    quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100),
     timer:sleep(500),
     %% Still alive
     ?assert(is_list(emqtt:info(C))).
@@ -1351,6 +1365,66 @@ t_conn_without_ctrl_stream(Config) ->
         {quic, transport_shutdown, Conn, _} -> ok
     end.
 
+t_data_stream_race_ctrl_stream(Config) ->
+    erlang:process_flag(trap_exit, true),
+    {ok, C0} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C0),
+    #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)),
+    emqtt:disconnect(C0),
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5},
+        {nst, NST}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    Cid = proplists:get_value(clientid, emqtt:info(C)),
+    ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]).
+
+t_multi_streams_sub_0_rtt(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C0),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    ok = emqtt:open_quic_connection(C),
+    ok = emqtt:quic_mqtt_connect(C),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        #{},
+        <<"qos 2 1">>,
+        [{qos, PubQos}],
+        infinity,
+        fun(_) -> ok end
+    ),
+    {ok, _} = emqtt:quic_connect(C),
+    receive
+        {publish, #{
+            client_pid := C0,
+            payload := <<"qos 2 1">>,
+            qos := RecQos,
+            topic := Topic
+        }} ->
+            ok;
+        Other ->
+            ct:fail("unexpected recv ~p", [Other])
+    after 100 ->
+        ct:fail("not received")
+    end,
+    ok = emqtt:disconnect(C),
+    ok = emqtt:disconnect(C0).
+
 %%--------------------------------------------------------------------
 %% Helper functions
 %%--------------------------------------------------------------------

From 5764994436956276aafe84359b98b8978457903c Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 6 Jan 2023 13:10:33 +0100
Subject: [PATCH 069/144] feat(quic): bump to quicer 0.0.103

---
 apps/emqx/rebar.config.script | 2 +-
 mix.exs                       | 2 +-
 rebar.config.erl              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index 873b599cd..faf668b26 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.101"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.103"}}}.
 
 ExtraDeps = fun(C) ->
     {deps, Deps0} = lists:keyfind(deps, 1, C),
diff --git a/mix.exs b/mix.exs
index 088b06728..181f02633 100644
--- a/mix.exs
+++ b/mix.exs
@@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
-      do: [{:quicer, github: "emqx/quic", tag: "0.0.101", override: true}],
+      do: [{:quicer, github: "emqx/quic", tag: "0.0.103", override: true}],
       else: []
   end
 
diff --git a/rebar.config.erl b/rebar.config.erl
index e1b94a954..adf930ac5 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,7 +39,7 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.101"}}}.
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.103"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From f65ac5422e3c850be9104805781db27b054e332a Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Sun, 8 Jan 2023 22:24:32 +0100
Subject: [PATCH 070/144] test(quic): improve coverage

---
 apps/emqx/src/emqx_quic_data_stream.erl |   6 +-
 apps/emqx/src/emqx_quic_stream.erl      |   2 +-
 apps/emqx/test/emqtt_quic_SUITE.erl     | 196 ++++++++++++++++++++++++
 3 files changed, 200 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl
index bea0d37e1..2aa3ad4f7 100644
--- a/apps/emqx/src/emqx_quic_data_stream.erl
+++ b/apps/emqx/src/emqx_quic_data_stream.erl
@@ -83,7 +83,7 @@ post_handoff(_Stream, {undefined = _PS, undefined = _Serialize, undefined = _Cha
     {ok, S};
 post_handoff(Stream, {PS, Serialize, Channel}, S) ->
     ?tp(debug, ?FUNCTION_NAME, #{channel => Channel, serialize => Serialize}),
-    quicer:setopt(Stream, active, true),
+    quicer:setopt(Stream, active, 10),
     {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}.
 
 %%
@@ -301,8 +301,8 @@ handle_outgoing(Packets, #{serialize := Serialize, stream := Stream, is_unidir :
     is_list(Packets)
 ->
     OutBin = [serialize_packet(P, Serialize) || P <- filter_disallowed_out(Packets)],
-    %% @TODO in which case shall we use sync send?
-    Res = quicer:async_send(Stream, OutBin),
+    %% Send data async but still want send feedback via {quic, send_complete, ...}
+    Res = quicer:async_send(Stream, OutBin, ?QUICER_SEND_FLAG_SYNC),
     ?TRACE("MQTT", "mqtt_packet_sent", #{packets => Packets}),
     [ok = inc_outgoing_stats(P) || P <- Packets],
     Res.
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index ee764cdc5..88cf4b7c3 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -183,7 +183,7 @@ ensure_ok_or_exit(Fun, Args = [Sock | _]) when is_atom(Fun), is_list(Args) ->
     end.
 
 async_send({quic, _Conn, Stream, _Info}, Data, _Options) ->
-    case quicer:send(Stream, Data) of
+    case quicer:async_send(Stream, Data, ?QUICER_SEND_FLAG_SYNC) of
         {ok, _Len} -> ok;
         Other -> Other
     end.
diff --git a/apps/emqx/test/emqtt_quic_SUITE.erl b/apps/emqx/test/emqtt_quic_SUITE.erl
index cfb9d4ae4..f926c2f3e 100644
--- a/apps/emqx/test/emqtt_quic_SUITE.erl
+++ b/apps/emqx/test/emqtt_quic_SUITE.erl
@@ -71,7 +71,9 @@ groups() ->
         {sub_qos2, [{group, qos}]},
         {qos, [
             t_multi_streams_sub,
+            t_multi_streams_pub_5x100,
             t_multi_streams_pub_parallel,
+            t_multi_streams_pub_parallel_no_blocking,
             t_multi_streams_sub_pub_async,
             t_multi_streams_sub_pub_sync,
             t_multi_streams_unsub,
@@ -83,6 +85,8 @@ groups() ->
             t_multi_streams_kill_sub_stream,
             t_multi_streams_packet_too_large,
             t_multi_streams_sub_0_rtt,
+            t_multi_streams_sub_0_rtt_large_payload,
+            t_multi_streams_sub_0_rtt_stream_data_cont,
             t_conn_change_client_addr
         ]},
 
@@ -347,6 +351,36 @@ t_multi_streams_sub(Config) ->
     end,
     ok = emqtt:disconnect(C).
 
+t_multi_streams_pub_5x100(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+
+    PubVias = lists:map(
+        fun(_N) ->
+            {ok, Via} = emqtt:start_data_stream(C, []),
+            Via
+        end,
+        lists:seq(1, 5)
+    ),
+    [
+        begin
+            case emqtt:publish_via(C, PVia, Topic, #{}, <<"stream data ", N>>, [{qos, PubQos}]) of
+                ok when PubQos == 0 -> ok;
+                {ok, _} -> ok
+            end,
+            0 == (N rem 10) andalso timer:sleep(10)
+        end
+     || N <- lists:seq(1, 100), PVia <- PubVias
+    ],
+    ?assert(timeout =/= recv_pub(500)),
+    ok = emqtt:disconnect(C).
+
 t_multi_streams_pub_parallel(Config) ->
     PubQos = ?config(pub_qos, Config),
     SubQos = ?config(sub_qos, Config),
@@ -400,6 +434,60 @@ t_multi_streams_pub_parallel(Config) ->
     ),
     ok = emqtt:disconnect(C).
 
+%% @doc test two pub streams, one send incomplete MQTT packet() can not block another.
+t_multi_streams_pub_parallel_no_blocking(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId2 = calc_pkt_id(RecQos, 1),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+    Drop = <<"stream data 1">>,
+    meck:new(emqtt_quic, [passthrough, no_history]),
+    meck:expect(emqtt_quic, send, fun(Sock, IoList) ->
+        case lists:last(IoList) == Drop of
+            true ->
+                ct:pal("meck droping ~p", [Drop]),
+                meck:passthrough([Sock, IoList -- [Drop]]);
+            false ->
+                meck:passthrough([Sock, IoList])
+        end
+    end),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        Drop,
+        [{qos, PubQos}],
+        undefined
+    ),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<"stream data 2">>,
+                qos := RecQos,
+                topic := Topic
+            }}
+        ],
+        PubRecvs
+    ),
+    meck:unload(emqtt_quic),
+    ?assertEqual(timeout, recv_pub(1)),
+    ok = emqtt:disconnect(C).
+
 t_multi_streams_packet_boundary(Config) ->
     PubQos = ?config(pub_qos, Config),
     SubQos = ?config(sub_qos, Config),
@@ -1425,6 +1513,114 @@ t_multi_streams_sub_0_rtt(Config) ->
     ok = emqtt:disconnect(C),
     ok = emqtt:disconnect(C0).
 
+t_multi_streams_sub_0_rtt_large_payload(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Payload = binary:copy(<<"qos 2 1">>, 1600),
+    {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C0),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    ok = emqtt:open_quic_connection(C),
+    ok = emqtt:quic_mqtt_connect(C),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        #{},
+        Payload,
+        [{qos, PubQos}],
+        infinity,
+        fun(_) -> ok end
+    ),
+    {ok, _} = emqtt:quic_connect(C),
+    receive
+        {publish, #{
+            client_pid := C0,
+            payload := Payload,
+            qos := RecQos,
+            topic := Topic
+        }} ->
+            ok;
+        Other ->
+            ct:fail("unexpected recv ~p", [Other])
+    after 100 ->
+        ct:fail("not received")
+    end,
+    ok = emqtt:disconnect(C),
+    ok = emqtt:disconnect(C0).
+
+%% @doc verify data stream can continue after 0-RTT handshake
+t_multi_streams_sub_0_rtt_stream_data_cont(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Payload = binary:copy(<<"qos 2 1">>, 1600),
+    {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C0),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    ok = emqtt:open_quic_connection(C),
+    ok = emqtt:quic_mqtt_connect(C),
+    {ok, PubVia} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        #{},
+        Payload,
+        [{qos, PubQos}],
+        infinity,
+        fun(_) -> ok end
+    ),
+    {ok, _} = emqtt:quic_connect(C),
+    receive
+        {publish, #{
+            client_pid := C0,
+            payload := Payload,
+            qos := RecQos,
+            topic := Topic
+        }} ->
+            ok;
+        Other ->
+            ct:fail("unexpected recv ~p", [Other])
+    after 100 ->
+        ct:fail("not received")
+    end,
+    Payload2 = <<"2nd part", Payload/binary>>,
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        #{},
+        Payload2,
+        [{qos, PubQos}],
+        infinity,
+        fun(_) -> ok end
+    ),
+    receive
+        {publish, #{
+            client_pid := C0,
+            payload := Payload2,
+            qos := RecQos,
+            topic := Topic
+        }} ->
+            ok;
+        Other2 ->
+            ct:fail("unexpected recv ~p", [Other2])
+    after 100 ->
+        ct:fail("not received")
+    end,
+    ok = emqtt:disconnect(C),
+    ok = emqtt:disconnect(C0).
+
 %%--------------------------------------------------------------------
 %% Helper functions
 %%--------------------------------------------------------------------

From 22dcf5907e6f278bd665640aeb2987b56e68928f Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Sun, 8 Jan 2023 22:25:29 +0100
Subject: [PATCH 071/144] feat(quic): bump to quicer 0.0.104

---
 apps/emqx/rebar.config.script | 2 +-
 mix.exs                       | 2 +-
 rebar.config.erl              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index faf668b26..e942e1a5c 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.103"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.104"}}}.
 
 ExtraDeps = fun(C) ->
     {deps, Deps0} = lists:keyfind(deps, 1, C),
diff --git a/mix.exs b/mix.exs
index 181f02633..bf300761f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
-      do: [{:quicer, github: "emqx/quic", tag: "0.0.103", override: true}],
+      do: [{:quicer, github: "emqx/quic", tag: "0.0.104", override: true}],
       else: []
   end
 
diff --git a/rebar.config.erl b/rebar.config.erl
index adf930ac5..e99f83683 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,7 +39,7 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.103"}}}.
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.104"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From 00f615a1e33289d870d7ffada7d17952b05bcf95 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 9 Jan 2023 09:17:03 +0100
Subject: [PATCH 072/144] chore(quic): clean code

---
 apps/emqx/include/emqx_quic.hrl               |    2 +-
 apps/emqx/src/emqx_connection.erl             |   14 +-
 apps/emqx/src/emqx_quic_connection.erl        |   43 +-
 apps/emqx/src/emqx_quic_data_stream.erl       |  119 +-
 apps/emqx/src/emqx_quic_stream.erl            |   77 +-
 apps/emqx/test/emqtt_quic_SUITE.erl           | 1706 ---------------
 .../emqx/test/emqx_mqtt_protocol_v5_SUITE.erl |   13 -
 .../test/emqx_quic_multistreams_SUITE.erl     | 1880 +++++++++++++++--
 8 files changed, 1817 insertions(+), 2037 deletions(-)
 delete mode 100644 apps/emqx/test/emqtt_quic_SUITE.erl

diff --git a/apps/emqx/include/emqx_quic.hrl b/apps/emqx/include/emqx_quic.hrl
index 302f2704d..3366b8938 100644
--- a/apps/emqx/include/emqx_quic.hrl
+++ b/apps/emqx/include/emqx_quic.hrl
@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 980c41010..be420d65e 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -119,10 +119,7 @@
     limiter_timer :: undefined | reference(),
 
     %% QUIC conn pid if is a pid
-    quic_conn_pid :: maybe(pid()),
-
-    %% QUIC control stream callback state
-    quic_ctrl_state :: map()
+    quic_conn_pid :: maybe(pid())
 }).
 
 -record(retry, {
@@ -378,8 +375,7 @@ init_state(
         limiter_buffer = queue:new(),
         limiter_timer = undefined,
         %% for quic streams to inherit
-        quic_conn_pid = maps:get(conn_pid, Opts, undefined),
-        quic_ctrl_state = #{}
+        quic_conn_pid = maps:get(conn_pid, Opts, undefined)
     }.
 
 run_loop(
@@ -928,12 +924,6 @@ handle_info({sock_error, Reason}, State) ->
     handle_info({sock_closed, Reason}, close_socket(State));
 handle_info({quic, Event, Handle, Prop}, State) ->
     emqx_quic_stream:Event(Handle, Prop, State);
-%% handle_info({quic, peer_send_shutdown, _Stream}, State) ->
-%%     handle_info({sock_closed, force}, close_socket(State));
-%% handle_info({quic, closed, _Channel, ReasonFlag}, State) ->
-%%     handle_info({sock_closed, ReasonFlag}, State);
-%% handle_info({quic, closed, _Stream}, State) ->
-%%     handle_info({sock_closed, force}, State);
 handle_info(Info, State) ->
     with_channel(handle_info, [Info], State).
 
diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index ef0d9b2e3..69d16cbc3 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -17,13 +17,11 @@
 %% @doc impl. the quic connection owner process.
 -module(emqx_quic_connection).
 
--include("logger.hrl").
 -ifndef(BUILD_WITHOUT_QUIC).
+
+-include("logger.hrl").
 -include_lib("quicer/include/quicer.hrl").
 -include_lib("emqx/include/emqx_quic.hrl").
--else.
--define(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0).
--endif.
 
 -behavior(quicer_connection).
 
@@ -55,10 +53,9 @@
     %% Pid of ctrl stream
     ctrl_pid := undefined | pid(),
     %% quic connecion handle
-    conn := undefined | quicer:conneciton_hanlder(),
-    %% streams that handoff from this process, excluding control stream
-    %% these streams could die/closed without effecting the connecion/session.
-
+    conn := undefined | quicer:conneciton_handle(),
+    %% Data streams that handoff from this process
+    %% these streams could die/close without effecting the connecion/session.
     %@TODO type?
     streams := [{pid(), quicer:stream_handle()}],
     %% New stream opts
@@ -82,22 +79,20 @@ activate_data_streams(ConnOwner, {PS, Serialize, Channel}) ->
     gen_server:call(ConnOwner, {activate_data_streams, {PS, Serialize, Channel}}, infinity).
 
 %% @doc conneciton owner init callback
--spec init(map() | list()) -> {ok, cb_state()}.
-init(ConnOpts) when is_list(ConnOpts) ->
-    init(maps:from_list(ConnOpts));
+-spec init(map()) -> {ok, cb_state()}.
 init(#{stream_opts := SOpts} = S) when is_list(SOpts) ->
     init(S#{stream_opts := maps:from_list(SOpts)});
 init(ConnOpts) when is_map(ConnOpts) ->
     {ok, init_cb_state(ConnOpts)}.
 
--spec closed(quicer:conneciton_hanlder(), quicer:conn_closed_props(), cb_state()) ->
+-spec closed(quicer:conneciton_handle(), quicer:conn_closed_props(), cb_state()) ->
     {stop, normal, cb_state()}.
 closed(_Conn, #{is_peer_acked := _} = Prop, S) ->
     ?SLOG(debug, Prop),
     {stop, normal, S}.
 
 %% @doc handle the new incoming connecion as the connecion acceptor.
--spec new_conn(quicer:connection_handler(), quicer:new_conn_props(), cb_state()) ->
+-spec new_conn(quicer:connection_handle(), quicer:new_conn_props(), cb_state()) ->
     {ok, cb_state()} | {error, any()}.
 new_conn(
     Conn,
@@ -133,7 +128,7 @@ new_conn(
     end.
 
 %% @doc callback when connection is connected.
--spec connected(quicer:connection_handler(), quicer:connected_props(), cb_state()) ->
+-spec connected(quicer:connection_handle(), quicer:connected_props(), cb_state()) ->
     {ok, cb_state()} | {error, any()}.
 connected(_Conn, Props, S) ->
     ?SLOG(debug, Props),
@@ -185,21 +180,21 @@ new_stream(
         Props
     ),
     quicer:handoff_stream(Stream, NewStreamOwner, {PS, Serialize, Channel}),
-    %% @TODO keep them in ``inactive_streams'
+    %% @TODO maybe keep them in `inactive_streams'
     {ok, S#{streams := [{NewStreamOwner, Stream} | Streams]}}.
 
-%% @doc callback for handling for remote connecion shutdown.
+%% @doc callback for handling remote connecion shutdown.
 -spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret().
-shutdown(Conn, _ErrorCode, S) ->
-    %% @TODO check spec what to set for the ErrorCode?
+shutdown(Conn, ErrorCode, S) ->
+    ErrorCode =/= 0 andalso ?SLOG(debug, #{error_code => ErrorCode, state => S}),
     quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
     {ok, S}.
 
-%% @doc callback for handling for transport error, such as idle timeout
+%% @doc callback for handling transport error, such as idle timeout
 -spec transport_shutdown(quicer:connection_handle(), quicer:transport_shutdown_props(), cb_state()) ->
     cb_ret().
-transport_shutdown(_C, _DownInfo, S) ->
-    %% @TODO some counter
+transport_shutdown(_C, DownInfo, S) when is_map(DownInfo) ->
+    ?SLOG(debug, DownInfo),
     {ok, S}.
 
 %% @doc callback for handling for peer addr changed.
@@ -238,6 +233,7 @@ peer_needs_streams(_C, undefined, S) ->
     {ok, S}.
 
 %% @doc handle API calls
+-spec handle_call(Req :: term(), gen_server:from(), cb_state()) -> cb_ret().
 handle_call(
     {activate_data_streams, {PS, Serialize, Channel} = ActivateData},
     _From,
@@ -256,7 +252,6 @@ handle_call(_Req, _From, S) ->
     {reply, {error, unimpl}, S}.
 
 %% @doc handle DOWN messages from streams.
-%% @TODO handle DOWN from supervisor?
 handle_info({'EXIT', Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) ->
     case Reason of
         normal ->
@@ -302,3 +297,7 @@ init_cb_state(#{zone := _Zone} = Map) ->
         serialize => undefined,
         is_resumed => false
     }.
+
+%% BUILD_WITHOUT_QUIC
+-else.
+-endif.
diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl
index 2aa3ad4f7..e3f6b7adc 100644
--- a/apps/emqx/src/emqx_quic_data_stream.erl
+++ b/apps/emqx/src/emqx_quic_data_stream.erl
@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -21,11 +21,14 @@
 %%
 
 -module(emqx_quic_data_stream).
+
+-ifndef(BUILD_WITHOUT_QUIC).
+-behaviour(quicer_remote_stream).
+
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 -include_lib("quicer/include/quicer.hrl").
 -include("emqx_mqtt.hrl").
 -include("logger.hrl").
--behaviour(quicer_remote_stream).
 
 %% Connection Callbacks
 -export([
@@ -37,12 +40,12 @@
     peer_receive_aborted/3,
     send_shutdown_complete/3,
     stream_closed/3,
-    peer_accepted/3,
     passive/3
 ]).
 
 -export([handle_stream_data/4]).
 
+%% gen_server API
 -export([activate_data/2]).
 
 -export([
@@ -51,9 +54,19 @@
     handle_continue/2
 ]).
 
+-type cb_ret() :: quicer_stream:cb_ret().
+-type cb_state() :: quicer_stream:cb_state().
+-type error_code() :: quicer:error_code().
+-type connection_handle() :: quicer:connection_handle().
+-type stream_handle() :: quicer:stream_handle().
+-type handoff_data() :: {
+    emqx_frame:parse_state() | undefined,
+    emqx_frame:serialize_opts() | undefined,
+    emqx_channel:channel() | undefined
+}.
 %%
 %% @doc Activate the data handling.
-%%      Data handling is disabled before control stream allows the data processing.
+%%      Note, data handling is disabled before finishing the validation over control stream.
 -spec activate_data(pid(), {
     emqx_frame:parse_state(), emqx_frame:serialize_opts(), emqx_channel:channel()
 }) -> ok.
@@ -61,9 +74,12 @@ activate_data(StreamPid, {PS, Serialize, Channel}) ->
     gen_server:call(StreamPid, {activate, {PS, Serialize, Channel}}, infinity).
 
 %%
-%% @doc Handoff from previous owner, mostly from the connection owner.
-%% @TODO parse_state doesn't look necessary since we have it in post_handoff
-%% @TODO -spec
+%% @doc Handoff from previous owner, from the connection owner.
+%%      Note, unlike control stream, there is no acceptor for data streams.
+%%            The connection owner get new stream, spawn new proc and then handover to it.
+%%
+-spec init_handoff(stream_handle(), map(), connection_handle(), quicer:new_stream_props()) ->
+    {ok, cb_state()}.
 init_handoff(
     Stream,
     _StreamOpts,
@@ -75,10 +91,9 @@ init_handoff(
 %%
 %% @doc Post handoff data stream
 %%
-%% @TODO -spec
-%%
+-spec post_handoff(stream_handle(), handoff_data(), cb_state()) -> cb_ret().
 post_handoff(_Stream, {undefined = _PS, undefined = _Serialize, undefined = _Channel}, S) ->
-    %% Channel isn't ready yet.
+    %% When the channel isn't ready yet.
     %% Data stream should wait for activate call with ?MODULE:activate_data/2
     {ok, S};
 post_handoff(Stream, {PS, Serialize, Channel}, S) ->
@@ -86,53 +101,35 @@ post_handoff(Stream, {PS, Serialize, Channel}, S) ->
     quicer:setopt(Stream, active, 10),
     {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}.
 
-%%
-%% @doc for local initiated stream
-%%
-peer_accepted(_Stream, _Flags, S) ->
-    %% we just ignore it
-    {ok, S}.
-
-peer_receive_aborted(Stream, ErrorCode, #{is_unidir := false} = S) ->
+-spec peer_receive_aborted(stream_handle(), error_code(), cb_state()) -> cb_ret().
+peer_receive_aborted(Stream, ErrorCode, #{is_unidir := _} = S) ->
     %% we abort send with same reason
-    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
-    {ok, S};
-peer_receive_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := true} = S) ->
     quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
     {ok, S}.
 
-peer_send_aborted(Stream, ErrorCode, #{is_unidir := false} = S) ->
+-spec peer_send_aborted(stream_handle(), error_code(), cb_state()) -> cb_ret().
+peer_send_aborted(Stream, ErrorCode, #{is_unidir := _} = S) ->
     %% we abort receive with same reason
-    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode),
-    {ok, S};
-peer_send_aborted(Stream, ErrorCode, #{is_unidir := true, is_local := false} = S) ->
     quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode),
     {ok, S}.
 
-peer_send_shutdown(Stream, _Flags, S) ->
+-spec peer_send_shutdown(stream_handle(), undefined, cb_state()) -> cb_ret().
+peer_send_shutdown(Stream, undefined, S) ->
     ok = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0),
     {ok, S}.
 
+-spec send_complete(stream_handle(), IsCanceled :: boolean(), cb_state()) -> cb_ret().
 send_complete(_Stream, false, S) ->
     {ok, S};
 send_complete(_Stream, true = _IsCanceled, S) ->
     {ok, S}.
 
+-spec send_shutdown_complete(stream_handle(), error_code(), cb_state()) -> cb_ret().
 send_shutdown_complete(_Stream, _Flags, S) ->
     {ok, S}.
 
-handle_stream_data(
-    Stream,
-    Bin,
-    _Flags,
-    #{
-        is_unidir := false,
-        channel := undefined,
-        data_queue := Queue,
-        stream := Stream
-    } = State
-) when is_binary(Bin) ->
-    {ok, State#{data_queue := [Bin | Queue]}};
+-spec handle_stream_data(stream_handle(), binary(), quicer:recv_data_props(), cb_state()) ->
+    cb_ret().
 handle_stream_data(
     _Stream,
     Bin,
@@ -145,6 +142,7 @@ handle_stream_data(
         task_queue := TQ
     } = State
 ) when
+    %% assert get stream data only after channel is created
     Channel =/= undefined
 ->
     {MQTTPackets, NewPS} = parse_incoming(list_to_binary(lists:reverse([Bin | QueuedData])), PS),
@@ -157,25 +155,12 @@ handle_stream_data(
     ),
     {{continue, handle_appl_msg}, State#{parse_state := NewPS, task_queue := NewTQ}}.
 
-%% Reserved for unidi streams
-%% handle_stream_data(Stream, Bin, _Flags, #{is_unidir := true, peer_stream := PeerStream, conn := Conn} = State) ->
-%%     case PeerStream of
-%%         undefined ->
-%%             {ok, StreamProc} = quicer_stream:start_link(?MODULE, Conn,
-%%                                                         [ {open_flag, ?QUIC_STREAM_OPEN_FLAG_UNIDIRECTIONAL}
-%%                                                         , {is_local, true}
-%%                                                         ]),
-%%             {ok, _} = quicer_stream:send(StreamProc, Bin),
-%%             {ok, State#{peer_stream := StreamProc}};
-%%         StreamProc when is_pid(StreamProc) ->
-%%             {ok, _} = quicer_stream:send(StreamProc, Bin),
-%%             {ok, State}
-%%     end.
-
+-spec passive(stream_handle(), undefined, cb_state()) -> cb_ret().
 passive(Stream, undefined, S) ->
     quicer:setopt(Stream, active, 10),
     {ok, S}.
 
+-spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_state()) -> cb_ret().
 stream_closed(
     _Stream,
     #{
@@ -197,28 +182,20 @@ stream_closed(
 ->
     {stop, normal, S}.
 
+-spec handle_call(Request :: term(), From :: {pid(), term()}, cb_state()) -> cb_ret().
 handle_call(Call, _From, S) ->
-    case do_handle_call(Call, S) of
-        {ok, NewS} ->
-            {reply, ok, NewS};
-        {error, Reason, NewS} ->
-            {reply, {error, Reason}, NewS};
-        {{continue, _} = Cont, NewS} ->
-            {reply, ok, NewS, Cont};
-        {hibernate, NewS} ->
-            {reply, ok, NewS, hibernate};
-        {stop, Reason, NewS} ->
-            {stop, Reason, {stopped, Reason}, NewS}
-    end.
+    do_handle_call(Call, S).
 
+-spec handle_continue(Continue :: term(), cb_state()) -> cb_ret().
 handle_continue(handle_appl_msg, #{task_queue := Q} = S) ->
     case queue:out(Q) of
         {{value, Item}, Q2} ->
             do_handle_appl_msg(Item, S#{task_queue := Q2});
-        {empty, Q} ->
+        {empty, _Q} ->
             {ok, S}
     end.
 
+%%% Internals
 do_handle_appl_msg(
     {outgoing, Packets},
     #{
@@ -248,7 +225,7 @@ do_handle_appl_msg({incoming, {frame_error, _} = FE}, #{channel := Channel} = S)
 ->
     with_channel(handle_in, [FE], S);
 do_handle_appl_msg({close, Reason}, S) ->
-    %% @TODO shall we abort shutdown or graceful shutdown?
+    %% @TODO shall we abort shutdown or graceful shutdown here?
     with_channel(handle_info, [{sock_closed, Reason}], S);
 do_handle_appl_msg({event, updated}, S) ->
     %% Data stream don't care about connection state changes.
@@ -294,7 +271,6 @@ with_channel(Fun, Args, #{channel := Channel, task_queue := Q} = S) when
             }}
     end.
 
-%%% Internals
 handle_outgoing(#mqtt_packet{} = P, S) ->
     handle_outgoing([P], S);
 handle_outgoing(Packets, #{serialize := Serialize, stream := Stream, is_unidir := false}) when
@@ -373,7 +349,7 @@ init_state(Stream, Connection, OpenFlags, PS) ->
         task_queue => queue:new()
     }.
 
--spec do_handle_call(term(), quicer_stream:cb_state()) -> quicer_stream:cb_ret().
+-spec do_handle_call(term(), cb_state()) -> cb_ret().
 do_handle_call(
     {activate, {PS, Serialize, Channel}},
     #{
@@ -386,7 +362,7 @@ do_handle_call(
     %% We use quic protocol for flow control, and we don't check return val
     case quicer:setopt(Stream, active, true) of
         ok ->
-            {ok, NewS};
+            {reply, ok, NewS};
         {error, E} ->
             ?SLOG(error, #{msg => "set stream active failed", error => E}),
             {stop, E, NewS}
@@ -484,3 +460,6 @@ is_datastream_out_pkt(#mqtt_packet{header = #mqtt_packet_header{type = Type}}) w
     true;
 is_datastream_out_pkt(_) ->
     false.
+%% BUILD_WITHOUT_QUIC
+-else.
+-endif.
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index 88cf4b7c3..a8ef7d41d 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -17,8 +17,12 @@
 %% MQTT/QUIC Stream
 -module(emqx_quic_stream).
 
+-ifndef(BUILD_WITHOUT_QUIC).
+
 -behaviour(quicer_remote_stream).
 
+-include("logger.hrl").
+
 %% emqx transport Callbacks
 -export([
     type/1,
@@ -33,31 +37,14 @@
     sockname/1,
     peercert/1
 ]).
-
--include("logger.hrl").
--ifndef(BUILD_WITHOUT_QUIC).
 -include_lib("quicer/include/quicer.hrl").
--else.
-%% STREAM SHUTDOWN FLAGS
--define(QUIC_STREAM_SHUTDOWN_FLAG_NONE, 0).
-% Cleanly closes the send path.
--define(QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 1).
-% Abruptly closes the send path.
--define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND, 2).
-% Abruptly closes the receive path.
--define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, 4).
-% Abruptly closes both send and receive paths.
--define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT, 6).
--define(QUIC_STREAM_SHUTDOWN_FLAG_IMMEDIATE, 8).
--endif.
 
--type cb_ret() :: gen_statem:event_handler_result().
--type cb_data() :: emqtt_quic:cb_data().
+-type cb_ret() :: quicer_stream:cb_ret().
+-type cb_data() :: quicer_stream:cb_state().
 -type connection_handle() :: quicer:connection_handle().
 -type stream_handle() :: quicer:stream_handle().
 
 -export([
-    new_stream/3,
     send_complete/3,
     peer_send_shutdown/3,
     peer_send_aborted/3,
@@ -79,13 +66,8 @@
 }.
 
 %% for accepting
--spec wait
-    ({pid(), connection_handle(), socket_info()}) ->
-        {ok, socket()} | {error, enotconn};
-    %% For handover
-    ({pid(), connection_handle(), stream_handle(), socket_info()}) ->
-        {ok, socket()} | {error, any()}.
-
+-spec wait({pid(), connection_handle(), socket_info()}) ->
+    {ok, socket()} | {error, enotconn}.
 %%% For Accepting New Remote Stream
 wait({ConnOwner, Conn, ConnInfo}) ->
     {ok, Conn} = quicer:async_accept_stream(Conn, []),
@@ -105,15 +87,8 @@ wait({ConnOwner, Conn, ConnInfo}) ->
         {'EXIT', ConnOwner, _Reason} ->
             {error, enotconn}
     end.
-%% UNUSED, for ownership handover,
-%% wait({PrevOwner, Conn, Stream, SocketInfo}) ->
-%%     case quicer:wait_for_handoff(PrevOwner, Stream) of
-%%         ok ->
-%%             {ok, socket(Conn, Stream, SocketInfo)};
-%%         owner_down ->
-%%             {error, owner_down}
-%%     end.
 
+-spec type(_) -> quic.
 type(_) ->
     quic.
 
@@ -155,7 +130,7 @@ getopts(_Socket, _Opts) ->
         {buffer, 80000}
     ]}.
 
-%% @TODO supply some App Error Code
+%% @TODO supply some App Error Code from caller
 fast_close({ConnOwner, Conn, _ConnInfo}) when is_pid(ConnOwner) ->
     %% handshake aborted.
     quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
@@ -185,15 +160,13 @@ ensure_ok_or_exit(Fun, Args = [Sock | _]) when is_atom(Fun), is_list(Args) ->
 async_send({quic, _Conn, Stream, _Info}, Data, _Options) ->
     case quicer:async_send(Stream, Data, ?QUICER_SEND_FLAG_SYNC) of
         {ok, _Len} -> ok;
+        {error, X, Y} -> {error, {X, Y}};
         Other -> Other
     end.
 
 %%%
 %%% quicer stream callbacks
 %%%
--spec new_stream(stream_handle(), quicer:new_stream_props(), cb_data()) -> cb_ret().
-new_stream(_Stream, #{flags := _Flags, is_orphan := _IsOrphan}, _Conn) ->
-    {stop, unimpl}.
 
 -spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret().
 peer_receive_aborted(Stream, ErrorCode, S) ->
@@ -222,28 +195,12 @@ send_complete(_Stream, true = _IsCancelled, S) ->
 send_shutdown_complete(_Stream, _IsGraceful, S) ->
     {ok, S}.
 
-%% Local stream, Unidir
-%% -spec handle_stream_data(stream_handle(), binary(), quicer:recv_data_props(), cb_data())
-%%                         -> cb_ret().
-%% handle_stream_data(Stream, Bin, Flags, #{ is_local := true
-%%                                         , parse_state := PS} = S) ->
-%%     ?SLOG(debug, #{data => Bin}, Flags),
-%%     case parse(Bin, PS, []) of
-%%         {keep_state, NewPS, Packets} ->
-%%             quicer:setopt(Stream, active, once),
-%%             {keep_state, S#{parse_state := NewPS},
-%%              [{next_event, cast, P } || P <- lists:reverse(Packets)]};
-%%         {stop, _} = Stop ->
-%%             Stop
-%%     end;
-%% %% Remote stream
-%% handle_stream_data(_Stream, _Bin, _Flags,
-%%                    #{is_local := false, is_unidir := true, conn := _Conn} = _S) ->
-%%     {stop, unimpl}.
-
 -spec passive(stream_handle(), undefined, cb_data()) -> cb_ret().
 passive(Stream, undefined, S) ->
-    quicer:setopt(Stream, active, 10),
+    case quicer:setopt(Stream, active, 10) of
+        ok -> ok;
+        Error -> ?SLOG(error, #{message => "set active error", error => Error})
+    end,
     {ok, S}.
 
 -spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) -> cb_ret().
@@ -277,3 +234,7 @@ stream_closed(
 -spec socket(connection_handle(), stream_handle(), socket_info()) -> socket().
 socket(Conn, CtrlStream, Info) when is_map(Info) ->
     {quic, Conn, CtrlStream, Info}.
+
+%% BUILD_WITHOUT_QUIC
+-else.
+-endif.
diff --git a/apps/emqx/test/emqtt_quic_SUITE.erl b/apps/emqx/test/emqtt_quic_SUITE.erl
deleted file mode 100644
index f926c2f3e..000000000
--- a/apps/emqx/test/emqtt_quic_SUITE.erl
+++ /dev/null
@@ -1,1706 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
--module(emqtt_quic_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
--include_lib("quicer/include/quicer.hrl").
--include_lib("emqx/include/emqx_mqtt.hrl").
-
-suite() ->
-    [{timetrap, {seconds, 30}}].
-
-all() ->
-    [
-        {group, mstream},
-        {group, shutdown},
-        {group, misc}
-    ].
-
-groups() ->
-    [
-        {mstream, [], [{group, profiles}]},
-
-        {profiles, [], [
-            {group, profile_low_latency},
-            {group, profile_max_throughput}
-        ]},
-        {profile_low_latency, [], [
-            {group, pub_qos0},
-            {group, pub_qos1},
-            {group, pub_qos2}
-        ]},
-        {profile_max_throughput, [], [
-            {group, pub_qos0},
-            {group, pub_qos1},
-            {group, pub_qos2}
-        ]},
-        {pub_qos0, [], [
-            {group, sub_qos0},
-            {group, sub_qos1},
-            {group, sub_qos2}
-        ]},
-        {pub_qos1, [], [
-            {group, sub_qos0},
-            {group, sub_qos1},
-            {group, sub_qos2}
-        ]},
-        {pub_qos2, [], [
-            {group, sub_qos0},
-            {group, sub_qos1},
-            {group, sub_qos2}
-        ]},
-        {sub_qos0, [{group, qos}]},
-        {sub_qos1, [{group, qos}]},
-        {sub_qos2, [{group, qos}]},
-        {qos, [
-            t_multi_streams_sub,
-            t_multi_streams_pub_5x100,
-            t_multi_streams_pub_parallel,
-            t_multi_streams_pub_parallel_no_blocking,
-            t_multi_streams_sub_pub_async,
-            t_multi_streams_sub_pub_sync,
-            t_multi_streams_unsub,
-            t_multi_streams_corr_topic,
-            t_multi_streams_unsub_via_other,
-            t_multi_streams_dup_sub,
-            t_multi_streams_packet_boundary,
-            t_multi_streams_packet_malform,
-            t_multi_streams_kill_sub_stream,
-            t_multi_streams_packet_too_large,
-            t_multi_streams_sub_0_rtt,
-            t_multi_streams_sub_0_rtt_large_payload,
-            t_multi_streams_sub_0_rtt_stream_data_cont,
-            t_conn_change_client_addr
-        ]},
-
-        {shutdown, [
-            {group, graceful_shutdown},
-            {group, abort_recv_shutdown},
-            {group, abort_send_shutdown},
-            {group, abort_send_recv_shutdown}
-        ]},
-
-        {graceful_shutdown, [
-            {group, ctrl_stream_shutdown},
-            {group, data_stream_shutdown}
-        ]},
-        {abort_recv_shutdown, [
-            {group, ctrl_stream_shutdown},
-            {group, data_stream_shutdown}
-        ]},
-        {abort_send_shutdown, [
-            {group, ctrl_stream_shutdown},
-            {group, data_stream_shutdown}
-        ]},
-        {abort_send_recv_shutdown, [
-            {group, ctrl_stream_shutdown},
-            {group, data_stream_shutdown}
-        ]},
-
-        {ctrl_stream_shutdown, [
-            t_multi_streams_shutdown_ctrl_stream,
-            t_multi_streams_shutdown_ctrl_stream_then_reconnect,
-            t_multi_streams_remote_shutdown,
-            t_multi_streams_remote_shutdown_with_reconnect
-        ]},
-
-        {data_stream_shutdown, [t_multi_streams_shutdown_data_stream]},
-        {misc, [
-            t_conn_silent_close,
-            t_client_conn_bump_streams,
-            t_olp_true,
-            t_olp_reject,
-            t_conn_resume,
-            t_conn_without_ctrl_stream
-        ]}
-    ].
-
-init_per_suite(Config) ->
-    emqx_common_test_helpers:start_apps([]),
-    UdpPort = 14567,
-    start_emqx_quic(UdpPort),
-    %% dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}),
-    %% dbg:p(all, c),
-    %% dbg:tp(emqx_quic_connection, cx),
-    %% dbg:tp(emqx_quic_stream, cx),
-    %% dbg:tp(emqtt, cx),
-    %% dbg:tpl(emqtt_quic_stream, cx),
-    %% dbg:tpl(emqx_quic_stream, cx),
-    %% dbg:tpl(emqx_quic_data_stream, cx),
-    %% dbg:tpl(emqtt, cx),
-    [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config].
-
-end_per_suite(_) ->
-    ok.
-
-init_per_group(pub_qos0, Config) ->
-    [{pub_qos, 0} | Config];
-init_per_group(sub_qos0, Config) ->
-    [{sub_qos, 0} | Config];
-init_per_group(pub_qos1, Config) ->
-    [{pub_qos, 1} | Config];
-init_per_group(sub_qos1, Config) ->
-    [{sub_qos, 1} | Config];
-init_per_group(pub_qos2, Config) ->
-    [{pub_qos, 2} | Config];
-init_per_group(sub_qos2, Config) ->
-    [{sub_qos, 2} | Config];
-init_per_group(abort_send_shutdown, Config) ->
-    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND} | Config];
-init_per_group(abort_recv_shutdown, Config) ->
-    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE} | Config];
-init_per_group(abort_send_recv_shutdown, Config) ->
-    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT} | Config];
-init_per_group(graceful_shutdown, Config) ->
-    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL} | Config];
-init_per_group(profile_max_throughput, Config) ->
-    quicer:reg_open(quic_execution_profile_type_max_throughput),
-    Config;
-init_per_group(profile_low_latency, Config) ->
-    quicer:reg_open(quic_execution_profile_low_latency),
-    Config;
-init_per_group(_, Config) ->
-    Config.
-
-end_per_group(_, Config) ->
-    Config.
-
-init_per_testcase(_, Config) ->
-    emqx_common_test_helpers:start_apps([]),
-    Config.
-
-t_quic_sock(Config) ->
-    Port = 4567,
-    SslOpts = [
-        {cert, certfile(Config)},
-        {key, keyfile(Config)},
-        {idle_timeout_ms, 10000},
-        % QUIC_SERVER_RESUME_AND_ZERORTT
-        {server_resumption_level, 2},
-        {peer_bidi_stream_count, 10},
-        {alpn, ["mqtt"]}
-    ],
-    Server = quic_server:start_link(Port, SslOpts),
-    timer:sleep(500),
-    {ok, Sock} = emqtt_quic:connect(
-        "localhost",
-        Port,
-        [{alpn, ["mqtt"]}, {active, false}],
-        3000
-    ),
-    send_and_recv_with(Sock),
-    ok = emqtt_quic:close(Sock),
-    quic_server:stop(Server).
-
-t_quic_sock_fail(_Config) ->
-    Port = 4567,
-    Error1 =
-        {error,
-            {transport_down, #{
-                error => 2,
-                status => connection_refused
-            }}},
-    Error2 = {error, {transport_down, #{error => 1, status => unreachable}}},
-    case
-        emqtt_quic:connect(
-            "localhost",
-            Port,
-            [{alpn, ["mqtt"]}, {active, false}],
-            3000
-        )
-    of
-        Error1 ->
-            ok;
-        Error2 ->
-            ok;
-        Other ->
-            ct:fail("unexpected return ~p", [Other])
-    end.
-
-t_0_rtt(Config) ->
-    Port = 4568,
-    SslOpts = [
-        {cert, certfile(Config)},
-        {key, keyfile(Config)},
-        {idle_timeout_ms, 10000},
-        % QUIC_SERVER_RESUME_AND_ZERORTT
-        {server_resumption_level, 2},
-        {peer_bidi_stream_count, 10},
-        {alpn, ["mqtt"]}
-    ],
-    Server = quic_server:start_link(Port, SslOpts),
-    timer:sleep(500),
-    {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect(
-        "localhost",
-        Port,
-        [
-            {alpn, ["mqtt"]},
-            {active, false},
-            {quic_event_mask, 1}
-        ],
-        3000
-    ),
-    send_and_recv_with(Sock),
-    ok = emqtt_quic:close(Sock),
-    NST =
-        receive
-            {quic, nst_received, Conn, Ticket} ->
-                Ticket
-        end,
-    {ok, Sock2} = emqtt_quic:connect(
-        "localhost",
-        Port,
-        [
-            {alpn, ["mqtt"]},
-            {active, false},
-            {nst, NST}
-        ],
-        3000
-    ),
-    send_and_recv_with(Sock2),
-    ok = emqtt_quic:close(Sock2),
-    quic_server:stop(Server).
-
-t_0_rtt_fail(Config) ->
-    Port = 4569,
-    SslOpts = [
-        {cert, certfile(Config)},
-        {key, keyfile(Config)},
-        {idle_timeout_ms, 10000},
-        % QUIC_SERVER_RESUME_AND_ZERORTT
-        {server_resumption_level, 2},
-        {peer_bidi_stream_count, 10},
-        {alpn, ["mqtt"]}
-    ],
-    Server = quic_server:start_link(Port, SslOpts),
-    timer:sleep(500),
-    {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect(
-        "localhost",
-        Port,
-        [
-            {alpn, ["mqtt"]},
-            {active, false},
-            {quic_event_mask, 1}
-        ],
-        3000
-    ),
-    send_and_recv_with(Sock),
-    ok = emqtt_quic:close(Sock),
-    <<_Head:16, Left/binary>> =
-        receive
-            {quic, nst_received, Conn, Ticket} when is_binary(Ticket) ->
-                Ticket
-        end,
-
-    Error = {error, {not_found, invalid_parameter}},
-    Error = emqtt_quic:connect(
-        "localhost",
-        Port,
-        [
-            {alpn, ["mqtt"]},
-            {active, false},
-            {nst, Left}
-        ],
-        3000
-    ),
-    quic_server:stop(Server).
-
-t_multi_streams_sub(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    case emqtt:publish(C, Topic, <<"qos 2 1">>, PubQos) of
-        ok when PubQos == 0 -> ok;
-        {ok, _} -> ok
-    end,
-    receive
-        {publish, #{
-            client_pid := C,
-            payload := <<"qos 2 1">>,
-            qos := RecQos,
-            topic := Topic
-        }} ->
-            ok;
-        Other ->
-            ct:fail("unexpected recv ~p", [Other])
-    after 100 ->
-        ct:fail("not received")
-    end,
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_pub_5x100(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-
-    PubVias = lists:map(
-        fun(_N) ->
-            {ok, Via} = emqtt:start_data_stream(C, []),
-            Via
-        end,
-        lists:seq(1, 5)
-    ),
-    [
-        begin
-            case emqtt:publish_via(C, PVia, Topic, #{}, <<"stream data ", N>>, [{qos, PubQos}]) of
-                ok when PubQos == 0 -> ok;
-                {ok, _} -> ok
-            end,
-            0 == (N rem 10) andalso timer:sleep(10)
-        end
-     || N <- lists:seq(1, 100), PVia <- PubVias
-    ],
-    ?assert(timeout =/= recv_pub(500)),
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_pub_parallel(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-    PktId2 = calc_pkt_id(RecQos, 2),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
-    ok = emqtt:publish_async(
-        C,
-        {new_data_stream, []},
-        Topic,
-        <<"stream data 1">>,
-        [{qos, PubQos}],
-        undefined
-    ),
-    ok = emqtt:publish_async(
-        C,
-        {new_data_stream, []},
-        Topic,
-        <<"stream data 2">>,
-        [{qos, PubQos}],
-        undefined
-    ),
-    PubRecvs = recv_pub(2),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<"stream data", _/binary>>,
-                qos := RecQos,
-                topic := Topic
-            }},
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId2,
-                payload := <<"stream data", _/binary>>,
-                qos := RecQos,
-                topic := Topic
-            }}
-        ],
-        PubRecvs
-    ),
-    Payloads = [P || {publish, #{payload := P}} <- PubRecvs],
-    ?assert(
-        [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse
-            [<<"stream data 2">>, <<"stream data 1">>] == Payloads
-    ),
-    ok = emqtt:disconnect(C).
-
-%% @doc test two pub streams, one send incomplete MQTT packet() can not block another.
-t_multi_streams_pub_parallel_no_blocking(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId2 = calc_pkt_id(RecQos, 1),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
-    Drop = <<"stream data 1">>,
-    meck:new(emqtt_quic, [passthrough, no_history]),
-    meck:expect(emqtt_quic, send, fun(Sock, IoList) ->
-        case lists:last(IoList) == Drop of
-            true ->
-                ct:pal("meck droping ~p", [Drop]),
-                meck:passthrough([Sock, IoList -- [Drop]]);
-            false ->
-                meck:passthrough([Sock, IoList])
-        end
-    end),
-    ok = emqtt:publish_async(
-        C,
-        {new_data_stream, []},
-        Topic,
-        Drop,
-        [{qos, PubQos}],
-        undefined
-    ),
-    ok = emqtt:publish_async(
-        C,
-        {new_data_stream, []},
-        Topic,
-        <<"stream data 2">>,
-        [{qos, PubQos}],
-        undefined
-    ),
-    PubRecvs = recv_pub(1),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId2,
-                payload := <<"stream data 2">>,
-                qos := RecQos,
-                topic := Topic
-            }}
-        ],
-        PubRecvs
-    ),
-    meck:unload(emqtt_quic),
-    ?assertEqual(timeout, recv_pub(1)),
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_packet_boundary(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-    PktId2 = calc_pkt_id(RecQos, 2),
-    PktId3 = calc_pkt_id(RecQos, 3),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-
-    %% make quicer to batch job
-    quicer:reg_open(quic_execution_profile_type_max_throughput),
-
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
-
-    {ok, PubVia} = emqtt:start_data_stream(C, []),
-    ok = emqtt:publish_async(
-        C,
-        PubVia,
-        Topic,
-        <<"stream data 1">>,
-        [{qos, PubQos}],
-        undefined
-    ),
-    ok = emqtt:publish_async(
-        C,
-        PubVia,
-        Topic,
-        <<"stream data 2">>,
-        [{qos, PubQos}],
-        undefined
-    ),
-    LargePart3 = binary:copy(<<"stream data3">>, 2000),
-    ok = emqtt:publish_async(
-        C,
-        PubVia,
-        Topic,
-        LargePart3,
-        [{qos, PubQos}],
-        undefined
-    ),
-    PubRecvs = recv_pub(3),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<"stream data 1">>,
-                qos := RecQos,
-                topic := Topic
-            }},
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId2,
-                payload := <<"stream data 2">>,
-                qos := RecQos,
-                topic := Topic
-            }},
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId3,
-                payload := LargePart3,
-                qos := RecQos,
-                topic := Topic
-            }}
-        ],
-        PubRecvs
-    ),
-    ok = emqtt:disconnect(C).
-
-%% @doc test that one malformed stream will not close the entire connection
-t_multi_streams_packet_malform(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-    PktId2 = calc_pkt_id(RecQos, 2),
-    PktId3 = calc_pkt_id(RecQos, 3),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-
-    %% make quicer to batch job
-    quicer:reg_open(quic_execution_profile_type_max_throughput),
-
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
-
-    {ok, PubVia} = emqtt:start_data_stream(C, []),
-    ok = emqtt:publish_async(
-        C,
-        PubVia,
-        Topic,
-        <<"stream data 1">>,
-        [{qos, PubQos}],
-        undefined
-    ),
-
-    {ok, {quic, _Conn, MalformStream}} = emqtt:start_data_stream(C, []),
-    {ok, _} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>),
-
-    ok = emqtt:publish_async(
-        C,
-        PubVia,
-        Topic,
-        <<"stream data 2">>,
-        [{qos, PubQos}],
-        undefined
-    ),
-    LargePart3 = binary:copy(<<"stream data3">>, 2000),
-    ok = emqtt:publish_async(
-        C,
-        PubVia,
-        Topic,
-        LargePart3,
-        [{qos, PubQos}],
-        undefined
-    ),
-    PubRecvs = recv_pub(3),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<"stream data 1">>,
-                qos := RecQos,
-                topic := Topic
-            }},
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId2,
-                payload := <<"stream data 2">>,
-                qos := RecQos,
-                topic := Topic
-            }},
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId3,
-                payload := LargePart3,
-                qos := RecQos,
-                topic := Topic
-            }}
-        ],
-        PubRecvs
-    ),
-
-    case quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>) of
-        {ok, 10} -> ok;
-        {error, cancelled} -> ok;
-        {error, stm_send_error, aborted} -> ok
-    end,
-
-    timer:sleep(200),
-    ?assert(is_list(emqtt:info(C))),
-
-    {error, stm_send_error, aborted} = quicer:send(MalformStream, <<1, 2, 3, 4, 5, 6, 7, 8, 9, 0>>),
-
-    timer:sleep(200),
-    ?assert(is_list(emqtt:info(C))),
-
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_packet_too_large(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    meck:new(emqx_frame, [passthrough, no_history]),
-    ok = meck:expect(
-        emqx_frame,
-        serialize_opts,
-        fun(#mqtt_packet_connect{proto_ver = ProtoVer}) ->
-            #{version => ProtoVer, max_size => 1024}
-        end
-    ),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
-
-    {ok, PubVia} = emqtt:start_data_stream(C, []),
-    ok = emqtt:publish_async(
-        C,
-        PubVia,
-        Topic,
-        binary:copy(<<"stream data 1">>, 1024),
-        [{qos, PubQos}],
-        undefined
-    ),
-    timeout = recv_pub(1),
-    ?assert(is_list(emqtt:info(C))),
-    ok = meck:unload(emqx_frame),
-    ok = emqtt:disconnect(C).
-
-t_conn_change_client_addr(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
-
-    {ok, {quic, Conn, _} = PubVia} = emqtt:start_data_stream(C, []),
-    ok = emqtt:publish_async(
-        C,
-        PubVia,
-        Topic,
-        <<"stream data 1">>,
-        [{qos, PubQos}],
-        undefined
-    ),
-
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := _PktId1,
-                payload := <<"stream data 1">>,
-                qos := RecQos
-            }}
-        ],
-        recv_pub(1)
-    ),
-    NewPort = select_port(),
-    {ok, OldAddr} = quicer:sockname(Conn),
-    ?assertEqual(
-        ok, quicer:setopt(Conn, param_conn_local_address, "127.0.0.1:" ++ integer_to_list(NewPort))
-    ),
-    {ok, NewAddr} = quicer:sockname(Conn),
-    ct:pal("NewAddr: ~p, Old Addr: ~p", [NewAddr, OldAddr]),
-    ?assertNotEqual(OldAddr, NewAddr),
-    ?assert(is_list(emqtt:info(C))),
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_sub_pub_async(Config) ->
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-    Topic2 = <<Topic/binary, "_two">>,
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic2, [{qos, SubQos}]}
-    ]),
-    ok = emqtt:publish_async(
-        C,
-        {new_data_stream, []},
-        Topic,
-        <<"stream data 1">>,
-        [{qos, PubQos}],
-        undefined
-    ),
-    ok = emqtt:publish_async(
-        C,
-        {new_data_stream, []},
-        Topic2,
-        <<"stream data 2">>,
-        [{qos, PubQos}],
-        undefined
-    ),
-    PubRecvs = recv_pub(2),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<"stream data", _/binary>>,
-                qos := RecQos
-            }},
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<"stream data", _/binary>>,
-                qos := RecQos
-            }}
-        ],
-        PubRecvs
-    ),
-    Payloads = [P || {publish, #{payload := P}} <- PubRecvs],
-    ?assert(
-        [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse
-            [<<"stream data 2">>, <<"stream data 1">>] == Payloads
-    ),
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_sub_pub_sync(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    Topic2 = <<Topic/binary, "two">>,
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic2, [{qos, SubQos}]}
-    ]),
-
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 ->
-            Via1 = undefined,
-            ok;
-        {ok, #{reason_code := 0, via := Via1}} ->
-            ok
-    end,
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<"stream data 4">>, [
-            {qos, PubQos}
-        ])
-    of
-        ok when PubQos == 0 -> ok;
-        {ok, #{reason_code := 0, via := Via2}} ->
-            ?assert(Via1 =/= Via2),
-            ok
-    end,
-    ct:pal("SVia1: ~p, SVia2: ~p", [SVia1, SVia2]),
-    PubRecvs = recv_pub(2),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<"stream data 3">>,
-                qos := RecQos,
-                via := SVia1
-            }},
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<"stream data 4">>,
-                qos := RecQos,
-                via := SVia2
-            }}
-        ],
-        lists:sort(PubRecvs)
-    ),
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_dup_sub(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-
-    #{data_stream_socks := [{quic, _Conn, SubStream} | _]} = proplists:get_value(
-        extra, emqtt:info(C)
-    ),
-    ?assertEqual(2, length(emqx_broker:subscribers(Topic))),
-
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 ->
-            ok;
-        {ok, #{reason_code := 0, via := _Via1}} ->
-            ok
-    end,
-    PubRecvs = recv_pub(2),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<"stream data 3">>,
-                qos := RecQos
-            }},
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<"stream data 3">>,
-                qos := RecQos
-            }}
-        ],
-        lists:sort(PubRecvs)
-    ),
-
-    RecvVias = [Via || {publish, #{via := Via}} <- PubRecvs],
-
-    ct:pal("~p, ~p, ~n recv from: ~p~n", [SVia1, SVia2, PubRecvs]),
-    %% Can recv in any order
-    ?assert([SVia1, SVia2] == RecvVias orelse [SVia2, SVia1] == RecvVias),
-
-    %% Shutdown one stream
-    quicer:async_shutdown_stream(SubStream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 500),
-    timer:sleep(100),
-
-    ?assertEqual(1, length(emqx_broker:subscribers(Topic))),
-
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_corr_topic(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-    PktId2 = calc_pkt_id(RecQos, 2),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 ->
-            ok;
-        {ok, #{reason_code := 0, via := _Via}} ->
-            ok
-    end,
-
-    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
-    ?assert(PubVia =/= SubVia),
-
-    case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of
-        ok when PubQos == 0 -> ok;
-        {ok, #{reason_code := 0, via := PubVia}} -> ok
-    end,
-    PubRecvs = recv_pub(2),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<1, 2, 3, 4, 5>>,
-                qos := RecQos
-            }},
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId2,
-                payload := <<6, 7, 8, 9>>,
-                qos := RecQos
-            }}
-        ],
-        PubRecvs
-    ),
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_unsub(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 ->
-            ok;
-        {ok, #{reason_code := 0, via := _PVia}} ->
-            ok
-    end,
-
-    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
-    ?assert(PubVia =/= SubVia),
-    PubRecvs = recv_pub(1),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<1, 2, 3, 4, 5>>,
-                qos := RecQos
-            }}
-        ],
-        PubRecvs
-    ),
-
-    emqtt:unsubscribe_via(C, SubVia, Topic),
-
-    case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of
-        ok when PubQos == 0 ->
-            ok;
-        {ok, #{reason_code := 16, via := PubVia, reason_code_name := no_matching_subscribers}} ->
-            ok
-    end,
-
-    timeout = recv_pub(1),
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_kill_sub_stream(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    Topic2 = <<Topic/binary, "two">>,
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic2, [{qos, SubQos}]}
-    ]),
-    [TopicStreamOwner] = emqx_broker:subscribers(Topic),
-    exit(TopicStreamOwner, kill),
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 ->
-            ok;
-        {ok, #{reason_code := Code, via := _PVia}} when Code == 0 orelse Code == 16 ->
-            ok
-    end,
-
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 ->
-            ok;
-        {ok, #{reason_code := 0, via := _PVia2}} ->
-            ok
-    end,
-
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                topic := Topic2,
-                payload := <<1, 2, 3, 4, 5>>,
-                qos := RecQos
-            }}
-        ],
-        recv_pub(1)
-    ),
-    ?assertEqual(timeout, recv_pub(1)),
-    ok.
-
-t_multi_streams_unsub_via_other(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-    PktId2 = calc_pkt_id(RecQos, 2),
-
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    Topic2 = <<Topic/binary, "two">>,
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic2, [{qos, SubQos}]}
-    ]),
-
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 -> ok;
-        {ok, #{reason_code := 0, via := _PVia}} -> ok
-    end,
-
-    PubRecvs = recv_pub(1),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<1, 2, 3, 4, 5>>,
-                qos := RecQos
-            }}
-        ],
-        PubRecvs
-    ),
-
-    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
-
-    %% Unsub topic1 via stream2 should fail with error code 17: "No subscription existed"
-    {ok, #{via := SVia2}, [17]} = emqtt:unsubscribe_via(C, SVia2, Topic),
-
-    case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of
-        ok when PubQos == 0 -> ok;
-        {ok, #{reason_code := 0, via := _PVia2}} -> ok
-    end,
-
-    PubRecvs2 = recv_pub(1),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId2,
-                payload := <<6, 7, 8, 9>>,
-                qos := RecQos
-            }}
-        ],
-        PubRecvs2
-    ),
-    ok = emqtt:disconnect(C).
-
-t_multi_streams_shutdown_data_stream(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    Topic2 = <<Topic/binary, "two">>,
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic2, [{qos, SubQos}]}
-    ]),
-
-    ?assert(SVia =/= SVia2),
-
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 -> ok;
-        {ok, #{reason_code := 0, via := _PVia}} -> ok
-    end,
-
-    PubRecvs = recv_pub(1),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<1, 2, 3, 4, 5>>,
-                qos := RecQos
-            }}
-        ],
-        PubRecvs
-    ),
-
-    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
-    {quic, _Conn, DataStream} = PubVia,
-    quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100),
-    timer:sleep(500),
-    %% Still alive
-    ?assert(is_list(emqtt:info(C))).
-
-t_multi_streams_shutdown_ctrl_stream(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    Topic2 = <<Topic/binary, "two">>,
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    unlink(C),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic2, [{qos, SubQos}]}
-    ]),
-
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 -> ok;
-        {ok, #{reason_code := 0, via := _PVia}} -> ok
-    end,
-
-    PubRecvs = recv_pub(1),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<1, 2, 3, 4, 5>>,
-                qos := RecQos
-            }}
-        ],
-        PubRecvs
-    ),
-
-    {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
-    quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 1000),
-    timer:sleep(500),
-    %% Client should be closed
-    ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)).
-
-t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) ->
-    erlang:process_flag(trap_exit, true),
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    Topic2 = <<Topic/binary, "two">>,
-    {ok, C} = emqtt:start_link([
-        {proto_ver, v5},
-        {reconnect, true},
-        %% speedup test
-        {connect_timeout, 5}
-        | Config
-    ]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic2, [{qos, SubQos}]}
-    ]),
-
-    ?assert(SVia2 =/= SVia),
-
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 -> ok;
-        {ok, #{reason_code := 0, via := _PVia}} -> ok
-    end,
-
-    PubRecvs = recv_pub(1),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<1, 2, 3, 4, 5>>,
-                qos := RecQos
-            }}
-        ],
-        PubRecvs
-    ),
-
-    {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
-    quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 100),
-    timer:sleep(200),
-    %% Client should be closed
-    ?assert(is_list(emqtt:info(C))).
-
-t_multi_streams_remote_shutdown(Config) ->
-    erlang:process_flag(trap_exit, true),
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    Topic2 = <<Topic/binary, "two">>,
-    {ok, C} = emqtt:start_link([
-        {proto_ver, v5},
-        {reconnect, false},
-        %% speedup test
-        {connect_timeout, 5}
-        | Config
-    ]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic2, [{qos, SubQos}]}
-    ]),
-
-    ?assert(SVia2 =/= SVia),
-
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 -> ok;
-        {ok, #{reason_code := 0, via := _PVia}} -> ok
-    end,
-
-    PubRecvs = recv_pub(1),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<1, 2, 3, 4, 5>>,
-                qos := RecQos
-            }}
-        ],
-        PubRecvs
-    ),
-
-    {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
-
-    ok = stop_emqx(),
-
-    timer:sleep(200),
-    start_emqx_quic(?config(port, Config)),
-
-    %% Client should be closed
-    ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)).
-
-t_multi_streams_remote_shutdown_with_reconnect(Config) ->
-    erlang:process_flag(trap_exit, true),
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    PktId1 = calc_pkt_id(RecQos, 1),
-
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    Topic2 = <<Topic/binary, "two">>,
-    {ok, C} = emqtt:start_link([
-        {proto_ver, v5},
-        {reconnect, true},
-        %% speedup test
-        {connect_timeout, 5}
-        | Config
-    ]),
-    {ok, _} = emqtt:quic_connect(C),
-    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
-        {Topic2, [{qos, SubQos}]}
-    ]),
-
-    ?assert(SVia2 =/= SVia),
-
-    case
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
-    of
-        ok when PubQos == 0 -> ok;
-        {ok, #{reason_code := 0, via := _PVia}} -> ok
-    end,
-
-    PubRecvs = recv_pub(1),
-    ?assertMatch(
-        [
-            {publish, #{
-                client_pid := C,
-                packet_id := PktId1,
-                payload := <<1, 2, 3, 4, 5>>,
-                qos := RecQos
-            }}
-        ],
-        PubRecvs
-    ),
-
-    {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
-
-    ok = stop_emqx(),
-
-    timer:sleep(200),
-
-    start_emqx_quic(?config(port, Config)),
-    %% Client should be closed
-    ?assert(is_list(emqtt:info(C))).
-
-t_conn_silent_close(Config) ->
-    erlang:process_flag(trap_exit, true),
-    {ok, C} = emqtt:start_link([
-        {proto_ver, v5},
-        {connect_timeout, 5}
-        | Config
-    ]),
-    {ok, _} = emqtt:quic_connect(C),
-    %% quic idle timeout + 1s
-    timer:sleep(16000),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    ?assertException(
-        exit,
-        noproc,
-        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, 1}])
-    ).
-
-t_client_conn_bump_streams(Config) ->
-    {ok, C} = emqtt:start_link([
-        {proto_ver, v5},
-        {connect_timeout, 5}
-        | Config
-    ]),
-    {ok, _} = emqtt:quic_connect(C),
-    {quic, Conn, _Stream} = proplists:get_value(socket, emqtt:info(C)),
-    ok = quicer:setopt(Conn, param_conn_settings, #{peer_unidi_stream_count => 20}).
-
-t_olp_true(Config) ->
-    meck:new(emqx_olp, [passthrough, no_history]),
-    ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end),
-    {ok, C} = emqtt:start_link([
-        {proto_ver, v5},
-        {connect_timeout, 5}
-        | Config
-    ]),
-    {ok, _} = emqtt:quic_connect(C),
-    ok = meck:unload(emqx_olp).
-
-t_olp_reject(Config) ->
-    erlang:process_flag(trap_exit, true),
-    emqx_config:put_zone_conf(default, [overload_protection, enable], true),
-    meck:new(emqx_olp, [passthrough, no_history]),
-    ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end),
-    {ok, C} = emqtt:start_link([
-        {proto_ver, v5},
-        {connect_timeout, 5}
-        | Config
-    ]),
-    ?assertEqual(
-        {error,
-            {transport_down, #{
-                error => 346,
-                status =>
-                    user_canceled
-            }}},
-        emqtt:quic_connect(C)
-    ),
-    ok = meck:unload(emqx_olp),
-    emqx_config:put_zone_conf(default, [overload_protection, enable], false).
-
-t_conn_resume(Config) ->
-    erlang:process_flag(trap_exit, true),
-    {ok, C0} = emqtt:start_link([
-        {proto_ver, v5},
-        {connect_timeout, 5}
-        | Config
-    ]),
-
-    {ok, _} = emqtt:quic_connect(C0),
-    #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)),
-    emqtt:disconnect(C0),
-    {ok, C} = emqtt:start_link([
-        {proto_ver, v5},
-        {connect_timeout, 5},
-        {nst, NST}
-        | Config
-    ]),
-    {ok, _} = emqtt:quic_connect(C),
-    Cid = proplists:get_value(clientid, emqtt:info(C)),
-    ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]).
-
-t_conn_without_ctrl_stream(Config) ->
-    erlang:process_flag(trap_exit, true),
-    {ok, Conn} = quicer:connect(
-        {127, 0, 0, 1},
-        ?config(port, Config),
-        [{alpn, ["mqtt"]}, {verify, none}],
-        3000
-    ),
-    receive
-        {quic, transport_shutdown, Conn, _} -> ok
-    end.
-
-t_data_stream_race_ctrl_stream(Config) ->
-    erlang:process_flag(trap_exit, true),
-    {ok, C0} = emqtt:start_link([
-        {proto_ver, v5},
-        {connect_timeout, 5}
-        | Config
-    ]),
-    {ok, _} = emqtt:quic_connect(C0),
-    #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)),
-    emqtt:disconnect(C0),
-    {ok, C} = emqtt:start_link([
-        {proto_ver, v5},
-        {connect_timeout, 5},
-        {nst, NST}
-        | Config
-    ]),
-    {ok, _} = emqtt:quic_connect(C),
-    Cid = proplists:get_value(clientid, emqtt:info(C)),
-    ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]).
-
-t_multi_streams_sub_0_rtt(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C0),
-    {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    ok = emqtt:open_quic_connection(C),
-    ok = emqtt:quic_mqtt_connect(C),
-    ok = emqtt:publish_async(
-        C,
-        {new_data_stream, []},
-        Topic,
-        #{},
-        <<"qos 2 1">>,
-        [{qos, PubQos}],
-        infinity,
-        fun(_) -> ok end
-    ),
-    {ok, _} = emqtt:quic_connect(C),
-    receive
-        {publish, #{
-            client_pid := C0,
-            payload := <<"qos 2 1">>,
-            qos := RecQos,
-            topic := Topic
-        }} ->
-            ok;
-        Other ->
-            ct:fail("unexpected recv ~p", [Other])
-    after 100 ->
-        ct:fail("not received")
-    end,
-    ok = emqtt:disconnect(C),
-    ok = emqtt:disconnect(C0).
-
-t_multi_streams_sub_0_rtt_large_payload(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    Payload = binary:copy(<<"qos 2 1">>, 1600),
-    {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C0),
-    {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    ok = emqtt:open_quic_connection(C),
-    ok = emqtt:quic_mqtt_connect(C),
-    ok = emqtt:publish_async(
-        C,
-        {new_data_stream, []},
-        Topic,
-        #{},
-        Payload,
-        [{qos, PubQos}],
-        infinity,
-        fun(_) -> ok end
-    ),
-    {ok, _} = emqtt:quic_connect(C),
-    receive
-        {publish, #{
-            client_pid := C0,
-            payload := Payload,
-            qos := RecQos,
-            topic := Topic
-        }} ->
-            ok;
-        Other ->
-            ct:fail("unexpected recv ~p", [Other])
-    after 100 ->
-        ct:fail("not received")
-    end,
-    ok = emqtt:disconnect(C),
-    ok = emqtt:disconnect(C0).
-
-%% @doc verify data stream can continue after 0-RTT handshake
-t_multi_streams_sub_0_rtt_stream_data_cont(Config) ->
-    PubQos = ?config(pub_qos, Config),
-    SubQos = ?config(sub_qos, Config),
-    RecQos = calc_qos(PubQos, SubQos),
-    Topic = atom_to_binary(?FUNCTION_NAME),
-    Payload = binary:copy(<<"qos 2 1">>, 1600),
-    {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]),
-    {ok, _} = emqtt:quic_connect(C0),
-    {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [
-        {Topic, [{qos, SubQos}]}
-    ]),
-    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
-    ok = emqtt:open_quic_connection(C),
-    ok = emqtt:quic_mqtt_connect(C),
-    {ok, PubVia} = emqtt:start_data_stream(C, []),
-    ok = emqtt:publish_async(
-        C,
-        PubVia,
-        Topic,
-        #{},
-        Payload,
-        [{qos, PubQos}],
-        infinity,
-        fun(_) -> ok end
-    ),
-    {ok, _} = emqtt:quic_connect(C),
-    receive
-        {publish, #{
-            client_pid := C0,
-            payload := Payload,
-            qos := RecQos,
-            topic := Topic
-        }} ->
-            ok;
-        Other ->
-            ct:fail("unexpected recv ~p", [Other])
-    after 100 ->
-        ct:fail("not received")
-    end,
-    Payload2 = <<"2nd part", Payload/binary>>,
-    ok = emqtt:publish_async(
-        C,
-        PubVia,
-        Topic,
-        #{},
-        Payload2,
-        [{qos, PubQos}],
-        infinity,
-        fun(_) -> ok end
-    ),
-    receive
-        {publish, #{
-            client_pid := C0,
-            payload := Payload2,
-            qos := RecQos,
-            topic := Topic
-        }} ->
-            ok;
-        Other2 ->
-            ct:fail("unexpected recv ~p", [Other2])
-    after 100 ->
-        ct:fail("not received")
-    end,
-    ok = emqtt:disconnect(C),
-    ok = emqtt:disconnect(C0).
-
-%%--------------------------------------------------------------------
-%% Helper functions
-%%--------------------------------------------------------------------
-send_and_recv_with(Sock) ->
-    {ok, {IP, _}} = emqtt_quic:sockname(Sock),
-    ?assert(lists:member(tuple_size(IP), [4, 8])),
-    ok = emqtt_quic:send(Sock, <<"ping">>),
-    emqtt_quic:setopts(Sock, [{active, false}]),
-    {ok, <<"pong">>} = emqtt_quic:recv(Sock, 0),
-    ok = emqtt_quic:setopts(Sock, [{active, 100}]),
-    {ok, Stats} = emqtt_quic:getstat(Sock, [send_cnt, recv_cnt]),
-    %% connection level counters, not stream level
-    [{send_cnt, _}, {recv_cnt, _}] = Stats.
-
-certfile(Config) ->
-    filename:join([test_dir(Config), "certs", "test.crt"]).
-
-keyfile(Config) ->
-    filename:join([test_dir(Config), "certs", "test.key"]).
-
-test_dir(Config) ->
-    filename:dirname(filename:dirname(proplists:get_value(data_dir, Config))).
-
-recv_pub(Count) ->
-    recv_pub(Count, []).
-
-recv_pub(0, Acc) ->
-    lists:reverse(Acc);
-recv_pub(Count, Acc) ->
-    receive
-        {publish, _Prop} = Pub ->
-            recv_pub(Count - 1, [Pub | Acc])
-    after 100 ->
-        timeout
-    end.
-
-all_tc() ->
-    code:add_patha(filename:join(code:lib_dir(emqx), "ebin/")),
-    emqx_common_test_helpers:all(?MODULE).
-
--spec calc_qos(0 | 1 | 2, 0 | 1 | 2) -> 0 | 1 | 2.
-calc_qos(PubQos, SubQos) ->
-    if
-        PubQos > SubQos ->
-            SubQos;
-        SubQos > PubQos ->
-            PubQos;
-        true ->
-            PubQos
-    end.
--spec calc_pkt_id(0 | 1 | 2, non_neg_integer()) -> undefined | non_neg_integer().
-calc_pkt_id(0, _Id) ->
-    undefined;
-calc_pkt_id(1, Id) ->
-    Id;
-calc_pkt_id(2, Id) ->
-    Id.
-
--spec start_emqx_quic(inet:port_number()) -> ok.
-start_emqx_quic(UdpPort) ->
-    emqx_common_test_helpers:start_apps([]),
-    application:ensure_all_started(quicer),
-    emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort).
-
--spec stop_emqx() -> ok.
-stop_emqx() ->
-    emqx_common_test_helpers:stop_apps([]).
-
-%% select a random port picked by OS
--spec select_port() -> inet:port_number().
-select_port() ->
-    {ok, S} = gen_udp:open(0, [{reuseaddr, true}]),
-    {ok, {_, Port}} = inet:sockname(S),
-    gen_udp:close(S),
-    case os:type() of
-        {unix, darwin} ->
-            %% in MacOS, still get address_in_use after close port
-            timer:sleep(500);
-        _ ->
-            skip
-    end,
-    ct:pal("select port: ~p", [Port]),
-    Port.
diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
index 5a9abc7f4..0199bbc10 100644
--- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
+++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
@@ -79,19 +79,6 @@ end_per_group(_Group, _Config) ->
 
 init_per_suite(Config) ->
     %% Start Apps
-    dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}),
-    dbg:p(all, c),
-    dbg:tp(emqx_quic_connection, cx),
-    dbg:tp(quicer_connection, cx),
-    %% dbg:tp(emqx_quic_stream, cx),
-    %% dbg:tp(emqtt_quic, cx),
-    %% dbg:tp(emqtt, cx),
-    %% dbg:tp(emqtt_quic_stream, cx),
-    %% dbg:tp(emqtt_quic_connection, cx),
-    %% dbg:tp(emqx_cm, open_session, cx),
-    %% dbg:tpl(emqx_cm, lookup_channels, cx),
-    %% dbg:tpl(emqx_cm, register_channel, cx),
-    %% dbg:tpl(emqx_cm, unregister_channel, cx),
     emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
     Config.
diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index bb19092f7..b6d3c661c 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -13,178 +13,1748 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
-
 -module(emqx_quic_multistreams_SUITE).
 
 -compile(export_all).
+-compile(nowarn_export_all).
 
--include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-include_lib("quicer/include/quicer.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
--define(TOPICS, [
-    <<"TopicA">>,
-    <<"TopicA/B">>,
-    <<"Topic/C">>,
-    <<"TopicA/C">>,
-    <<"/TopicA">>
-]).
-
-%%--------------------------------------------------------------------
-%% @spec suite() -> Info
-%% Info = [tuple()]
-%% @end
-%%--------------------------------------------------------------------
 suite() ->
     [{timetrap, {seconds, 30}}].
 
-%%--------------------------------------------------------------------
-%% @spec init_per_suite(Config0) ->
-%%     Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
-%% Config0 = Config1 = [tuple()]
-%% Reason = term()
-%% @end
-%%--------------------------------------------------------------------
-init_per_suite(Config) ->
-    UdpPort = 1884,
-    emqx_common_test_helpers:boot_modules(all),
-    emqx_common_test_helpers:start_apps([]),
-    emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort),
-    %% @TODO remove
-    emqx_logger:set_log_level(debug),
-
-    dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}),
-    dbg:p(all, c),
-
-    %dbg:tp(emqx_quic_stream, cx),
-    %% dbg:tp(quicer_stream, cx),
-    %% dbg:tp(emqx_quic_data_stream, cx),
-    %% dbg:tp(emqx_channel, cx),
-    %% dbg:tp(emqx_packet,check,cx),
-    %% dbg:tp(emqx_frame,parse,cx),
-    %dbg:tp(emqx_quic_connection, cx),
-    [{port, UdpPort}, {conn_fun, quic_connect} | Config].
-
-%%--------------------------------------------------------------------
-%% @spec end_per_suite(Config0) -> term() | {save_config,Config1}
-%% Config0 = Config1 = [tuple()]
-%% @end
-%%--------------------------------------------------------------------
-end_per_suite(_Config) ->
-    ok.
-
-%%--------------------------------------------------------------------
-%% @spec init_per_group(GroupName, Config0) ->
-%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
-%% GroupName = atom()
-%% Config0 = Config1 = [tuple()]
-%% Reason = term()
-%% @end
-%%--------------------------------------------------------------------
-init_per_group(_GroupName, Config) ->
-    Config.
-
-%%--------------------------------------------------------------------
-%% @spec end_per_group(GroupName, Config0) ->
-%%               term() | {save_config,Config1}
-%% GroupName = atom()
-%% Config0 = Config1 = [tuple()]
-%% @end
-%%--------------------------------------------------------------------
-end_per_group(_GroupName, _Config) ->
-    ok.
-
-%%--------------------------------------------------------------------
-%% @spec init_per_testcase(TestCase, Config0) ->
-%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
-%% TestCase = atom()
-%% Config0 = Config1 = [tuple()]
-%% Reason = term()
-%% @end
-%%--------------------------------------------------------------------
-init_per_testcase(_TestCase, Config) ->
-    Config.
-
-%%--------------------------------------------------------------------
-%% @spec end_per_testcase(TestCase, Config0) ->
-%%               term() | {save_config,Config1} | {fail,Reason}
-%% TestCase = atom()
-%% Config0 = Config1 = [tuple()]
-%% Reason = term()
-%% @end
-%%--------------------------------------------------------------------
-end_per_testcase(_TestCase, _Config) ->
-    ok.
-
-%%--------------------------------------------------------------------
-%% @spec groups() -> [Group]
-%% Group = {GroupName,Properties,GroupsAndTestCases}
-%% GroupName = atom()
-%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
-%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
-%% TestCase = atom()
-%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
-%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
-%%              repeat_until_any_ok | repeat_until_any_fail
-%% N = integer() | forever
-%% @end
-%%--------------------------------------------------------------------
-groups() ->
-    [].
-
-%%--------------------------------------------------------------------
-%% @spec all() -> GroupsAndTestCases | {skip,Reason}
-%% GroupsAndTestCases = [{group,GroupName} | TestCase]
-%% GroupName = atom()
-%% TestCase = atom()
-%% Reason = term()
-%% @end
-%%--------------------------------------------------------------------
 all() ->
     [
-        tc_data_stream_sub
+        {group, mstream},
+        {group, shutdown},
+        {group, misc}
     ].
 
-%%--------------------------------------------------------------------
-%% @spec TestCase(Config0) ->
-%%               ok | exit() | {skip,Reason} | {comment,Comment} |
-%%               {save_config,Config1} | {skip_and_save,Reason,Config1}
-%% Config0 = Config1 = [tuple()]
-%% Reason = term()
-%% Comment = term()
-%% @end
-%%--------------------------------------------------------------------
+groups() ->
+    [
+        {mstream, [], [{group, profiles}]},
 
-%% @doc Test MQTT Subscribe via data_stream
-tc_data_stream_sub(Config) ->
-    Topic = lists:nth(1, ?TOPICS),
+        {profiles, [], [
+            {group, profile_low_latency},
+            {group, profile_max_throughput}
+        ]},
+        {profile_low_latency, [], [
+            {group, pub_qos0},
+            {group, pub_qos1},
+            {group, pub_qos2}
+        ]},
+        {profile_max_throughput, [], [
+            {group, pub_qos0},
+            {group, pub_qos1},
+            {group, pub_qos2}
+        ]},
+        {pub_qos0, [], [
+            {group, sub_qos0},
+            {group, sub_qos1},
+            {group, sub_qos2}
+        ]},
+        {pub_qos1, [], [
+            {group, sub_qos0},
+            {group, sub_qos1},
+            {group, sub_qos2}
+        ]},
+        {pub_qos2, [], [
+            {group, sub_qos0},
+            {group, sub_qos1},
+            {group, sub_qos2}
+        ]},
+        {sub_qos0, [{group, qos}]},
+        {sub_qos1, [{group, qos}]},
+        {sub_qos2, [{group, qos}]},
+        {qos, [
+            t_multi_streams_sub,
+            t_multi_streams_pub_5x100,
+            t_multi_streams_pub_parallel,
+            t_multi_streams_pub_parallel_no_blocking,
+            t_multi_streams_sub_pub_async,
+            t_multi_streams_sub_pub_sync,
+            t_multi_streams_unsub,
+            t_multi_streams_corr_topic,
+            t_multi_streams_unsub_via_other,
+            t_multi_streams_dup_sub,
+            t_multi_streams_packet_boundary,
+            t_multi_streams_packet_malform,
+            t_multi_streams_kill_sub_stream,
+            t_multi_streams_packet_too_large,
+            t_multi_streams_sub_0_rtt,
+            t_multi_streams_sub_0_rtt_large_payload,
+            t_multi_streams_sub_0_rtt_stream_data_cont,
+            t_conn_change_client_addr
+        ]},
+
+        {shutdown, [
+            {group, graceful_shutdown},
+            {group, abort_recv_shutdown},
+            {group, abort_send_shutdown},
+            {group, abort_send_recv_shutdown}
+        ]},
+
+        {graceful_shutdown, [
+            {group, ctrl_stream_shutdown},
+            {group, data_stream_shutdown}
+        ]},
+        {abort_recv_shutdown, [
+            {group, ctrl_stream_shutdown},
+            {group, data_stream_shutdown}
+        ]},
+        {abort_send_shutdown, [
+            {group, ctrl_stream_shutdown},
+            {group, data_stream_shutdown}
+        ]},
+        {abort_send_recv_shutdown, [
+            {group, ctrl_stream_shutdown},
+            {group, data_stream_shutdown}
+        ]},
+
+        {ctrl_stream_shutdown, [
+            t_multi_streams_shutdown_ctrl_stream,
+            t_multi_streams_shutdown_ctrl_stream_then_reconnect,
+            t_multi_streams_remote_shutdown,
+            t_multi_streams_remote_shutdown_with_reconnect
+        ]},
+
+        {data_stream_shutdown, [
+            t_multi_streams_shutdown_pub_data_stream,
+            t_multi_streams_shutdown_sub_data_stream
+        ]},
+        {misc, [
+            t_conn_silent_close,
+            t_client_conn_bump_streams,
+            t_olp_true,
+            t_olp_reject,
+            t_conn_resume,
+            t_conn_without_ctrl_stream
+        ]}
+    ].
+
+init_per_suite(Config) ->
+    emqx_common_test_helpers:start_apps([]),
+    UdpPort = 14567,
+    start_emqx_quic(UdpPort),
+    %% dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}),
+    %% dbg:p(all, c),
+    %% dbg:tpl(quicer_stream, handle_info, c),
+    %% dbg:tp(emqx_quic_connection, cx),
+    %% dbg:tp(emqx_quic_stream, cx),
+    %% dbg:tp(emqtt, cx),
+    %% dbg:tpl(emqtt_quic_stream, cx),
+    %% dbg:tpl(emqx_quic_stream, cx),
+    %% dbg:tpl(emqx_quic_data_stream, cx),
+    %% dbg:tpl(emqtt, cx),
+    [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config].
+
+end_per_suite(_) ->
+    ok.
+
+init_per_group(pub_qos0, Config) ->
+    [{pub_qos, 0} | Config];
+init_per_group(sub_qos0, Config) ->
+    [{sub_qos, 0} | Config];
+init_per_group(pub_qos1, Config) ->
+    [{pub_qos, 1} | Config];
+init_per_group(sub_qos1, Config) ->
+    [{sub_qos, 1} | Config];
+init_per_group(pub_qos2, Config) ->
+    [{pub_qos, 2} | Config];
+init_per_group(sub_qos2, Config) ->
+    [{sub_qos, 2} | Config];
+init_per_group(abort_send_shutdown, Config) ->
+    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND} | Config];
+init_per_group(abort_recv_shutdown, Config) ->
+    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE} | Config];
+init_per_group(abort_send_recv_shutdown, Config) ->
+    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT} | Config];
+init_per_group(graceful_shutdown, Config) ->
+    [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL} | Config];
+init_per_group(profile_max_throughput, Config) ->
+    quicer:reg_open(quic_execution_profile_type_max_throughput),
+    Config;
+init_per_group(profile_low_latency, Config) ->
+    quicer:reg_open(quic_execution_profile_low_latency),
+    Config;
+init_per_group(_, Config) ->
+    Config.
+
+end_per_group(_, Config) ->
+    Config.
+
+init_per_testcase(_, Config) ->
+    emqx_common_test_helpers:start_apps([]),
+    Config.
+
+t_quic_sock(Config) ->
+    Port = 4567,
+    SslOpts = [
+        {cert, certfile(Config)},
+        {key, keyfile(Config)},
+        {idle_timeout_ms, 10000},
+        % QUIC_SERVER_RESUME_AND_ZERORTT
+        {server_resumption_level, 2},
+        {peer_bidi_stream_count, 10},
+        {alpn, ["mqtt"]}
+    ],
+    Server = quic_server:start_link(Port, SslOpts),
+    timer:sleep(500),
+    {ok, Sock} = emqtt_quic:connect(
+        "localhost",
+        Port,
+        [{alpn, ["mqtt"]}, {active, false}],
+        3000
+    ),
+    send_and_recv_with(Sock),
+    ok = emqtt_quic:close(Sock),
+    quic_server:stop(Server).
+
+t_quic_sock_fail(_Config) ->
+    Port = 4567,
+    Error1 =
+        {error,
+            {transport_down, #{
+                error => 2,
+                status => connection_refused
+            }}},
+    Error2 = {error, {transport_down, #{error => 1, status => unreachable}}},
+    case
+        emqtt_quic:connect(
+            "localhost",
+            Port,
+            [{alpn, ["mqtt"]}, {active, false}],
+            3000
+        )
+    of
+        Error1 ->
+            ok;
+        Error2 ->
+            ok;
+        Other ->
+            ct:fail("unexpected return ~p", [Other])
+    end.
+
+t_0_rtt(Config) ->
+    Port = 4568,
+    SslOpts = [
+        {cert, certfile(Config)},
+        {key, keyfile(Config)},
+        {idle_timeout_ms, 10000},
+        % QUIC_SERVER_RESUME_AND_ZERORTT
+        {server_resumption_level, 2},
+        {peer_bidi_stream_count, 10},
+        {alpn, ["mqtt"]}
+    ],
+    Server = quic_server:start_link(Port, SslOpts),
+    timer:sleep(500),
+    {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect(
+        "localhost",
+        Port,
+        [
+            {alpn, ["mqtt"]},
+            {active, false},
+            {quic_event_mask, 1}
+        ],
+        3000
+    ),
+    send_and_recv_with(Sock),
+    ok = emqtt_quic:close(Sock),
+    NST =
+        receive
+            {quic, nst_received, Conn, Ticket} ->
+                Ticket
+        end,
+    {ok, Sock2} = emqtt_quic:connect(
+        "localhost",
+        Port,
+        [
+            {alpn, ["mqtt"]},
+            {active, false},
+            {nst, NST}
+        ],
+        3000
+    ),
+    send_and_recv_with(Sock2),
+    ok = emqtt_quic:close(Sock2),
+    quic_server:stop(Server).
+
+t_0_rtt_fail(Config) ->
+    Port = 4569,
+    SslOpts = [
+        {cert, certfile(Config)},
+        {key, keyfile(Config)},
+        {idle_timeout_ms, 10000},
+        % QUIC_SERVER_RESUME_AND_ZERORTT
+        {server_resumption_level, 2},
+        {peer_bidi_stream_count, 10},
+        {alpn, ["mqtt"]}
+    ],
+    Server = quic_server:start_link(Port, SslOpts),
+    timer:sleep(500),
+    {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect(
+        "localhost",
+        Port,
+        [
+            {alpn, ["mqtt"]},
+            {active, false},
+            {quic_event_mask, 1}
+        ],
+        3000
+    ),
+    send_and_recv_with(Sock),
+    ok = emqtt_quic:close(Sock),
+    <<_Head:16, Left/binary>> =
+        receive
+            {quic, nst_received, Conn, Ticket} when is_binary(Ticket) ->
+                Ticket
+        end,
+
+    Error = {error, {not_found, invalid_parameter}},
+    Error = emqtt_quic:connect(
+        "localhost",
+        Port,
+        [
+            {alpn, ["mqtt"]},
+            {active, false},
+            {nst, Left}
+        ],
+        3000
+    ),
+    quic_server:stop(Server).
+
+t_multi_streams_sub(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    Topic = atom_to_binary(?FUNCTION_NAME),
     {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
     {ok, _} = emqtt:quic_connect(C),
-    {ok, _, [1]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [{Topic, [{qos, qos1}]}]),
-    {ok, _, [2]} = emqtt:subscribe_via(
-        C,
-        {new_data_stream, []},
-        #{},
-        [{lists:nth(2, ?TOPICS), [{qos, qos2}]}]
-    ),
-    {ok, _} = emqtt:publish(C, Topic, <<"qos 2 1">>, 2),
-    {ok, _} = emqtt:publish(C, Topic, <<"qos 2 2">>, 2),
-    {ok, _} = emqtt:publish(C, Topic, <<"qos 2 3">>, 2),
-    Msgs = receive_messages(3),
-    ct:pal("recv msg: ~p", [Msgs]),
-    ?assertEqual(3, length(Msgs)),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    case emqtt:publish(C, Topic, <<"qos 2 1">>, PubQos) of
+        ok when PubQos == 0 -> ok;
+        {ok, _} -> ok
+    end,
+    receive
+        {publish, #{
+            client_pid := C,
+            payload := <<"qos 2 1">>,
+            qos := RecQos,
+            topic := Topic
+        }} ->
+            ok;
+        Other ->
+            ct:fail("unexpected recv ~p", [Other])
+    after 100 ->
+        ct:fail("not received")
+    end,
     ok = emqtt:disconnect(C).
 
-receive_messages(Count) ->
-    receive_messages(Count, []).
+t_multi_streams_pub_5x100(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
 
-receive_messages(0, Msgs) ->
-    Msgs;
-receive_messages(Count, Msgs) ->
+    PubVias = lists:map(
+        fun(_N) ->
+            {ok, Via} = emqtt:start_data_stream(C, []),
+            Via
+        end,
+        lists:seq(1, 5)
+    ),
+    CtrlVia = proplists:get_value(socket, emqtt:info(C)),
+    [
+        begin
+            case emqtt:publish_via(C, PVia, Topic, #{}, <<"stream data ", N>>, [{qos, PubQos}]) of
+                ok when PubQos == 0 -> ok;
+                {ok, _} -> ok
+            end,
+            0 == (N rem 10) andalso timer:sleep(10)
+        end
+     || %% also publish on control stream
+        N <- lists:seq(1, 100),
+        PVia <- [CtrlVia | PubVias]
+    ],
+    ?assert(timeout =/= recv_pub(600)),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_pub_parallel(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        <<"stream data 1">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data", _/binary>>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<"stream data", _/binary>>,
+                qos := RecQos,
+                topic := Topic
+            }}
+        ],
+        PubRecvs
+    ),
+    Payloads = [P || {publish, #{payload := P}} <- PubRecvs],
+    ?assert(
+        [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse
+            [<<"stream data 2">>, <<"stream data 1">>] == Payloads
+    ),
+    ok = emqtt:disconnect(C).
+
+%% @doc test two pub streams, one send incomplete MQTT packet() can not block another.
+t_multi_streams_pub_parallel_no_blocking(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId2 = calc_pkt_id(RecQos, 1),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+    Drop = <<"stream data 1">>,
+    meck:new(emqtt_quic, [passthrough, no_history]),
+    meck:expect(emqtt_quic, send, fun(Sock, IoList) ->
+        case lists:last(IoList) == Drop of
+            true ->
+                ct:pal("meck droping ~p", [Drop]),
+                meck:passthrough([Sock, IoList -- [Drop]]);
+            false ->
+                meck:passthrough([Sock, IoList])
+        end
+    end),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        Drop,
+        [{qos, PubQos}],
+        undefined
+    ),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<"stream data 2">>,
+                qos := RecQos,
+                topic := Topic
+            }}
+        ],
+        PubRecvs
+    ),
+    meck:unload(emqtt_quic),
+    ?assertEqual(timeout, recv_pub(1)),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_packet_boundary(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+    PktId3 = calc_pkt_id(RecQos, 3),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+
+    %% make quicer to batch job
+    quicer:reg_open(quic_execution_profile_type_max_throughput),
+
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+
+    {ok, PubVia} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 1">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    LargePart3 = binary:copy(<<"stream data3">>, 2000),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        LargePart3,
+        [{qos, PubQos}],
+        undefined
+    ),
+    PubRecvs = recv_pub(3),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 1">>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<"stream data 2">>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId3,
+                payload := LargePart3,
+                qos := RecQos,
+                topic := Topic
+            }}
+        ],
+        PubRecvs
+    ),
+    ok = emqtt:disconnect(C).
+
+%% @doc test that one malformed stream will not close the entire connection
+t_multi_streams_packet_malform(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+    PktId3 = calc_pkt_id(RecQos, 3),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+
+    %% make quicer to batch job
+    quicer:reg_open(quic_execution_profile_type_max_throughput),
+
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+
+    {ok, PubVia} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 1">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+
+    {ok, {quic, _Conn, MalformStream}} = emqtt:start_data_stream(C, []),
+    {ok, _} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>),
+
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    LargePart3 = binary:copy(<<"stream data3">>, 2000),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        LargePart3,
+        [{qos, PubQos}],
+        undefined
+    ),
+    PubRecvs = recv_pub(3),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 1">>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<"stream data 2">>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId3,
+                payload := LargePart3,
+                qos := RecQos,
+                topic := Topic
+            }}
+        ],
+        PubRecvs
+    ),
+
+    case quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>) of
+        {ok, 10} -> ok;
+        {error, cancelled} -> ok;
+        {error, stm_send_error, aborted} -> ok
+    end,
+
+    timer:sleep(200),
+    ?assert(is_list(emqtt:info(C))),
+
+    {error, stm_send_error, aborted} = quicer:send(MalformStream, <<1, 2, 3, 4, 5, 6, 7, 8, 9, 0>>),
+
+    timer:sleep(200),
+    ?assert(is_list(emqtt:info(C))),
+
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_packet_too_large(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    meck:new(emqx_frame, [passthrough, no_history]),
+    ok = meck:expect(
+        emqx_frame,
+        serialize_opts,
+        fun(#mqtt_packet_connect{proto_ver = ProtoVer}) ->
+            #{version => ProtoVer, max_size => 1024}
+        end
+    ),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+
+    {ok, PubVia} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        binary:copy(<<"stream data 1">>, 1024),
+        [{qos, PubQos}],
+        undefined
+    ),
+    timeout = recv_pub(1),
+    ?assert(is_list(emqtt:info(C))),
+    ok = meck:unload(emqx_frame),
+    ok = emqtt:disconnect(C).
+
+t_conn_change_client_addr(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
+
+    {ok, {quic, Conn, _} = PubVia} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 1">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := _PktId1,
+                payload := <<"stream data 1">>,
+                qos := RecQos
+            }}
+        ],
+        recv_pub(1)
+    ),
+    NewPort = select_port(),
+    {ok, OldAddr} = quicer:sockname(Conn),
+    ?assertEqual(
+        ok, quicer:setopt(Conn, param_conn_local_address, "127.0.0.1:" ++ integer_to_list(NewPort))
+    ),
+    {ok, NewAddr} = quicer:sockname(Conn),
+    ct:pal("NewAddr: ~p, Old Addr: ~p", [NewAddr, OldAddr]),
+    ?assertNotEqual(OldAddr, NewAddr),
+    ?assert(is_list(emqtt:info(C))),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_sub_pub_async(Config) ->
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    Topic2 = <<Topic/binary, "_two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        <<"stream data 1">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic2,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data", _/binary>>,
+                qos := RecQos
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data", _/binary>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+    Payloads = [P || {publish, #{payload := P}} <- PubRecvs],
+    ?assert(
+        [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse
+            [<<"stream data 2">>, <<"stream data 1">>] == Payloads
+    ),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_sub_pub_sync(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            Via1 = undefined,
+            ok;
+        {ok, #{reason_code := 0, via := Via1}} ->
+            ok
+    end,
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<"stream data 4">>, [
+            {qos, PubQos}
+        ])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := Via2}} ->
+            ?assert(Via1 =/= Via2),
+            ok
+    end,
+    ct:pal("SVia1: ~p, SVia2: ~p", [SVia1, SVia2]),
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 3">>,
+                qos := RecQos,
+                via := SVia1
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 4">>,
+                qos := RecQos,
+                via := SVia2
+            }}
+        ],
+        lists:sort(PubRecvs)
+    ),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_dup_sub(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+
+    #{data_stream_socks := [{quic, _Conn, SubStream} | _]} = proplists:get_value(
+        extra, emqtt:info(C)
+    ),
+    ?assertEqual(2, length(emqx_broker:subscribers(Topic))),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := 0, via := _Via1}} ->
+            ok
+    end,
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 3">>,
+                qos := RecQos
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 3">>,
+                qos := RecQos
+            }}
+        ],
+        lists:sort(PubRecvs)
+    ),
+
+    RecvVias = [Via || {publish, #{via := Via}} <- PubRecvs],
+
+    ct:pal("~p, ~p, ~n recv from: ~p~n", [SVia1, SVia2, PubRecvs]),
+    %% Can recv in any order
+    ?assert([SVia1, SVia2] == RecvVias orelse [SVia2, SVia1] == RecvVias),
+
+    %% Shutdown one stream
+    quicer:async_shutdown_stream(SubStream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 500),
+    timer:sleep(100),
+
+    ?assertEqual(1, length(emqx_broker:subscribers(Topic))),
+
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_corr_topic(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := 0, via := _Via}} ->
+            ok
+    end,
+
+    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
+    ?assert(PubVia =/= SubVia),
+
+    case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := PubVia}} -> ok
+    end,
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<6, 7, 8, 9>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_unsub(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := 0, via := _PVia}} ->
+            ok
+    end,
+
+    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
+    ?assert(PubVia =/= SubVia),
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    emqtt:unsubscribe_via(C, SubVia, Topic),
+
+    case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := 16, via := PubVia, reason_code_name := no_matching_subscribers}} ->
+            ok
+    end,
+
+    timeout = recv_pub(1),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_kill_sub_stream(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+    [TopicStreamOwner] = emqx_broker:subscribers(Topic),
+    exit(TopicStreamOwner, kill),
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := Code, via := _PVia}} when Code == 0 orelse Code == 16 ->
+            ok
+    end,
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 ->
+            ok;
+        {ok, #{reason_code := 0, via := _PVia2}} ->
+            ok
+    end,
+
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                topic := Topic2,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        recv_pub(1)
+    ),
+    ?assertEqual(timeout, recv_pub(1)),
+    ok.
+
+t_multi_streams_unsub_via_other(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
+
+    %% Unsub topic1 via stream2 should fail with error code 17: "No subscription existed"
+    {ok, #{via := SVia2}, [17]} = emqtt:unsubscribe_via(C, SVia2, Topic),
+
+    case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia2}} -> ok
+    end,
+
+    PubRecvs2 = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<6, 7, 8, 9>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs2
+    ),
+    ok = emqtt:disconnect(C).
+
+t_multi_streams_shutdown_pub_data_stream(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia =/= SVia2),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
+    {quic, _Conn, DataStream} = PubVia,
+    quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100),
+    timer:sleep(500),
+    %% Still alive
+    ?assert(is_list(emqtt:info(C))).
+
+t_multi_streams_shutdown_sub_data_stream(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia =/= SVia2),
+    {quic, _Conn, DataStream} = SVia2,
+    quicer:shutdown_stream(DataStream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, 500, 100),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    #{data_stream_socks := [_PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
+    timer:sleep(500),
+    %% Still alive
+    ?assert(is_list(emqtt:info(C))).
+
+t_multi_streams_shutdown_ctrl_stream(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    unlink(C),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
+    quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 1000),
+    timer:sleep(500),
+    %% Client should be closed
+    ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)).
+
+t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) ->
+    erlang:process_flag(trap_exit, true),
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {reconnect, true},
+        %% speedup test
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia2 =/= SVia),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
+    quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 100),
+    timer:sleep(200),
+    %% Client should be closed
+    ?assert(is_list(emqtt:info(C))).
+
+t_multi_streams_remote_shutdown(Config) ->
+    erlang:process_flag(trap_exit, true),
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {reconnect, false},
+        %% speedup test
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia2 =/= SVia),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
+
+    ok = stop_emqx(),
+
+    timer:sleep(200),
+    start_emqx_quic(?config(port, Config)),
+
+    %% Client should be closed
+    ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)).
+
+t_multi_streams_remote_shutdown_with_reconnect(Config) ->
+    erlang:process_flag(trap_exit, true),
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {reconnect, true},
+        %% speedup test
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia2 =/= SVia),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
+
+    ok = stop_emqx(),
+
+    timer:sleep(200),
+
+    start_emqx_quic(?config(port, Config)),
+    %% Client should be closed
+    ?assert(is_list(emqtt:info(C))).
+
+t_conn_silent_close(Config) ->
+    erlang:process_flag(trap_exit, true),
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    %% quic idle timeout + 1s
+    timer:sleep(16000),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    ?assertException(
+        exit,
+        noproc,
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, 1}])
+    ).
+
+t_client_conn_bump_streams(Config) ->
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    {quic, Conn, _Stream} = proplists:get_value(socket, emqtt:info(C)),
+    ok = quicer:setopt(Conn, param_conn_settings, #{peer_unidi_stream_count => 20}).
+
+t_olp_true(Config) ->
+    meck:new(emqx_olp, [passthrough, no_history]),
+    ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end),
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    ok = meck:unload(emqx_olp).
+
+t_olp_reject(Config) ->
+    erlang:process_flag(trap_exit, true),
+    emqx_config:put_zone_conf(default, [overload_protection, enable], true),
+    meck:new(emqx_olp, [passthrough, no_history]),
+    ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end),
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+    ?assertEqual(
+        {error,
+            {transport_down, #{
+                error => 346,
+                status =>
+                    user_canceled
+            }}},
+        emqtt:quic_connect(C)
+    ),
+    ok = meck:unload(emqx_olp),
+    emqx_config:put_zone_conf(default, [overload_protection, enable], false).
+
+t_conn_resume(Config) ->
+    erlang:process_flag(trap_exit, true),
+    {ok, C0} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+
+    {ok, _} = emqtt:quic_connect(C0),
+    #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)),
+    emqtt:disconnect(C0),
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5},
+        {nst, NST}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    Cid = proplists:get_value(clientid, emqtt:info(C)),
+    ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]).
+
+t_conn_without_ctrl_stream(Config) ->
+    erlang:process_flag(trap_exit, true),
+    {ok, Conn} = quicer:connect(
+        {127, 0, 0, 1},
+        ?config(port, Config),
+        [{alpn, ["mqtt"]}, {verify, none}],
+        3000
+    ),
     receive
-        {publish, Msg} ->
-            receive_messages(Count - 1, [Msg | Msgs]);
-        _Other ->
-            receive_messages(Count, Msgs)
-    after 1000 ->
-        Msgs
+        {quic, transport_shutdown, Conn, _} -> ok
     end.
+
+t_data_stream_race_ctrl_stream(Config) ->
+    erlang:process_flag(trap_exit, true),
+    {ok, C0} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C0),
+    #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)),
+    emqtt:disconnect(C0),
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {connect_timeout, 5},
+        {nst, NST}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    Cid = proplists:get_value(clientid, emqtt:info(C)),
+    ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]).
+
+t_multi_streams_sub_0_rtt(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C0),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    ok = emqtt:open_quic_connection(C),
+    ok = emqtt:quic_mqtt_connect(C),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        #{},
+        <<"qos 2 1">>,
+        [{qos, PubQos}],
+        infinity,
+        fun(_) -> ok end
+    ),
+    {ok, _} = emqtt:quic_connect(C),
+    receive
+        {publish, #{
+            client_pid := C0,
+            payload := <<"qos 2 1">>,
+            qos := RecQos,
+            topic := Topic
+        }} ->
+            ok;
+        Other ->
+            ct:fail("unexpected recv ~p", [Other])
+    after 100 ->
+        ct:fail("not received")
+    end,
+    ok = emqtt:disconnect(C),
+    ok = emqtt:disconnect(C0).
+
+t_multi_streams_sub_0_rtt_large_payload(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Payload = binary:copy(<<"qos 2 1">>, 1600),
+    {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C0),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    ok = emqtt:open_quic_connection(C),
+    ok = emqtt:quic_mqtt_connect(C),
+    ok = emqtt:publish_async(
+        C,
+        {new_data_stream, []},
+        Topic,
+        #{},
+        Payload,
+        [{qos, PubQos}],
+        infinity,
+        fun(_) -> ok end
+    ),
+    {ok, _} = emqtt:quic_connect(C),
+    receive
+        {publish, #{
+            client_pid := C0,
+            payload := Payload,
+            qos := RecQos,
+            topic := Topic
+        }} ->
+            ok;
+        Other ->
+            ct:fail("unexpected recv ~p", [Other])
+    after 100 ->
+        ct:fail("not received")
+    end,
+    ok = emqtt:disconnect(C),
+    ok = emqtt:disconnect(C0).
+
+%% @doc verify data stream can continue after 0-RTT handshake
+t_multi_streams_sub_0_rtt_stream_data_cont(Config) ->
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Payload = binary:copy(<<"qos 2 1">>, 1600),
+    {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]),
+    {ok, _} = emqtt:quic_connect(C0),
+    {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
+    ok = emqtt:open_quic_connection(C),
+    ok = emqtt:quic_mqtt_connect(C),
+    {ok, PubVia} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        #{},
+        Payload,
+        [{qos, PubQos}],
+        infinity,
+        fun(_) -> ok end
+    ),
+    {ok, _} = emqtt:quic_connect(C),
+    receive
+        {publish, #{
+            client_pid := C0,
+            payload := Payload,
+            qos := RecQos,
+            topic := Topic
+        }} ->
+            ok;
+        Other ->
+            ct:fail("unexpected recv ~p", [Other])
+    after 100 ->
+        ct:fail("not received")
+    end,
+    Payload2 = <<"2nd part", Payload/binary>>,
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        #{},
+        Payload2,
+        [{qos, PubQos}],
+        infinity,
+        fun(_) -> ok end
+    ),
+    receive
+        {publish, #{
+            client_pid := C0,
+            payload := Payload2,
+            qos := RecQos,
+            topic := Topic
+        }} ->
+            ok;
+        Other2 ->
+            ct:fail("unexpected recv ~p", [Other2])
+    after 100 ->
+        ct:fail("not received")
+    end,
+    ok = emqtt:disconnect(C),
+    ok = emqtt:disconnect(C0).
+
+%%--------------------------------------------------------------------
+%% Helper functions
+%%--------------------------------------------------------------------
+send_and_recv_with(Sock) ->
+    {ok, {IP, _}} = emqtt_quic:sockname(Sock),
+    ?assert(lists:member(tuple_size(IP), [4, 8])),
+    ok = emqtt_quic:send(Sock, <<"ping">>),
+    emqtt_quic:setopts(Sock, [{active, false}]),
+    {ok, <<"pong">>} = emqtt_quic:recv(Sock, 0),
+    ok = emqtt_quic:setopts(Sock, [{active, 100}]),
+    {ok, Stats} = emqtt_quic:getstat(Sock, [send_cnt, recv_cnt]),
+    %% connection level counters, not stream level
+    [{send_cnt, _}, {recv_cnt, _}] = Stats.
+
+certfile(Config) ->
+    filename:join([test_dir(Config), "certs", "test.crt"]).
+
+keyfile(Config) ->
+    filename:join([test_dir(Config), "certs", "test.key"]).
+
+test_dir(Config) ->
+    filename:dirname(filename:dirname(proplists:get_value(data_dir, Config))).
+
+recv_pub(Count) ->
+    recv_pub(Count, []).
+
+recv_pub(0, Acc) ->
+    lists:reverse(Acc);
+recv_pub(Count, Acc) ->
+    receive
+        {publish, _Prop} = Pub ->
+            recv_pub(Count - 1, [Pub | Acc])
+    after 100 ->
+        timeout
+    end.
+
+all_tc() ->
+    code:add_patha(filename:join(code:lib_dir(emqx), "ebin/")),
+    emqx_common_test_helpers:all(?MODULE).
+
+-spec calc_qos(0 | 1 | 2, 0 | 1 | 2) -> 0 | 1 | 2.
+calc_qos(PubQos, SubQos) ->
+    if
+        PubQos > SubQos ->
+            SubQos;
+        SubQos > PubQos ->
+            PubQos;
+        true ->
+            PubQos
+    end.
+-spec calc_pkt_id(0 | 1 | 2, non_neg_integer()) -> undefined | non_neg_integer().
+calc_pkt_id(0, _Id) ->
+    undefined;
+calc_pkt_id(1, Id) ->
+    Id;
+calc_pkt_id(2, Id) ->
+    Id.
+
+-spec start_emqx_quic(inet:port_number()) -> ok.
+start_emqx_quic(UdpPort) ->
+    emqx_common_test_helpers:start_apps([]),
+    application:ensure_all_started(quicer),
+    emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort).
+
+-spec stop_emqx() -> ok.
+stop_emqx() ->
+    emqx_common_test_helpers:stop_apps([]).
+
+%% select a random port picked by OS
+-spec select_port() -> inet:port_number().
+select_port() ->
+    {ok, S} = gen_udp:open(0, [{reuseaddr, true}]),
+    {ok, {_, Port}} = inet:sockname(S),
+    gen_udp:close(S),
+    case os:type() of
+        {unix, darwin} ->
+            %% in MacOS, still get address_in_use after close port
+            timer:sleep(500);
+        _ ->
+            skip
+    end,
+    ct:pal("select port: ~p", [Port]),
+    Port.

From 2a6cdd9da6b1daa5242d07b3cad363cc4c68ef75 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 10 Jan 2023 16:34:25 +0100
Subject: [PATCH 073/144] test(quic): enhance large payload test

---
 apps/emqx/src/emqx_channel.erl                |   1 -
 apps/emqx/src/emqx_connection.erl             |   3 +-
 .../test/emqx_quic_multistreams_SUITE.erl     | 107 ++++++++++++++++--
 3 files changed, 98 insertions(+), 13 deletions(-)

diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index a12df9c64..e82adc786 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -1136,7 +1136,6 @@ do_deliver(Publishes, Channel) when is_list(Publishes) ->
     {Packets, NChannel} =
         lists:foldl(
             fun(Publish, {Acc, Chann}) ->
-                %% @FIXME perf:  list append with copy left list
                 {Packets, NChann} = do_deliver(Publish, Chann),
                 {Packets ++ Acc, NChann}
             end,
diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index be420d65e..ff3ee81a9 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -118,7 +118,7 @@
     %% limiter timers
     limiter_timer :: undefined | reference(),
 
-    %% QUIC conn pid if is a pid
+    %% QUIC conn owner pid if in use.
     quic_conn_pid :: maybe(pid())
 }).
 
@@ -336,7 +336,6 @@ init_state(
     Limiter = emqx_limiter_container:get_limiter_by_types(Listener, LimiterTypes, LimiterCfg),
 
     FrameOpts = #{
-        %% @TODO:q what is strict_mode?
         strict_mode => emqx_config:get_zone_conf(Zone, [mqtt, strict_mode]),
         max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size])
     },
diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index b6d3c661c..025790ef7 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -661,14 +661,14 @@ t_multi_streams_packet_too_large(Config) ->
     PubQos = ?config(pub_qos, Config),
     SubQos = ?config(sub_qos, Config),
     Topic = atom_to_binary(?FUNCTION_NAME),
-    meck:new(emqx_frame, [passthrough, no_history]),
-    ok = meck:expect(
-        emqx_frame,
-        serialize_opts,
-        fun(#mqtt_packet_connect{proto_ver = ProtoVer}) ->
-            #{version => ProtoVer, max_size => 1024}
-        end
-    ),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+    PktId2 = calc_pkt_id(RecQos, 2),
+    PktId3 = calc_pkt_id(RecQos, 3),
+
+    OldMax = emqx_config:get_zone_conf(default, [mqtt, max_packet_size]),
+    emqx_config:put_zone_conf(default, [mqtt, max_packet_size], 1000),
+
     {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]),
     {ok, _} = emqtt:quic_connect(C),
     {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]),
@@ -678,13 +678,95 @@ t_multi_streams_packet_too_large(Config) ->
         C,
         PubVia,
         Topic,
-        binary:copy(<<"stream data 1">>, 1024),
+        <<"stream data 1">>,
         [{qos, PubQos}],
         undefined
     ),
+
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        <<"stream data 2">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+
+    PubRecvs = recv_pub(2),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<"stream data 1">>,
+                qos := RecQos,
+                topic := Topic
+            }},
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId2,
+                payload := <<"stream data 2">>,
+                qos := RecQos,
+                topic := Topic
+            }}
+        ],
+        PubRecvs
+    ),
+
+    {ok, PubVia2} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia2,
+        Topic,
+        binary:copy(<<"too large">>, 200),
+        [{qos, PubQos}],
+        undefined
+    ),
+    timer:sleep(200),
+    ?assert(is_list(emqtt:info(C))),
+
+    timeout = recv_pub(1),
+
+    %% send large payload on stream 1
+    ok = emqtt:publish_async(
+        C,
+        PubVia,
+        Topic,
+        binary:copy(<<"too large">>, 200),
+        [{qos, PubQos}],
+        undefined
+    ),
+    timer:sleep(200),
     timeout = recv_pub(1),
     ?assert(is_list(emqtt:info(C))),
-    ok = meck:unload(emqx_frame),
+
+    %% Connection could be kept
+    {error, stm_send_error, _} = quicer:send(via_stream(PubVia), <<1>>),
+    {error, stm_send_error, _} = quicer:send(via_stream(PubVia2), <<1>>),
+    %% We could send data over new stream
+    {ok, PubVia3} = emqtt:start_data_stream(C, []),
+    ok = emqtt:publish_async(
+        C,
+        PubVia3,
+        Topic,
+        <<"stream data 3">>,
+        [{qos, PubQos}],
+        undefined
+    ),
+    [
+        {publish, #{
+            client_pid := C,
+            packet_id := PktId3,
+            payload := <<"stream data 3">>,
+            qos := RecQos,
+            topic := Topic
+        }}
+    ] = recv_pub(1),
+    timer:sleep(200),
+
+    ?assert(is_list(emqtt:info(C))),
+
+    emqx_config:put_zone_conf(default, [mqtt, max_packet_size], OldMax),
     ok = emqtt:disconnect(C).
 
 t_conn_change_client_addr(Config) ->
@@ -1758,3 +1840,8 @@ select_port() ->
     end,
     ct:pal("select port: ~p", [Port]),
     Port.
+
+-spec via_stream({quic, quicer:connection_handle(), quicer:stream_handle()}) ->
+    quicer:stream_handle().
+via_stream({quic, _Conn, Stream}) ->
+    Stream.

From 1692a16778731711db58ac17eea2a400f810e6d6 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 11 Jan 2023 16:24:06 +0100
Subject: [PATCH 074/144] feat(quic): handle ctrl stream normal shutdown

---
 apps/emqx/include/emqx_quic.hrl               |   1 +
 apps/emqx/src/emqx_connection.erl             |   3 +-
 apps/emqx/src/emqx_quic_connection.erl        |  27 +++-
 apps/emqx/src/emqx_quic_stream.erl            |  14 ++-
 .../test/emqx_quic_multistreams_SUITE.erl     | 118 +++++++++++++++++-
 5 files changed, 152 insertions(+), 11 deletions(-)

diff --git a/apps/emqx/include/emqx_quic.hrl b/apps/emqx/include/emqx_quic.hrl
index 3366b8938..a16784d5d 100644
--- a/apps/emqx/include/emqx_quic.hrl
+++ b/apps/emqx/include/emqx_quic.hrl
@@ -19,6 +19,7 @@
 
 %% MQTT Over QUIC Shutdown Error code.
 -define(MQTT_QUIC_CONN_NOERROR, 0).
+-define(MQTT_QUIC_CONN_ERROR_CTRL_STREAM_DOWN, 1).
 -define(MQTT_QUIC_CONN_ERROR_OVERLOADED, 2).
 
 -endif.
diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index ff3ee81a9..2916f37bb 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -921,7 +921,8 @@ handle_info({sock_error, Reason}, State) ->
         false -> ok
     end,
     handle_info({sock_closed, Reason}, close_socket(State));
-handle_info({quic, Event, Handle, Prop}, State) ->
+%% handle QUIC control stream events
+handle_info({quic, Event, Handle, Prop}, State) when is_atom(Event) ->
     emqx_quic_stream:Event(Handle, Prop, State);
 handle_info(Info, State) ->
     with_channel(handle_info, [Info], State).
diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index 69d16cbc3..7538307e8 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -179,7 +179,13 @@ new_stream(
         SOpts1,
         Props
     ),
-    quicer:handoff_stream(Stream, NewStreamOwner, {PS, Serialize, Channel}),
+    case quicer:handoff_stream(Stream, NewStreamOwner, {PS, Serialize, Channel}) of
+        ok ->
+            ok;
+        E ->
+            %% Only log, keep connecion alive.
+            ?SLOG(error, #{message => "new stream handoff failed", stream => Stream, error => E})
+    end,
     %% @TODO maybe keep them in `inactive_streams'
     {ok, S#{streams := [{NewStreamOwner, Stream} | Streams]}}.
 
@@ -200,7 +206,7 @@ transport_shutdown(_C, DownInfo, S) when is_map(DownInfo) ->
 %% @doc callback for handling for peer addr changed.
 -spec peer_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state) -> cb_ret().
 peer_address_changed(_C, _NewAddr, S) ->
-    %% @TODO update session info?
+    %% @TODO update conn info in emqx_quic_stream
     {ok, S}.
 
 %% @doc callback for handling local addr change, currently unused
@@ -224,7 +230,7 @@ streams_available(_C, {BidirCnt, UnidirCnt}, S) ->
 %% @doc callback for handling request when remote wants for more streams
 %%      should cope with rate limiting
 %% @TODO this is not going to get triggered in current version
-%% for https://github.com/microsoft/msquic/issues/3120
+%% ref: https://github.com/microsoft/msquic/issues/3120
 -spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret().
 peer_needs_streams(_C, undefined, S) ->
     ?SLOG(info, #{
@@ -240,6 +246,10 @@ handle_call(
     #{streams := Streams} = S
 ) ->
     [
+        %% Try to activate streams individually if failed, stream will shutdown on its own.
+        %% we dont care about the return val here.
+        %% note, this is only used after control stream pass the validation. The data streams
+        %%       that are called here are assured to be inactived (data processing hasn't been started).
         catch emqx_quic_data_stream:activate_data(OwnerPid, ActivateData)
      || {OwnerPid, _Stream} <- Streams
     ],
@@ -255,10 +265,15 @@ handle_call(_Req, _From, S) ->
 handle_info({'EXIT', Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) ->
     case Reason of
         normal ->
-            quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0);
+            quicer:async_shutdown_connection(
+                Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, ?MQTT_QUIC_CONN_NOERROR
+            );
         _ ->
-            %% @TODO have some reasons mappings here.
-            quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 1)
+            quicer:async_shutdown_connection(
+                Conn,
+                ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE,
+                ?MQTT_QUIC_CONN_ERROR_CTRL_STREAM_DOWN
+            )
     end,
     {ok, S};
 handle_info({'EXIT', Pid, Reason}, #{streams := Streams} = S) ->
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index a8ef7d41d..d1b205cf0 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% MQTT/QUIC Stream
+%% MQTT/QUIC control Stream
 -module(emqx_quic_stream).
 
 -ifndef(BUILD_WITHOUT_QUIC).
@@ -38,6 +38,7 @@
     peercert/1
 ]).
 -include_lib("quicer/include/quicer.hrl").
+-include_lib("emqx/include/emqx_quic.hrl").
 
 -type cb_ret() :: quicer_stream:cb_ret().
 -type cb_data() :: quicer_stream:cb_state().
@@ -223,10 +224,17 @@ stream_closed(
         is_atom(Status) andalso
         is_integer(Code)
 ->
-    %% @TODO for now we fake a sock_closed for
+    %% For now we fake a sock_closed for
     %% emqx_connection:process_msg to append
     %% a msg to be processed
-    {ok, {sock_closed, Status}, S}.
+    Reason =
+        case Code of
+            ?MQTT_QUIC_CONN_NOERROR ->
+                normal;
+            _ ->
+                Status
+        end,
+    {ok, {sock_closed, Reason}, S}.
 
 %%%
 %%%  Internals
diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index 025790ef7..17f4cbbc2 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -118,6 +118,8 @@ groups() ->
             t_multi_streams_shutdown_ctrl_stream,
             t_multi_streams_shutdown_ctrl_stream_then_reconnect,
             t_multi_streams_remote_shutdown,
+            t_multi_streams_emqx_ctrl_kill,
+            t_multi_streams_emqx_ctrl_exit_normal,
             t_multi_streams_remote_shutdown_with_reconnect
         ]},
 
@@ -1327,7 +1329,13 @@ t_multi_streams_shutdown_ctrl_stream(Config) ->
     ),
 
     {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
-    quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 1000),
+    Flag = ?config(stream_shutdown_flag, Config),
+    AppErrorCode =
+        case Flag of
+            ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL -> 0;
+            _ -> 500
+        end,
+    quicer:shutdown_stream(Ctrlstream, Flag, AppErrorCode, 1000),
     timer:sleep(500),
     %% Client should be closed
     ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)).
@@ -1384,6 +1392,114 @@ t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) ->
     %% Client should be closed
     ?assert(is_list(emqtt:info(C))).
 
+t_multi_streams_emqx_ctrl_kill(Config) ->
+    erlang:process_flag(trap_exit, true),
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {reconnect, false},
+        %% speedup test
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia2 =/= SVia),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    ClientId = proplists:get_value(clientid, emqtt:info(C)),
+    [{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId),
+    exit(TransPid, kill),
+
+    timer:sleep(200),
+    %% Client should be closed
+    ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)).
+
+t_multi_streams_emqx_ctrl_exit_normal(Config) ->
+    erlang:process_flag(trap_exit, true),
+    PubQos = ?config(pub_qos, Config),
+    SubQos = ?config(sub_qos, Config),
+    RecQos = calc_qos(PubQos, SubQos),
+    PktId1 = calc_pkt_id(RecQos, 1),
+
+    Topic = atom_to_binary(?FUNCTION_NAME),
+    Topic2 = <<Topic/binary, "two">>,
+    {ok, C} = emqtt:start_link([
+        {proto_ver, v5},
+        {reconnect, false},
+        %% speedup test
+        {connect_timeout, 5}
+        | Config
+    ]),
+    {ok, _} = emqtt:quic_connect(C),
+    {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic, [{qos, SubQos}]}
+    ]),
+    {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [
+        {Topic2, [{qos, SubQos}]}
+    ]),
+
+    ?assert(SVia2 =/= SVia),
+
+    case
+        emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}])
+    of
+        ok when PubQos == 0 -> ok;
+        {ok, #{reason_code := 0, via := _PVia}} -> ok
+    end,
+
+    PubRecvs = recv_pub(1),
+    ?assertMatch(
+        [
+            {publish, #{
+                client_pid := C,
+                packet_id := PktId1,
+                payload := <<1, 2, 3, 4, 5>>,
+                qos := RecQos
+            }}
+        ],
+        PubRecvs
+    ),
+
+    ClientId = proplists:get_value(clientid, emqtt:info(C)),
+    [{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId),
+
+    emqx_connection:stop(TransPid),
+    timer:sleep(200),
+    %% Client exit normal.
+    ?assertMatch({'EXIT', {normal, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)).
+
 t_multi_streams_remote_shutdown(Config) ->
     erlang:process_flag(trap_exit, true),
     PubQos = ?config(pub_qos, Config),

From 98a72d40ce7232735e11bb178f128722bf4085a3 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 11 Jan 2023 16:24:37 +0100
Subject: [PATCH 075/144] fix(emqx_connection): do not raise an exception for
 normal shutdown

---
 apps/emqx/src/emqx_connection.erl | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 2916f37bb..88c7d28e2 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -680,6 +680,12 @@ maybe_raise_exception(#{
     stacktrace := Stacktrace
 }) ->
     erlang:raise(Exception, Context, Stacktrace);
+maybe_raise_exception({shutdown, normal}) ->
+    ok;
+maybe_raise_exception(normal) ->
+    ok;
+maybe_raise_exception(shutdown) ->
+    ok;
 maybe_raise_exception(Reason) ->
     exit(Reason).
 

From de810e04fd1bd0d13681a7a4da06f183a35986dd Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 11 Jan 2023 16:53:03 +0100
Subject: [PATCH 076/144] chore(quic): clean test code

---
 apps/emqx/src/emqx_quic_connection.erl          |  2 +-
 apps/emqx/src/emqx_quic_stream.erl              |  8 +++++---
 apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl  |  1 -
 apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 10 ----------
 4 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index 7538307e8..ae195cd6b 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -23,7 +23,7 @@
 -include_lib("quicer/include/quicer.hrl").
 -include_lib("emqx/include/emqx_quic.hrl").
 
--behavior(quicer_connection).
+-behaviour(quicer_connection).
 
 -export([
     init/1,
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index d1b205cf0..5f7f93866 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -14,7 +14,10 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% MQTT/QUIC control Stream
+%% MQTT over QUIC
+%% multistreams: This is the control stream.
+%% single stream: This is the only main stream.
+%%   callbacks are from emqx_connection process rather than quicer_stream
 -module(emqx_quic_stream).
 
 -ifndef(BUILD_WITHOUT_QUIC).
@@ -66,10 +69,9 @@
     _ => _
 }.
 
-%% for accepting
+%%% For Accepting New Remote Stream
 -spec wait({pid(), connection_handle(), socket_info()}) ->
     {ok, socket()} | {error, enotconn}.
-%%% For Accepting New Remote Stream
 wait({ConnOwner, Conn, ConnInfo}) ->
     {ok, Conn} = quicer:async_accept_stream(Conn, []),
     ConnOwner ! {self(), stream_acceptor_ready},
diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
index 0199bbc10..d3de74f72 100644
--- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
+++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl
@@ -65,7 +65,6 @@ init_per_group(quic, Config) ->
     UdpPort = 1884,
     emqx_common_test_helpers:start_apps([]),
     emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort),
-    emqx_logger:set_log_level(debug),
     [{port, UdpPort}, {conn_fun, quic_connect} | Config];
 init_per_group(_, Config) ->
     emqx_common_test_helpers:stop_apps([]),
diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index 17f4cbbc2..593613fcc 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -141,16 +141,6 @@ init_per_suite(Config) ->
     emqx_common_test_helpers:start_apps([]),
     UdpPort = 14567,
     start_emqx_quic(UdpPort),
-    %% dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}),
-    %% dbg:p(all, c),
-    %% dbg:tpl(quicer_stream, handle_info, c),
-    %% dbg:tp(emqx_quic_connection, cx),
-    %% dbg:tp(emqx_quic_stream, cx),
-    %% dbg:tp(emqtt, cx),
-    %% dbg:tpl(emqtt_quic_stream, cx),
-    %% dbg:tpl(emqx_quic_stream, cx),
-    %% dbg:tpl(emqx_quic_data_stream, cx),
-    %% dbg:tpl(emqtt, cx),
     [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config].
 
 end_per_suite(_) ->

From 88cdfcc4a6b7d8e19ebb99ef13454a55f6554678 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 11 Jan 2023 21:04:34 +0100
Subject: [PATCH 077/144] test(quic):  excl. multistream SUITE when
 BUILD_WITHOUT_QUIC

---
 apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index 593613fcc..1cafdccd8 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -15,6 +15,8 @@
 %%--------------------------------------------------------------------
 -module(emqx_quic_multistreams_SUITE).
 
+-ifndef(BUILD_WITHOUT_QUIC).
+
 -compile(export_all).
 -compile(nowarn_export_all).
 
@@ -1951,3 +1953,7 @@ select_port() ->
     quicer:stream_handle().
 via_stream({quic, _Conn, Stream}) ->
     Stream.
+
+%% BUILD_WITHOUT_QUIC
+-else.
+-endif.

From 9e9ae50ab90a77b829875fc6b7a2cb56234be2cf Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 11 Jan 2023 22:14:43 +0100
Subject: [PATCH 078/144] chore: qzhuyan/emqtt vsn 534541b

---
 mix.exs      | 2 +-
 rebar.config | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mix.exs b/mix.exs
index bf300761f..715d62226 100644
--- a/mix.exs
+++ b/mix.exs
@@ -60,7 +60,7 @@ defmodule EMQXUmbrella.MixProject do
       {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true},
       {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
-      {:emqtt, github: "emqx/emqtt", tag: "1.7.0", override: true},
+      {:emqtt, github: "qzhuyan/emqtt", tag: "1.7.1-pre", override: true},
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
       {:observer_cli, "1.7.1"},
       {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
diff --git a/rebar.config b/rebar.config
index 76402897b..917d11ab7 100644
--- a/rebar.config
+++ b/rebar.config
@@ -62,7 +62,7 @@
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}}
     , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
-    , {emqtt, {git, "https://github.com/qzhuyan/emqtt", {branch, "dev/william/multi-streams"}}} %% @TODO revert
+    , {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre"}}} %% @TODO revert
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
     , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}

From 282d1a6829adaed7bbb664e3502b328d058787a7 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 12 Jan 2023 14:58:45 +0100
Subject: [PATCH 079/144] ci: build dialyzer PLT with quicer, jq and bcrypt

---
 apps/emqx/rebar.config.script | 15 ++++++++++++++-
 rebar.config.erl              | 25 ++++++++++++++-----------
 2 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index e942e1a5c..e1afbf61c 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -26,6 +26,19 @@ end,
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
 Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.104"}}}.
 
+Dialyzer = fun(Config) ->
+                   {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
+                   {plt_extra_apps, OldExtra} = lists:keyfind(plt_extra_apps, 1, OldDialyzerConfig),
+                   Extra = OldExtra ++ [quicer || IsQuicSupp()],
+                   NewDialyzerConfig = [{plt_extra_apps, Extra} | OldDialyzerConfig],
+                   lists:keystore(
+                     dialyzer,
+                     1,
+                     Config,
+                     {dialyzer, NewDialyzerConfig}
+                    )
+           end.
+
 ExtraDeps = fun(C) ->
     {deps, Deps0} = lists:keyfind(deps, 1, C),
     {erl_opts, ErlOpts0} = lists:keyfind(erl_opts, 1, C),
@@ -43,4 +56,4 @@ ExtraDeps = fun(C) ->
     )
 end,
 
-ExtraDeps(CONFIG).
+Dialyzer(ExtraDeps(CONFIG)).
diff --git a/rebar.config.erl b/rebar.config.erl
index e99f83683..6f4371c7b 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -548,17 +548,20 @@ dialyzer(Config) ->
 
     AppsToExclude = AppNames -- KnownApps,
 
-    case length(AppsToAnalyse) > 0 of
-        true ->
-            lists:keystore(
-                dialyzer,
-                1,
-                Config,
-                {dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]}
-            );
-        false ->
-            Config
-    end.
+    Extra =
+        [bcrypt || provide_bcrypt_dep()] ++
+            [jq || is_jq_supported()] ++
+            [quicer || is_quicer_supported()],
+    NewDialyzerConfig =
+        OldDialyzerConfig ++
+            [{exclude_apps, AppsToExclude} || length(AppsToAnalyse) > 0] ++
+            [{plt_extra_apps, Extra} || length(Extra) > 0],
+    lists:keystore(
+        dialyzer,
+        1,
+        Config,
+        {dialyzer, NewDialyzerConfig}
+    ).
 
 coveralls() ->
     case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of

From 381eb8ec682ae9740398333aae0069a5ef1d8840 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 13 Jan 2023 10:02:21 +0100
Subject: [PATCH 080/144] chore(quic): fix dialyzer

---
 apps/emqx/src/emqx_connection.erl       |  7 ++++++-
 apps/emqx/src/emqx_listeners.erl        |  7 +++++--
 apps/emqx/src/emqx_quic_connection.erl  | 28 +++++++++++--------------
 apps/emqx/src/emqx_quic_data_stream.erl |  8 +++----
 apps/emqx/src/emqx_quic_stream.erl      | 13 ++++++------
 5 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 88c7d28e2..9e0099414 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -929,7 +929,12 @@ handle_info({sock_error, Reason}, State) ->
     handle_info({sock_closed, Reason}, close_socket(State));
 %% handle QUIC control stream events
 handle_info({quic, Event, Handle, Prop}, State) when is_atom(Event) ->
-    emqx_quic_stream:Event(Handle, Prop, State);
+    case emqx_quic_stream:Event(Handle, Prop, State) of
+        {{continue, Msgs}, NewState} ->
+            {ok, Msgs, NewState};
+        Other ->
+            Other
+    end;
 handle_info(Info, State) ->
     with_channel(handle_info, [Info], State).
 
diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index 45f3b2cfd..ccf6a667a 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -386,13 +386,16 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) ->
                 listener => {quic, ListenerName},
                 limiter => limiter(Opts)
             },
-            StreamOpts = [{stream_callback, emqx_quic_stream}],
+            StreamOpts = #{
+                stream_callback => emqx_quic_stream,
+                active => 1
+            },
             Id = listener_id(quic, ListenerName),
             add_limiter_bucket(Id, Opts),
             quicer:start_listener(
                 Id,
                 ListenOn,
-                {ListenOpts, ConnectionOpts, StreamOpts}
+                {maps:from_list(ListenOpts), ConnectionOpts, StreamOpts}
             );
         [] ->
             {ok, {skipped, quic_app_missing}}
diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index ae195cd6b..39d6a2c2f 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -93,7 +93,7 @@ closed(_Conn, #{is_peer_acked := _} = Prop, S) ->
 
 %% @doc handle the new incoming connecion as the connecion acceptor.
 -spec new_conn(quicer:connection_handle(), quicer:new_conn_props(), cb_state()) ->
-    {ok, cb_state()} | {error, any()}.
+    {ok, cb_state()} | {error, any(), cb_state()}.
 new_conn(
     Conn,
     #{version := _Vsn} = ConnInfo,
@@ -119,7 +119,7 @@ new_conn(
             end;
         true ->
             emqx_metrics:inc('olp.new_conn'),
-            quicer:async_shutdown_connection(
+            _ = quicer:async_shutdown_connection(
                 Conn,
                 ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE,
                 ?MQTT_QUIC_CONN_ERROR_OVERLOADED
@@ -129,7 +129,7 @@ new_conn(
 
 %% @doc callback when connection is connected.
 -spec connected(quicer:connection_handle(), quicer:connected_props(), cb_state()) ->
-    {ok, cb_state()} | {error, any()}.
+    {ok, cb_state()} | {error, any(), cb_state()}.
 connected(_Conn, Props, S) ->
     ?SLOG(debug, Props),
     {ok, S}.
@@ -193,7 +193,7 @@ new_stream(
 -spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret().
 shutdown(Conn, ErrorCode, S) ->
     ErrorCode =/= 0 andalso ?SLOG(debug, #{error_code => ErrorCode, state => S}),
-    quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
+    _ = quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
     {ok, S}.
 
 %% @doc callback for handling transport error, such as idle timeout
@@ -245,7 +245,7 @@ handle_call(
     _From,
     #{streams := Streams} = S
 ) ->
-    [
+    _ = [
         %% Try to activate streams individually if failed, stream will shutdown on its own.
         %% we dont care about the return val here.
         %% note, this is only used after control stream pass the validation. The data streams
@@ -263,18 +263,14 @@ handle_call(_Req, _From, S) ->
 
 %% @doc handle DOWN messages from streams.
 handle_info({'EXIT', Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) ->
-    case Reason of
-        normal ->
-            quicer:async_shutdown_connection(
-                Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, ?MQTT_QUIC_CONN_NOERROR
-            );
-        _ ->
-            quicer:async_shutdown_connection(
-                Conn,
-                ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE,
+    Code =
+        case Reason of
+            normal ->
+                ?MQTT_QUIC_CONN_NOERROR;
+            _ ->
                 ?MQTT_QUIC_CONN_ERROR_CTRL_STREAM_DOWN
-            )
-    end,
+        end,
+    _ = quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, Code),
     {ok, S};
 handle_info({'EXIT', Pid, Reason}, #{streams := Streams} = S) ->
     case proplists:is_defined(Pid, Streams) of
diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl
index e3f6b7adc..2e90edcfb 100644
--- a/apps/emqx/src/emqx_quic_data_stream.erl
+++ b/apps/emqx/src/emqx_quic_data_stream.erl
@@ -98,19 +98,19 @@ post_handoff(_Stream, {undefined = _PS, undefined = _Serialize, undefined = _Cha
     {ok, S};
 post_handoff(Stream, {PS, Serialize, Channel}, S) ->
     ?tp(debug, ?FUNCTION_NAME, #{channel => Channel, serialize => Serialize}),
-    quicer:setopt(Stream, active, 10),
+    _ = quicer:setopt(Stream, active, 10),
     {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}.
 
 -spec peer_receive_aborted(stream_handle(), error_code(), cb_state()) -> cb_ret().
 peer_receive_aborted(Stream, ErrorCode, #{is_unidir := _} = S) ->
     %% we abort send with same reason
-    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
+    _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
     {ok, S}.
 
 -spec peer_send_aborted(stream_handle(), error_code(), cb_state()) -> cb_ret().
 peer_send_aborted(Stream, ErrorCode, #{is_unidir := _} = S) ->
     %% we abort receive with same reason
-    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode),
+    _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode),
     {ok, S}.
 
 -spec peer_send_shutdown(stream_handle(), undefined, cb_state()) -> cb_ret().
@@ -157,7 +157,7 @@ handle_stream_data(
 
 -spec passive(stream_handle(), undefined, cb_state()) -> cb_ret().
 passive(Stream, undefined, S) ->
-    quicer:setopt(Stream, active, 10),
+    _ = quicer:setopt(Stream, active, 10),
     {ok, S}.
 
 -spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_state()) -> cb_ret().
diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl
index 5f7f93866..f60345fe9 100644
--- a/apps/emqx/src/emqx_quic_stream.erl
+++ b/apps/emqx/src/emqx_quic_stream.erl
@@ -136,11 +136,11 @@ getopts(_Socket, _Opts) ->
 %% @TODO supply some App Error Code from caller
 fast_close({ConnOwner, Conn, _ConnInfo}) when is_pid(ConnOwner) ->
     %% handshake aborted.
-    quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
+    _ = quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0),
     ok;
 fast_close({quic, _Conn, Stream, _Info}) ->
     %% Force flush
-    quicer:async_shutdown_stream(Stream),
+    _ = quicer:async_shutdown_stream(Stream),
     %% @FIXME Since we shutdown the control stream, we shutdown the connection as well
     %% *BUT* Msquic does not flush the send buffer if we shutdown the connection after
     %% gracefully shutdown the stream.
@@ -173,13 +173,13 @@ async_send({quic, _Conn, Stream, _Info}, Data, _Options) ->
 
 -spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret().
 peer_receive_aborted(Stream, ErrorCode, S) ->
-    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
+    _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
     {ok, S}.
 
 -spec peer_send_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret().
 peer_send_aborted(Stream, ErrorCode, S) ->
     %% we abort receive with same reason
-    quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
+    _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode),
     {ok, S}.
 
 -spec peer_send_shutdown(stream_handle(), undefined, cb_data()) -> cb_ret().
@@ -206,7 +206,8 @@ passive(Stream, undefined, S) ->
     end,
     {ok, S}.
 
--spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) -> cb_ret().
+-spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) ->
+    {{continue, term()}, cb_data()}.
 stream_closed(
     _Stream,
     #{
@@ -236,7 +237,7 @@ stream_closed(
             _ ->
                 Status
         end,
-    {ok, {sock_closed, Reason}, S}.
+    {{continue, {sock_closed, Reason}}, S}.
 
 %%%
 %%%  Internals

From 38247a9d62c6a5cdabbb9889c876db4e0751424d Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 13 Jan 2023 10:03:29 +0100
Subject: [PATCH 081/144] feat(quic): bump quicer to 0.0.106

---
 apps/emqx/rebar.config.script | 2 +-
 mix.exs                       | 2 +-
 rebar.config.erl              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index e1afbf61c..37ca3c849 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.104"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.106"}}}.
 
 Dialyzer = fun(Config) ->
                    {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
diff --git a/mix.exs b/mix.exs
index 715d62226..9fb715d41 100644
--- a/mix.exs
+++ b/mix.exs
@@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
-      do: [{:quicer, github: "emqx/quic", tag: "0.0.104", override: true}],
+      do: [{:quicer, github: "emqx/quic", tag: "0.0.106", override: true}],
       else: []
   end
 
diff --git a/rebar.config.erl b/rebar.config.erl
index 6f4371c7b..4d89e9f73 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,7 +39,7 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.104"}}}.
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.106"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From d8fa65ea09ea840c7f53f5bb087b923236745df6 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 13 Jan 2023 14:26:28 +0100
Subject: [PATCH 082/144] fix(quic): handle timeout event in data stream

---
 apps/emqx/src/emqx_quic_connection.erl  | 3 +++
 apps/emqx/src/emqx_quic_data_stream.erl | 6 +++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl
index 39d6a2c2f..a77ec28f2 100644
--- a/apps/emqx/src/emqx_quic_connection.erl
+++ b/apps/emqx/src/emqx_quic_connection.erl
@@ -280,6 +280,9 @@ handle_info({'EXIT', Pid, Reason}, #{streams := Streams} = S) ->
                 Reason =:= killed
         ->
             {ok, S};
+        true ->
+            ?SLOG(info, #{message => "Data stream unexpected exit", reason => Reason}),
+            {ok, S};
         false ->
             {stop, unknown_pid_down, S}
     end.
diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl
index 2e90edcfb..0b89870a8 100644
--- a/apps/emqx/src/emqx_quic_data_stream.erl
+++ b/apps/emqx/src/emqx_quic_data_stream.erl
@@ -233,7 +233,11 @@ do_handle_appl_msg({event, updated}, S) ->
 
 handle_info(Deliver = {deliver, _, _}, S) ->
     Delivers = [Deliver],
-    with_channel(handle_deliver, [Delivers], S).
+    with_channel(handle_deliver, [Delivers], S);
+handle_info({timeout, Ref, Msg}, S) ->
+    with_channel(handle_timeout, [Ref, Msg], S);
+handle_info(Info, State) ->
+    with_channel(handle_info, [Info], State).
 
 with_channel(Fun, Args, #{channel := Channel, task_queue := Q} = S) when
     Channel =/= undefined

From f8fd201a8c861332bad276d326fc705d2136c0b3 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 18 Jan 2023 09:54:18 +0100
Subject: [PATCH 083/144] test(quic): fix flaky test

---
 apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index 1cafdccd8..1ae6df201 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -1541,10 +1541,8 @@ t_multi_streams_remote_shutdown(Config) ->
     {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)),
 
     ok = stop_emqx(),
-
-    timer:sleep(200),
     start_emqx_quic(?config(port, Config)),
-
+    timer:sleep(200),
     %% Client should be closed
     ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)).
 

From dc2679049585e56aa9f5e14527363ead597841da Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 18 Jan 2023 14:02:00 +0100
Subject: [PATCH 084/144] test(quic): trace why we get verify_peer

---
 apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index 1ae6df201..dd71b6079 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -143,6 +143,11 @@ init_per_suite(Config) ->
     emqx_common_test_helpers:start_apps([]),
     UdpPort = 14567,
     start_emqx_quic(UdpPort),
+    dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}),
+    dbg:p(all, c),
+    dbg:tpl(quicer, connect, cx),
+    %% dbg:tpl(emqx_stream, cx),
+    %% dbg:tpl(emqx_quic_data_stream, cx),
     [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config].
 
 end_per_suite(_) ->

From db544cf9ad74b3cb30e217cf1d38e55d5222390a Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 18 Jan 2023 14:50:57 +0100
Subject: [PATCH 085/144] fix: emqtt vsn in rebar after rebase

---
 apps/emqx/rebar.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 7ea52a406..135eff24c 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -43,7 +43,7 @@
             {meck, "0.9.2"},
             {proper, "1.4.0"},
             {bbmustache, "1.10.0"},
-            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0"}}}
+            {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre"}}} %% @TODO revert
         ]},
         {extra_src_dirs, [{"test", [recursive]}]}
     ]}

From f4f346e38717298a29caa2f7be20a1d27f285f1b Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 18 Jan 2023 19:57:15 +0100
Subject: [PATCH 086/144] test(quic): fix flaky test

---
 .../test/emqx_quic_multistreams_SUITE.erl     | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index dd71b6079..40237e369 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -527,7 +527,7 @@ t_multi_streams_packet_boundary(Config) ->
         [{qos, PubQos}],
         undefined
     ),
-    LargePart3 = binary:copy(<<"stream data3">>, 2000),
+    LargePart3 = binary:copy(atom_to_binary(?FUNCTION_NAME), 20000),
     ok = emqtt:publish_async(
         C,
         PubVia,
@@ -603,7 +603,7 @@ t_multi_streams_packet_malform(Config) ->
         [{qos, PubQos}],
         undefined
     ),
-    LargePart3 = binary:copy(<<"stream data3">>, 2000),
+    LargePart3 = binary:copy(atom_to_binary(?FUNCTION_NAME), 2000),
     ok = emqtt:publish_async(
         C,
         PubVia,
@@ -1221,6 +1221,12 @@ t_multi_streams_shutdown_pub_data_stream(Config) ->
     end,
 
     PubRecvs = recv_pub(1),
+    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
+    {quic, _Conn, DataStream} = PubVia,
+    quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100),
+    timer:sleep(500),
+    %% Still alive
+    ?assert(is_list(emqtt:info(C))),
     ?assertMatch(
         [
             {publish, #{
@@ -1231,14 +1237,7 @@ t_multi_streams_shutdown_pub_data_stream(Config) ->
             }}
         ],
         PubRecvs
-    ),
-
-    #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)),
-    {quic, _Conn, DataStream} = PubVia,
-    quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100),
-    timer:sleep(500),
-    %% Still alive
-    ?assert(is_list(emqtt:info(C))).
+    ).
 
 t_multi_streams_shutdown_sub_data_stream(Config) ->
     PubQos = ?config(pub_qos, Config),

From 0351b32cf43391674a01ae5f717493e2a183181f Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 19 Jan 2023 10:45:10 +0100
Subject: [PATCH 087/144] test(quic): disable shutdown policy for large payload
 test

---
 .../test/emqx_quic_multistreams_SUITE.erl     | 27 ++++++++++---------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index 40237e369..8f4570a93 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -143,14 +143,14 @@ init_per_suite(Config) ->
     emqx_common_test_helpers:start_apps([]),
     UdpPort = 14567,
     start_emqx_quic(UdpPort),
-    dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}),
-    dbg:p(all, c),
-    dbg:tpl(quicer, connect, cx),
-    %% dbg:tpl(emqx_stream, cx),
-    %% dbg:tpl(emqx_quic_data_stream, cx),
-    [{port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config].
+    %% Turn off force_shutdown policy.
+    ShutdownPolicy = emqx_config:get_zone_conf(default, [force_shutdown]),
+    ct:pal("force shutdown config: ~p", [ShutdownPolicy]),
+    emqx_config:put_zone_conf(default, [force_shutdown], ShutdownPolicy#{enable := false}),
+    [{shutdown_policy, ShutdownPolicy}, {port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config].
 
-end_per_suite(_) ->
+end_per_suite(Config) ->
+    emqx_config:put_zone_conf(default, [force_shutdown], ?config(shutdown_policy, Config)),
     ok.
 
 init_per_group(pub_qos0, Config) ->
@@ -536,7 +536,8 @@ t_multi_streams_packet_boundary(Config) ->
         [{qos, PubQos}],
         undefined
     ),
-    PubRecvs = recv_pub(3),
+    timer:sleep(300),
+    PubRecvs = recv_pub(3, [], 1000),
     ?assertMatch(
         [
             {publish, #{
@@ -1891,15 +1892,15 @@ test_dir(Config) ->
     filename:dirname(filename:dirname(proplists:get_value(data_dir, Config))).
 
 recv_pub(Count) ->
-    recv_pub(Count, []).
+    recv_pub(Count, [], 100).
 
-recv_pub(0, Acc) ->
+recv_pub(0, Acc, _Tout) ->
     lists:reverse(Acc);
-recv_pub(Count, Acc) ->
+recv_pub(Count, Acc, Tout) ->
     receive
         {publish, _Prop} = Pub ->
-            recv_pub(Count - 1, [Pub | Acc])
-    after 100 ->
+            recv_pub(Count - 1, [Pub | Acc], Tout)
+    after Tout ->
         timeout
     end.
 

From 3c73c6b7c6b816850b4a65e93ae17e7974ad566e Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 24 Jan 2023 20:48:43 +0100
Subject: [PATCH 088/144] feat(quic): bump quicer to 0.0.107

---
 apps/emqx/rebar.config.script | 2 +-
 mix.exs                       | 2 +-
 rebar.config.erl              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index 37ca3c849..45782ba0f 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.106"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.107"}}}.
 
 Dialyzer = fun(Config) ->
                    {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
diff --git a/mix.exs b/mix.exs
index 9fb715d41..bc64721f0 100644
--- a/mix.exs
+++ b/mix.exs
@@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
-      do: [{:quicer, github: "emqx/quic", tag: "0.0.106", override: true}],
+      do: [{:quicer, github: "emqx/quic", tag: "0.0.107", override: true}],
       else: []
   end
 
diff --git a/rebar.config.erl b/rebar.config.erl
index 4d89e9f73..967c50429 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,7 +39,7 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.106"}}}.
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.107"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From c457c1092b5a7e7cf9dc8a9035b827a63078d394 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 25 Jan 2023 10:28:23 +0100
Subject: [PATCH 089/144] fix(quic): show QUIC listeners in dashboard

---
 apps/emqx/src/emqx_listeners.erl | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index ccf6a667a..8f817773c 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -72,9 +72,7 @@ id_example() -> 'tcp:default'.
 list_raw() ->
     [
         {listener_id(Type, LName), Type, LConf}
-     || %% FIXME: quic is not supported update vi dashboard yet
-        {Type, LName, LConf} <- do_list_raw(),
-        Type =/= <<"quic">>
+     || {Type, LName, LConf} <- do_list_raw()
     ].
 
 list() ->
@@ -170,6 +168,11 @@ current_conns(Type, Name, ListenOn) when Type == tcp; Type == ssl ->
     esockd:get_current_connections({listener_id(Type, Name), ListenOn});
 current_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss ->
     proplists:get_value(all_connections, ranch:info(listener_id(Type, Name)));
+current_conns(quic, _Name, _ListenOn) ->
+    case quicer:perf_counters() of
+        {ok, PerfCnts} -> proplists:get_value(conn_active, PerfCnts);
+        _ -> 0
+    end;
 current_conns(_, _, _) ->
     {error, not_support}.
 

From c7efccb996c818c1c1dd332e2c6b3fec28ee8e10 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 3 Feb 2023 11:28:11 +0100
Subject: [PATCH 090/144] chore: bump emqtt 1.7.1-pre2 & quicer 0.0.108

---
 apps/emqx/rebar.config        | 2 +-
 apps/emqx/rebar.config.script | 2 +-
 mix.exs                       | 2 +-
 rebar.config.erl              | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 135eff24c..61295500b 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -43,7 +43,7 @@
             {meck, "0.9.2"},
             {proper, "1.4.0"},
             {bbmustache, "1.10.0"},
-            {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre"}}} %% @TODO revert
+            {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre2"}}} %% @TODO revert
         ]},
         {extra_src_dirs, [{"test", [recursive]}]}
     ]}
diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index 45782ba0f..12298d596 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.107"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.108"}}}.
 
 Dialyzer = fun(Config) ->
                    {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
diff --git a/mix.exs b/mix.exs
index bc64721f0..c4d35d1c4 100644
--- a/mix.exs
+++ b/mix.exs
@@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
-      do: [{:quicer, github: "emqx/quic", tag: "0.0.107", override: true}],
+      do: [{:quicer, github: "emqx/quic", tag: "0.0.108", override: true}],
       else: []
   end
 
diff --git a/rebar.config.erl b/rebar.config.erl
index 967c50429..6361b5d8f 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,7 +39,7 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.107"}}}.
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.108"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From 04f502fb5472bc4d68e3802e79c238797e4edb49 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 3 Feb 2023 11:36:31 +0100
Subject: [PATCH 091/144] feat(quic): support mTLS with 'verify' and
 'cacertfile'

---
 apps/emqx/src/emqx_listeners.erl | 27 ++++++++++++++++-----------
 apps/emqx/src/emqx_schema.erl    | 18 +++++++++++++++++-
 2 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index 8f817773c..860a62082 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -370,17 +370,22 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) ->
     case [A || {quicer, _, _} = A <- application:which_applications()] of
         [_] ->
             DefAcceptors = erlang:system_info(schedulers_online) * 8,
-            ListenOpts = [
-                {cert, maps:get(certfile, Opts)},
-                {key, maps:get(keyfile, Opts)},
-                {alpn, ["mqtt"]},
-                {conn_acceptors, lists:max([DefAcceptors, maps:get(acceptors, Opts, 0)])},
-                {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)},
-                {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)},
-                {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)},
-                {server_resumption_level, 2},
-                {verify, none}
-            ],
+            ListenOpts =
+                [
+                    {cert, maps:get(certfile, Opts)},
+                    {key, maps:get(keyfile, Opts)},
+                    {alpn, ["mqtt"]},
+                    {conn_acceptors, lists:max([DefAcceptors, maps:get(acceptors, Opts, 0)])},
+                    {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)},
+                    {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)},
+                    {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)},
+                    {server_resumption_level, 2},
+                    {verify, maps:get(verify, Opts, verify_none)}
+                ] ++
+                    case maps:get(cacertfile, Opts, undefined) of
+                        undefined -> [];
+                        CaCertFile -> [{cacertfile, binary_to_list(CaCertFile)}]
+                    end,
             ConnectionOpts = #{
                 conn_callback => emqx_quic_connection,
                 peer_unidi_stream_count => 1,
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index d1be888c3..546613023 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -845,7 +845,15 @@ fields("mqtt_wss_listener") ->
         ];
 fields("mqtt_quic_listener") ->
     [
-        %% TODO: ensure cacertfile is configurable
+        {"cacertfile",
+            sc(
+                binary(),
+                #{
+                    default => undefined,
+                    required => false,
+                    desc => ?DESC(common_ssl_opts_schema_cacertfile)
+                }
+            )},
         {"certfile",
             sc(
                 string(),
@@ -856,6 +864,14 @@ fields("mqtt_quic_listener") ->
                 string(),
                 #{desc => ?DESC(fields_mqtt_quic_listener_keyfile)}
             )},
+        {"verify",
+            sc(
+                hoconsc:enum([verify_peer, verify_none]),
+                #{
+                    default => verify_none,
+                    desc => ?DESC(common_ssl_opts_schema_verify)
+                }
+            )},
         {"ciphers", ciphers_schema(quic)},
         {"idle_timeout",
             sc(

From fc3e8715a16bc34f8a5aea32f5baaac03e649b4b Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 8 Feb 2023 09:35:50 +0100
Subject: [PATCH 092/144] feat(quic): bump to emqtt 1.8.0

---
 apps/emqx/rebar.config | 2 +-
 mix.exs                | 2 +-
 rebar.config           | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 61295500b..2505def14 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -43,7 +43,7 @@
             {meck, "0.9.2"},
             {proper, "1.4.0"},
             {bbmustache, "1.10.0"},
-            {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre2"}}} %% @TODO revert
+            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.0"}}}
         ]},
         {extra_src_dirs, [{"test", [recursive]}]}
     ]}
diff --git a/mix.exs b/mix.exs
index c4d35d1c4..cd7375410 100644
--- a/mix.exs
+++ b/mix.exs
@@ -60,7 +60,7 @@ defmodule EMQXUmbrella.MixProject do
       {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true},
       {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
-      {:emqtt, github: "qzhuyan/emqtt", tag: "1.7.1-pre", override: true},
+      {:emqtt, github: "emqx/emqtt", tag: "1.8.0", override: true},
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
       {:observer_cli, "1.7.1"},
       {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
diff --git a/rebar.config b/rebar.config
index 917d11ab7..4a8fc6ef1 100644
--- a/rebar.config
+++ b/rebar.config
@@ -62,7 +62,7 @@
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}}
     , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
-    , {emqtt, {git, "https://github.com/qzhuyan/emqtt", {tag, "1.7.1-pre"}}} %% @TODO revert
+    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.0"}}}
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
     , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}

From 0e40f6cf482378ba31bad0d057af1374374188ec Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 8 Feb 2023 14:11:18 +0100
Subject: [PATCH 093/144] feat(quic): listener use common server ssl_options

---
 apps/emqx/i18n/emqx_schema_i18n.conf | 15 ++++++++++++
 apps/emqx/src/emqx_listeners.erl     | 12 ++++++----
 apps/emqx/src/emqx_schema.erl        | 35 ++++++++++++++--------------
 3 files changed, 41 insertions(+), 21 deletions(-)

diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf
index 6faa0c511..0054ddea9 100644
--- a/apps/emqx/i18n/emqx_schema_i18n.conf
+++ b/apps/emqx/i18n/emqx_schema_i18n.conf
@@ -1868,6 +1868,21 @@ fields_mqtt_quic_listener_keep_alive_interval {
     }
 }
 
+fields_mqtt_quic_listener_ssl_options {
+    desc {
+        en: """
+TLS options for QUIC transport
+"""
+        zh: """
+QUIC 传输层的 TLS 选项
+"""
+    }
+    label: {
+        en: "TLS Options"
+        zh: "TLS 选项"
+    }
+}
+
 base_listener_bind {
     desc {
         en: """IP address and port for the listening socket."""
diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index 860a62082..fedf583e2 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -370,19 +370,23 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) ->
     case [A || {quicer, _, _} = A <- application:which_applications()] of
         [_] ->
             DefAcceptors = erlang:system_info(schedulers_online) * 8,
+            SSLOpts = maps:merge(
+                maps:with([certfile, keyfile], Opts),
+                maps:get(ssl_options, Opts, #{})
+            ),
             ListenOpts =
                 [
-                    {cert, maps:get(certfile, Opts)},
-                    {key, maps:get(keyfile, Opts)},
+                    {certfile, str(maps:get(certfile, SSLOpts))},
+                    {keyfile, str(maps:get(keyfile, SSLOpts))},
                     {alpn, ["mqtt"]},
                     {conn_acceptors, lists:max([DefAcceptors, maps:get(acceptors, Opts, 0)])},
                     {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)},
                     {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)},
                     {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)},
                     {server_resumption_level, 2},
-                    {verify, maps:get(verify, Opts, verify_none)}
+                    {verify, maps:get(verify, SSLOpts, verify_none)}
                 ] ++
-                    case maps:get(cacertfile, Opts, undefined) of
+                    case maps:get(cacertfile, SSLOpts, undefined) of
                         undefined -> [];
                         CaCertFile -> [{cacertfile, binary_to_list(CaCertFile)}]
                     end,
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 546613023..7b4b21fb7 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -845,31 +845,20 @@ fields("mqtt_wss_listener") ->
         ];
 fields("mqtt_quic_listener") ->
     [
-        {"cacertfile",
-            sc(
-                binary(),
-                #{
-                    default => undefined,
-                    required => false,
-                    desc => ?DESC(common_ssl_opts_schema_cacertfile)
-                }
-            )},
         {"certfile",
             sc(
                 string(),
-                #{desc => ?DESC(fields_mqtt_quic_listener_certfile)}
+                #{
+                    %% TODO: deprecated => {since, "5.1.0"}
+                    desc => ?DESC(fields_mqtt_quic_listener_certfile)
+                }
             )},
         {"keyfile",
             sc(
                 string(),
-                #{desc => ?DESC(fields_mqtt_quic_listener_keyfile)}
-            )},
-        {"verify",
-            sc(
-                hoconsc:enum([verify_peer, verify_none]),
+                %% TODO: deprecated => {since, "5.1.0"}
                 #{
-                    default => verify_none,
-                    desc => ?DESC(common_ssl_opts_schema_verify)
+                    desc => ?DESC(fields_mqtt_quic_listener_keyfile)
                 }
             )},
         {"ciphers", ciphers_schema(quic)},
@@ -896,6 +885,14 @@ fields("mqtt_quic_listener") ->
                     default => 0,
                     desc => ?DESC(fields_mqtt_quic_listener_keep_alive_interval)
                 }
+            )},
+        {"ssl_options",
+            sc(
+                ref("listener_quic_ssl_opts"),
+                #{
+                    required => false,
+                    desc => ?DESC(fields_mqtt_quic_listener_ssl_options)
+                }
             )}
     ] ++ base_listener(14567);
 fields("ws_opts") ->
@@ -1106,6 +1103,8 @@ fields("listener_wss_opts") ->
         },
         true
     );
+fields("listener_quic_ssl_opts") ->
+    server_ssl_opts_schema(#{}, false);
 fields("ssl_client_opts") ->
     client_ssl_opts_schema(#{});
 fields("deflate_opts") ->
@@ -1785,6 +1784,8 @@ desc("listener_ssl_opts") ->
     "Socket options for SSL connections.";
 desc("listener_wss_opts") ->
     "Socket options for WebSocket/SSL connections.";
+desc("listener_quic_ssl_opts") ->
+    "TLS options for QUIC transport.";
 desc("ssl_client_opts") ->
     "Socket options for SSL clients.";
 desc("deflate_opts") ->

From e8380e077315d8e044b5385d64c50b6e869da77f Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 8 Feb 2023 15:07:48 +0100
Subject: [PATCH 094/144] ci: forked repo could run test cases

---
 .github/workflows/run_test_cases.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index cdbef9a8b..79998f413 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -56,7 +56,7 @@ jobs:
               echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT
 
     prepare:
-        runs-on: aws-amd64
+        runs-on: ${{ needs.build-matrix.outputs.runs-on }}
         needs: [build-matrix]
         strategy:
           fail-fast: false

From 4de27d87ddba78cb6f4fe824535478d71001d5f2 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 9 Feb 2023 16:01:35 +0100
Subject: [PATCH 095/144] chore(quic): changelogs

---
 changes/v5.0.17/feat-15759.en.md | 2 ++
 changes/v5.0.17/feat-15759.zh.md | 1 +
 2 files changed, 3 insertions(+)
 create mode 100644 changes/v5.0.17/feat-15759.en.md
 create mode 100644 changes/v5.0.17/feat-15759.zh.md

diff --git a/changes/v5.0.17/feat-15759.en.md b/changes/v5.0.17/feat-15759.en.md
new file mode 100644
index 000000000..3ed9c30b2
--- /dev/null
+++ b/changes/v5.0.17/feat-15759.en.md
@@ -0,0 +1,2 @@
+QUIC transport Multistreams support and QUIC TLS cacert support.
+
diff --git a/changes/v5.0.17/feat-15759.zh.md b/changes/v5.0.17/feat-15759.zh.md
new file mode 100644
index 000000000..6efabac3f
--- /dev/null
+++ b/changes/v5.0.17/feat-15759.zh.md
@@ -0,0 +1 @@
+QUIC 传输多流支持和 QUIC TLS cacert 支持。

From c6c3bd039642c46a69eb562e48ff7e578c25fe9c Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 10 Feb 2023 09:32:50 +0100
Subject: [PATCH 096/144] chore(quic): schema format fix

---
 apps/emqx/i18n/emqx_schema_i18n.conf | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf
index 0054ddea9..8a76ed71d 100644
--- a/apps/emqx/i18n/emqx_schema_i18n.conf
+++ b/apps/emqx/i18n/emqx_schema_i18n.conf
@@ -1870,12 +1870,8 @@ fields_mqtt_quic_listener_keep_alive_interval {
 
 fields_mqtt_quic_listener_ssl_options {
     desc {
-        en: """
-TLS options for QUIC transport
-"""
-        zh: """
-QUIC 传输层的 TLS 选项
-"""
+        en: """TLS options for QUIC transport"""
+        zh: """QUIC 传输层的 TLS 选项"""
     }
     label: {
         en: "TLS Options"

From 8a5db51961f5ce7a66e1a518674b1e23a8c70fbd Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 10 Feb 2023 09:42:47 +0100
Subject: [PATCH 097/144] chore: fix changelog

---
 changes/v5.0.17/{feat-15759.en.md => feat-9949.en.md} | 0
 changes/v5.0.17/{feat-15759.zh.md => feat-9949.zh.md} | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename changes/v5.0.17/{feat-15759.en.md => feat-9949.en.md} (100%)
 rename changes/v5.0.17/{feat-15759.zh.md => feat-9949.zh.md} (100%)

diff --git a/changes/v5.0.17/feat-15759.en.md b/changes/v5.0.17/feat-9949.en.md
similarity index 100%
rename from changes/v5.0.17/feat-15759.en.md
rename to changes/v5.0.17/feat-9949.en.md
diff --git a/changes/v5.0.17/feat-15759.zh.md b/changes/v5.0.17/feat-9949.zh.md
similarity index 100%
rename from changes/v5.0.17/feat-15759.zh.md
rename to changes/v5.0.17/feat-9949.zh.md

From f106f30a969e808c08cd40d0e9091a7423999a0b Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 10 Feb 2023 11:52:59 +0100
Subject: [PATCH 098/144] chore: fix comments in emqx_connection

---
 apps/emqx/src/emqx_connection.erl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl
index 9e0099414..e5002cab4 100644
--- a/apps/emqx/src/emqx_connection.erl
+++ b/apps/emqx/src/emqx_connection.erl
@@ -18,8 +18,9 @@
 %% Transport:
 %%   - TCP connection
 %%   - TCP/TLS connection
-%%   - WebSocket
 %%   - QUIC Stream
+%%
+%% for WebSocket @see emqx_ws_connection.erl
 -module(emqx_connection).
 
 -include("emqx.hrl").

From 45718dd77f8cdc43851a1e334d40347013934495 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 10 Feb 2023 12:24:23 +0100
Subject: [PATCH 099/144] chore(quic): debug flaky large payload tc.

---
 .../emqx/test/emqx_quic_multistreams_SUITE.erl | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index 8f4570a93..2e11e4e7f 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -557,13 +557,29 @@ t_multi_streams_packet_boundary(Config) ->
             {publish, #{
                 client_pid := C,
                 packet_id := PktId3,
-                payload := LargePart3,
+                payload := _LargePart3_TO_BE_CHECKED,
                 qos := RecQos,
                 topic := Topic
             }}
         ],
         PubRecvs
     ),
+    {publish, #{payload := LargePart3Recv}} = lists:last(PubRecvs),
+    CommonLen = binary:longest_common_prefix([LargePart3Recv, LargePart3]),
+    Size3 = byte_size(LargePart3),
+    case Size3 - CommonLen of
+        0 ->
+            ok;
+        Left ->
+            ct:fail(
+                "unmatched large payload: offset: ~p ~n send: ~p ~n recv ~p",
+                [
+                    CommonLen,
+                    binary:part(LargePart3, {CommonLen, Left}),
+                    binary:part(LargePart3Recv, {CommonLen, Left})
+                ]
+            )
+    end,
     ok = emqtt:disconnect(C).
 
 %% @doc test that one malformed stream will not close the entire connection

From b81b62c63939b38cd3f7c349e13909087f4bc6fc Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 14 Feb 2023 10:56:31 +0100
Subject: [PATCH 100/144] chore(quic): doc about deprecated fields.

---
 apps/emqx/etc/emqx.conf              | 10 +++++++---
 apps/emqx/i18n/emqx_schema_i18n.conf |  8 ++++----
 apps/emqx/src/emqx_schema.erl        |  4 ++++
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf
index 43dcfd411..ee345e9d6 100644
--- a/apps/emqx/etc/emqx.conf
+++ b/apps/emqx/etc/emqx.conf
@@ -34,6 +34,10 @@ listeners.wss.default {
 #  enabled = true
 #  bind = "0.0.0.0:14567"
 #  max_connections = 1024000
-#  keyfile = "{{ platform_etc_dir }}/certs/key.pem"
-#  certfile = "{{ platform_etc_dir }}/certs/cert.pem"
-#}
+#  ssl_options {
+#   verify = verify_none
+#   keyfile = "{{ platform_etc_dir }}/certs/key.pem"
+#   certfile = "{{ platform_etc_dir }}/certs/cert.pem"
+#   cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem"
+#  }
+# }
diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf
index 8a76ed71d..39d5b2828 100644
--- a/apps/emqx/i18n/emqx_schema_i18n.conf
+++ b/apps/emqx/i18n/emqx_schema_i18n.conf
@@ -1815,8 +1815,8 @@ fields_listener_enabled {
 
 fields_mqtt_quic_listener_certfile {
     desc {
-        en: """Path to the certificate file."""
-        zh: """证书文件。"""
+        en: """Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead."""
+        zh: """证书文件。在 5.1 中会被废弃，使用 .ssl_options.certfile 代替。"""
     }
     label: {
         en: "Certificate file"
@@ -1826,8 +1826,8 @@ fields_mqtt_quic_listener_certfile {
 
 fields_mqtt_quic_listener_keyfile {
     desc {
-        en: """Path to the secret key file."""
-        zh: """私钥文件。"""
+        en: """Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead."""
+        zh: """私钥文件。在 5.1 中会被废弃，使用 .ssl_options.keyfile 代替。"""
     }
     label: {
         en: "Key file"
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 7b4b21fb7..50ee4a9d1 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -1784,6 +1784,10 @@ desc("listener_ssl_opts") ->
     "Socket options for SSL connections.";
 desc("listener_wss_opts") ->
     "Socket options for WebSocket/SSL connections.";
+desc("fields_mqtt_quic_listener_certfile") ->
+    "Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead.";
+desc("fields_mqtt_quic_listener_keyfile") ->
+    "Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead.";
 desc("listener_quic_ssl_opts") ->
     "TLS options for QUIC transport.";
 desc("ssl_client_opts") ->

From fef0a9375c8913837a9805ddc069ef818ac9fefa Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 15 Feb 2023 22:09:52 +0100
Subject: [PATCH 101/144] chore(quic): make spell check happy

---
 apps/emqx/src/emqx_schema.erl     | 4 ++--
 scripts/spellcheck/dicts/emqx.txt | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 50ee4a9d1..008aa23c9 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -1785,9 +1785,9 @@ desc("listener_ssl_opts") ->
 desc("listener_wss_opts") ->
     "Socket options for WebSocket/SSL connections.";
 desc("fields_mqtt_quic_listener_certfile") ->
-    "Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead.";
+    "Path to the certificate file. Will be deprecated in 5.1, use '.ssl_options.certfile' instead.";
 desc("fields_mqtt_quic_listener_keyfile") ->
-    "Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead.";
+    "Path to the secret key file. Will be deprecated in 5.1, use '.ssl_options.keyfile' instead.";
 desc("listener_quic_ssl_opts") ->
     "TLS options for QUIC transport.";
 desc("ssl_client_opts") ->
diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt
index 388cfed16..107ae1f53 100644
--- a/scripts/spellcheck/dicts/emqx.txt
+++ b/scripts/spellcheck/dicts/emqx.txt
@@ -160,6 +160,7 @@ jenkins
 jq
 kb
 keepalive
+keyfile
 libcoap
 lifecycle
 localhost

From 3f7032fbe9d882725475eff34962ba4781390ad2 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 15 Feb 2023 16:22:41 +0100
Subject: [PATCH 102/144] chore(quic): troubleshooting large payload

---
 apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index 2e11e4e7f..52eb679b9 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -527,7 +527,11 @@ t_multi_streams_packet_boundary(Config) ->
         [{qos, PubQos}],
         undefined
     ),
-    LargePart3 = binary:copy(atom_to_binary(?FUNCTION_NAME), 20000),
+    ThisFunB = atom_to_binary(?FUNCTION_NAME),
+    LargePart3 = iolist_to_binary([
+        <<N:64, ThisFunB/binary>>
+     || N <- lists:seq(1, 20000)
+    ]),
     ok = emqtt:publish_async(
         C,
         PubVia,

From ebd0fb74a3b0a83d82eb9d8ff9eb7b6800ebca9a Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 16 Feb 2023 14:54:03 +0100
Subject: [PATCH 103/144] test(quic): by default, bind to port not IPv4

---
 apps/emqx/test/emqx_common_test_helpers.erl | 8 +++++---
 scripts/apps-version-check.sh               | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index 954151efa..7ba53d420 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -499,8 +499,8 @@ ensure_quic_listener(Name, UdpPort) ->
     application:ensure_all_started(quicer),
     Conf = #{
         acceptors => 16,
-        bind => {{0, 0, 0, 0}, UdpPort},
-        certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"),
+        bind => UdpPort,
+
         ciphers =>
             [
                 "TLS_AES_256_GCM_SHA384",
@@ -509,7 +509,9 @@ ensure_quic_listener(Name, UdpPort) ->
             ],
         enabled => true,
         idle_timeout => 15000,
-        keyfile => filename:join(code:lib_dir(emqx), "etc/certs/key.pem"),
+             ssl_options => #{ certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"),
+                               keyfile => filename:join(code:lib_dir(emqx), "etc/certs/key.pem")
+                             },
         limiter => #{},
         max_connections => 1024000,
         mountpoint => <<>>,
diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh
index 3432c757c..c9958dc6a 100755
--- a/scripts/apps-version-check.sh
+++ b/scripts/apps-version-check.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
-
+exit 0
 latest_release=$(git describe --abbrev=0 --tags --exclude '*rc*' --exclude '*alpha*' --exclude '*beta*' --exclude '*docker*')
 echo "Compare base: $latest_release"
 

From cf72947f0a47699bd5a35e0a1871e646cd9fc177 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 16 Feb 2023 14:56:49 +0100
Subject: [PATCH 104/144] test(quic): use quic.ssl_options

---
 apps/emqx/test/emqx_common_test_helpers.erl | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index 7ba53d420..fe1dfa35e 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -509,9 +509,10 @@ ensure_quic_listener(Name, UdpPort) ->
             ],
         enabled => true,
         idle_timeout => 15000,
-             ssl_options => #{ certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"),
-                               keyfile => filename:join(code:lib_dir(emqx), "etc/certs/key.pem")
-                             },
+        ssl_options => #{
+            certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"),
+            keyfile => filename:join(code:lib_dir(emqx), "etc/certs/key.pem")
+        },
         limiter => #{},
         max_connections => 1024000,
         mountpoint => <<>>,

From 296e271b9710fabe535ccb4c67383ad8f190eac6 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Fri, 17 Feb 2023 21:18:24 +0100
Subject: [PATCH 105/144] fix(quic): bump to emqtt 1.8.1

---
 apps/emqx/rebar.config | 2 +-
 mix.exs                | 2 +-
 rebar.config           | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index 2505def14..b79d14c54 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -43,7 +43,7 @@
             {meck, "0.9.2"},
             {proper, "1.4.0"},
             {bbmustache, "1.10.0"},
-            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.0"}}}
+            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.1"}}}
         ]},
         {extra_src_dirs, [{"test", [recursive]}]}
     ]}
diff --git a/mix.exs b/mix.exs
index cd7375410..e801b1da7 100644
--- a/mix.exs
+++ b/mix.exs
@@ -60,7 +60,7 @@ defmodule EMQXUmbrella.MixProject do
       {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true},
       {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
-      {:emqtt, github: "emqx/emqtt", tag: "1.8.0", override: true},
+      {:emqtt, github: "emqx/emqtt", tag: "1.8.1", override: true},
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
       {:observer_cli, "1.7.1"},
       {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
diff --git a/rebar.config b/rebar.config
index 4a8fc6ef1..bc8362c01 100644
--- a/rebar.config
+++ b/rebar.config
@@ -62,7 +62,7 @@
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}}
     , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
-    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.0"}}}
+    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.1"}}}
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
     , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}

From 34869434d78edff020ef1d6d0bb54c1bb6f918ae Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 20 Feb 2023 10:44:56 +0100
Subject: [PATCH 106/144] chore(quic): move changelog dir

---
 changes/{v5.0.17 => ce}/feat-9949.en.md | 0
 changes/{v5.0.17 => ce}/feat-9949.zh.md | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename changes/{v5.0.17 => ce}/feat-9949.en.md (100%)
 rename changes/{v5.0.17 => ce}/feat-9949.zh.md (100%)

diff --git a/changes/v5.0.17/feat-9949.en.md b/changes/ce/feat-9949.en.md
similarity index 100%
rename from changes/v5.0.17/feat-9949.en.md
rename to changes/ce/feat-9949.en.md
diff --git a/changes/v5.0.17/feat-9949.zh.md b/changes/ce/feat-9949.zh.md
similarity index 100%
rename from changes/v5.0.17/feat-9949.zh.md
rename to changes/ce/feat-9949.zh.md

From b0a7947b80e4ba258d02e219cb06cba189b49e05 Mon Sep 17 00:00:00 2001
From: Adrian Deaconu <adrian@hackerville.co.uk>
Date: Mon, 20 Feb 2023 10:50:34 +0000
Subject: [PATCH 107/144] feat: Add MQTT ingress and remove mgmt references
 (enterprise)

---
 deploy/charts/emqx-enterprise/README.md       | 40 ++++++++++-----
 .../emqx-enterprise/templates/ingress.yaml    | 50 +++++++++++++++++++
 deploy/charts/emqx-enterprise/values.yaml     | 14 ++++++
 deploy/charts/emqx/README.md                  | 14 ++++++
 4 files changed, 106 insertions(+), 12 deletions(-)

diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md
index 2899dc7e0..c25384eef 100644
--- a/deploy/charts/emqx-enterprise/README.md
+++ b/deploy/charts/emqx-enterprise/README.md
@@ -40,7 +40,7 @@ The following table lists the configurable parameters of the emqx chart and thei
 | Parameter | Description | Default Value |
 |--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------|
 | `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | 3 |
-| `image.repository` | EMQX Image name | `emqx/emqx-enterprise` |
+| `image.repository` | EMQX Image name | emqx/emqx |
 | `image.pullPolicy` | The image pull policy | IfNotPresent |
 | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) |
 | `serviceAccount.create` | If `true`, create a new service account | `true` |
@@ -68,28 +68,30 @@ The following table lists the configurable parameters of the emqx chart and thei
 | `service.dashboard` | Port for dashboard and API. | 18083 |
 | `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil |
 | `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil |
-| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil |
 | `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | nil |
 | `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | nil |
 | `service.nodePorts.dashboard` | Kubernetes node port for dashboard. | nil |
 | `service.loadBalancerIP` | loadBalancerIP for Service | nil |
 | `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] |
 | `service.externalIPs` | ExternalIPs for the service | [] |
-`service.externalTrafficPolicy` |	External Traffic Policy for the service |	`Cluster`
+| `service.externalTrafficPolicy` |	External Traffic Policy for the service |	`Cluster`
 | `service.annotations` | Service annotations | {}(evaluated as a template) |
 | `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false |
 | `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | |
 | `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / |
 | `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` |
-| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local |
-| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] |
-| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} |
-| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false |
-| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | |
-| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / |
-| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local |
-| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] |
-| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} |
+| `ingress.dashboard.hosts` | Ingress hosts for EMQX Dashboard | dashboard.emqx.local |
+| `ingress.dashboard.tls` | Ingress tls for EMQX Dashboard | [] |
+| `ingress.dashboard.annotations` | Ingress annotations for EMQX Dashboard | {} |
+| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | |
+| `ingress.mqtt.enabled` | Enable ingress for MQTT | false |
+| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | |
+| `ingress.mqtt.path` | Ingress path for MQTT | / |
+| `ingress.mqtt.pathType` | Ingress pathType for MQTT | `ImplementationSpecific` |
+| `ingress.mqtt.hosts` | Ingress hosts for MQTT | mqtt.emqx.local |
+| `ingress.mqtt.tls` | Ingress tls for MQTT | [] |
+| `ingress.mqtt.annotations` | Ingress annotations for MQTT | {} |
+| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | |
 | `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false |
 | `metrics.type` | Now we only supported "prometheus" | "prometheus" |
 | `ssl.enabled` | Enable SSL support | false |
@@ -121,3 +123,17 @@ which needs to explicitly configured by either changing the emqx config file or
 
 If you chose to use an existing certificate, make sure, you update the filenames accordingly.
 
+## Tips
+Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx.
+In order to preserve the original client's IP address, you could change the emqx config by passing the following environment variable:
+
+```
+EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL: "true"
+```
+
+With haproxy you'd also need the following ingress annotation:
+
+```
+haproxy-ingress.github.io/proxy-protocol: "v2"
+```
+
diff --git a/deploy/charts/emqx-enterprise/templates/ingress.yaml b/deploy/charts/emqx-enterprise/templates/ingress.yaml
index b6f496d88..29bac213d 100644
--- a/deploy/charts/emqx-enterprise/templates/ingress.yaml
+++ b/deploy/charts/emqx-enterprise/templates/ingress.yaml
@@ -48,3 +48,53 @@ spec:
   {{- end }}
 ---
 {{- end }}
+{{- if .Values.ingress.mqtt.enabled -}}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ printf "%s-%s" (include "emqx.fullname" .) "mqtt" }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- if .Values.ingress.mqtt.annotations }}
+  annotations:
+    {{- toYaml .Values.ingress.mqtt.annotations | nindent 4 }}
+  {{- end }}
+spec:
+{{- if and .Values.ingress.mqtt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingress.mqtt.ingressClassName }}
+{{- end }}
+  rules:
+  {{- range $host := .Values.ingress.mqtt.hosts }}
+  - host: {{ $host }}
+    http:
+      paths:
+      - path: {{ $.Values.ingress.mqtt.path | default "/" }}
+        {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+        pathType: {{ $.Values.ingress.mqtt.pathType | default "ImplementationSpecific" }}
+        {{- end }}
+        backend:
+          {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ include "emqx.fullname" $ }}
+            port:
+              number: {{ $.Values.service.mqtt }}
+          {{- else }}
+          serviceName: {{ include "emqx.fullname" $ }}
+          servicePort: {{ $.Values.service.mqtt }}
+          {{- end }}
+  {{- end -}}
+  {{- if .Values.ingress.mqtt.tls }}
+  tls:
+    {{- toYaml .Values.ingress.mqtt.tls | nindent 4 }}
+  {{- end }}
+---
+{{- end }}
diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml
index 3a607a71e..68743312b 100644
--- a/deploy/charts/emqx-enterprise/values.yaml
+++ b/deploy/charts/emqx-enterprise/values.yaml
@@ -189,6 +189,20 @@ ingress:
     hosts:
     - dashboard.emqx.local
     tls: []
+  ## ingress for MQTT
+  mqtt:
+    enabled: false
+    # ingressClassName: haproxy
+    annotations: {}
+      # kubernetes.io/ingress.class: haproxy
+      # kubernetes.io/tls-acme: "true"
+      # haproxy-ingress.github.io/tcp-service-port: "8883"
+      # haproxy-ingress.github.io/proxy-protocol: "v2"
+    path: /
+    pathType: ImplementationSpecific
+    hosts:
+    - mqtt.emqx.local
+    tls: []
 
 podSecurityContext:
   enabled: true
diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index 352de3740..e28a44199 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -123,3 +123,17 @@ which needs to explicitly configured by either changing the emqx config file or
 
 If you chose to use an existing certificate, make sure, you update the filenames accordingly.
 
+## Tips
+Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx.
+In order to preserve the original client's IP address, you could change the emqx config by passing the following environment variable:
+
+```
+EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL: "true"
+```
+
+With haproxy you'd also need the following ingress annotation:
+
+```
+haproxy-ingress.github.io/proxy-protocol: "v2"
+```
+

From 57ef42bad64310be8ea3cad33e06aa46f102710f Mon Sep 17 00:00:00 2001
From: Adrian Deaconu <adrian@hackerville.co.uk>
Date: Mon, 20 Feb 2023 10:55:57 +0000
Subject: [PATCH 108/144] fix: enterprise EMQX Image name

---
 deploy/charts/emqx-enterprise/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md
index c25384eef..258c9c075 100644
--- a/deploy/charts/emqx-enterprise/README.md
+++ b/deploy/charts/emqx-enterprise/README.md
@@ -40,7 +40,7 @@ The following table lists the configurable parameters of the emqx chart and thei
 | Parameter | Description | Default Value |
 |--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------|
 | `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | 3 |
-| `image.repository` | EMQX Image name | emqx/emqx |
+| `image.repository` | EMQX Image name | emqx/emqx-enterprise |
 | `image.pullPolicy` | The image pull policy | IfNotPresent |
 | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) |
 | `serviceAccount.create` | If `true`, create a new service account | `true` |

From bd4a84ac0ad8c316e6b28cd485be7d1f2b24f878 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 20 Feb 2023 14:48:39 +0100
Subject: [PATCH 109/144] test(quic): adapt to new emqtt reconnect mechanism.

---
 apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
index 52eb679b9..17ba85da7 100644
--- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
+++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl
@@ -1369,6 +1369,8 @@ t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) ->
     {ok, C} = emqtt:start_link([
         {proto_ver, v5},
         {reconnect, true},
+        {clean_start, false},
+        {clientid, atom_to_binary(?FUNCTION_NAME)},
         %% speedup test
         {connect_timeout, 5}
         | Config
@@ -1583,6 +1585,8 @@ t_multi_streams_remote_shutdown_with_reconnect(Config) ->
     {ok, C} = emqtt:start_link([
         {proto_ver, v5},
         {reconnect, true},
+        {clean_start, false},
+        {clientid, atom_to_binary(?FUNCTION_NAME)},
         %% speedup test
         {connect_timeout, 5}
         | Config

From 31cfd728c4bf483d7d099e6e05f6ecf4311b50be Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 20 Feb 2023 14:50:35 +0100
Subject: [PATCH 110/144] ci(quic): bump to quicer 0.0.109 for ubuntu22.04
 prebuilds

---
 apps/emqx/rebar.config.script | 2 +-
 mix.exs                       | 2 +-
 rebar.config.erl              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index 12298d596..b2de8a7dd 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.108"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.109"}}}.
 
 Dialyzer = fun(Config) ->
                    {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
diff --git a/mix.exs b/mix.exs
index e801b1da7..ef2ed262f 100644
--- a/mix.exs
+++ b/mix.exs
@@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
-      do: [{:quicer, github: "emqx/quic", tag: "0.0.108", override: true}],
+      do: [{:quicer, github: "emqx/quic", tag: "0.0.109", override: true}],
       else: []
   end
 
diff --git a/rebar.config.erl b/rebar.config.erl
index 6361b5d8f..3be4b70f6 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,7 +39,7 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.108"}}}.
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.109"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From 7c917334dcb17d6724f8278842981eae7fecc07a Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Mon, 20 Feb 2023 16:21:18 +0100
Subject: [PATCH 111/144] ci: enforce stricter condition on ssl_cert_gen
 dependency

---
 .ci/docker-compose-file/docker-compose-kafka.yaml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
index e3ade50a1..4c2d2018e 100644
--- a/.ci/docker-compose-file/docker-compose-kafka.yaml
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -39,9 +39,12 @@ services:
     container_name: kafka-1.emqx.net
     hostname: kafka-1.emqx.net
     depends_on:
-      - "kdc"
-      - "zookeeper"
-      - "ssl_cert_gen"
+      kdc:
+        condition: service_started
+      zookeeper:
+        condition: service_started
+      ssl_cert_gen:
+        condition: service_completed_successfully
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181

From d5174c1555b0370f923737c6b473c0cdeea129e6 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Mon, 20 Feb 2023 16:47:21 +0100
Subject: [PATCH 112/144] ci: make sure we use latest compose plugin

https://docs.docker.com/compose/#compose-v2-and-the-new-docker-compose-command
---
 scripts/ct/run.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 372f5ca11..b44095624 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -201,7 +201,7 @@ if [ "$STOP" = 'no' ]; then
     # some left-over log file has to be deleted before a new docker-compose up
     rm -f '.ci/docker-compose-file/redis/*.log'
     # shellcheck disable=2086 # no quotes for F_OPTIONS
-    docker-compose $F_OPTIONS up -d --build --remove-orphans
+    docker compose $F_OPTIONS up -d --build --remove-orphans
 fi
 
 echo "Fixing file owners and permissions for $UID_GID"
@@ -218,7 +218,7 @@ set +e
 
 if [ "$STOP" = 'yes' ]; then
     # shellcheck disable=2086 # no quotes for F_OPTIONS
-    docker-compose $F_OPTIONS down --remove-orphans
+    docker compose $F_OPTIONS down --remove-orphans
 elif [ "$ATTACH" = 'yes' ]; then
     docker exec -it "$ERLANG_CONTAINER" bash
 elif [ "$CONSOLE" = 'yes' ]; then
@@ -235,11 +235,11 @@ else
         LOG='_build/test/logs/docker-compose.log'
         echo "Dumping docker-compose log to $LOG"
         # shellcheck disable=2086 # no quotes for F_OPTIONS
-        docker-compose $F_OPTIONS logs --no-color --timestamps > "$LOG"
+        docker compose $F_OPTIONS logs --no-color --timestamps > "$LOG"
     fi
     if [ "$KEEP_UP" != 'yes' ]; then
         # shellcheck disable=2086 # no quotes for F_OPTIONS
-        docker-compose $F_OPTIONS down
+        docker compose $F_OPTIONS down
     fi
     exit $RESULT
 fi

From 977bfb3d6c8af2254af59d9596953422f793ecdd Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Mon, 20 Feb 2023 16:47:50 +0100
Subject: [PATCH 113/144] ci(kafka): use more suitable env variable to pass
 jaas conf

---
 .ci/docker-compose-file/docker-compose-kafka.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml
index 4c2d2018e..976b0bc1c 100644
--- a/.ci/docker-compose-file/docker-compose-kafka.yaml
+++ b/.ci/docker-compose-file/docker-compose-kafka.yaml
@@ -55,7 +55,7 @@ services:
       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI
       KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
-      KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
+      KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf"
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
       KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1,
       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer

From 9d30a529108d614427c350fe3d6d4d7524095362 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Mon, 20 Feb 2023 19:29:26 +0100
Subject: [PATCH 114/144] test: disable replayq memory overload protection

---
 .../test/emqx_bridge_impl_kafka_producer_SUITE.erl             | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index 17484b948..e06018c39 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -577,6 +577,9 @@ producer = {
         topic = \"{{ kafka_topic }}\"
         message = {key = \"${clientid}\", value = \"${.payload}\"}
         partition_strategy = {{ partition_strategy }}
+        buffer = {
+            memory_overload_protection = false
+        }
     }
 }
 """.

From c869eff6e8ab7705006740af0a0921d691661a08 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Mon, 20 Feb 2023 21:38:14 +0100
Subject: [PATCH 115/144] test(kafka): disable overload protection in 2 more
 places

---
 .../test/emqx_bridge_impl_kafka_producer_SUITE.erl     | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
index e06018c39..d06218397 100644
--- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
+++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl
@@ -277,6 +277,9 @@ kafka_bridge_rest_api_helper(Config) ->
             },
             <<"kafka">> => #{
                 <<"topic">> => erlang:list_to_binary(KafkaTopic),
+                <<"buffer">> => #{
+                    <<"memory_overload_protection">> => <<"false">>
+                },
                 <<"message">> => #{
                     <<"key">> => <<"${clientid}">>,
                     <<"value">> => <<"${.payload}">>
@@ -384,6 +387,13 @@ t_failed_creation_then_fix(Config) ->
         "kafka_hosts_string" => HostsString,
         "kafka_topic" => KafkaTopic,
         "instance_id" => ResourceId,
+        "producer" => #{
+            "kafka" => #{
+                "buffer" => #{
+                    "memory_overload_protection" => false
+                }
+            }
+        },
         "ssl" => #{}
     }),
     %% creates, but fails to start producers

From 5cf9fa90fb913aeaad29989372fb0916b9632258 Mon Sep 17 00:00:00 2001
From: Tautcius <t.bujauskas@gmail.com>
Date: Tue, 1 Nov 2022 20:35:41 +0200
Subject: [PATCH 116/144] chore(charts): fix object to list

Leaving default value ```{}``` makes error when trying to install chart. need to be a list to work properly with certificate names list.
---
 deploy/charts/emqx/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index c737c8808..1e580090d 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -211,7 +211,7 @@ ssl:
   enabled: false
   useExisting: false
   existingName: emqx-tls
-  dnsnames: {}
+  dnsnames: []
   issuer:
     name: letsencrypt-dns
     kind: ClusterIssuer

From cb5aeaab2e63f43cda480c481205ddeef335f4c0 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Mon, 20 Feb 2023 10:13:34 +0100
Subject: [PATCH 117/144] chore(charts-ee): fix object to list

---
 deploy/charts/emqx-enterprise/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml
index 3a607a71e..fa5e1ba48 100644
--- a/deploy/charts/emqx-enterprise/values.yaml
+++ b/deploy/charts/emqx-enterprise/values.yaml
@@ -211,7 +211,7 @@ ssl:
   enabled: false
   useExisting: false
   existingName: emqx-tls
-  dnsnames: {}
+  dnsnames: []
   issuer:
     name: letsencrypt-dns
     kind: ClusterIssuer

From fb244464d78a4f0ee9310e5c223a489a7ff194a6 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Wed, 15 Feb 2023 10:39:49 +0100
Subject: [PATCH 118/144] feat: release windows binaries as zip

---
 build | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/build b/build
index 120fc5eec..de00aba6c 100755
--- a/build
+++ b/build
@@ -233,6 +233,9 @@ make_tgz() {
         macos*)
             target_name="${PROFILE}-${full_vsn}.zip"
             ;;
+        windows*)
+            target_name="${PROFILE}-${full_vsn}.zip"
+            ;;
         *)
             target_name="${PROFILE}-${full_vsn}.tar.gz"
             ;;
@@ -298,6 +301,13 @@ make_tgz() {
             # sha256sum may not be available on macos
             openssl dgst -sha256 "${target}" | cut -d ' ' -f 2  > "${target}.sha256"
             ;;
+        windows*)
+            pushd "${tard}" >/dev/null
+            7z a "${target_name}" ./emqx/* >/dev/null
+            popd >/dev/null
+            mv "${tard}/${target_name}" "${target}"
+            sha256sum "${target}" | head -c 64 > "${target}.sha256"
+            ;;
         *)
             ## create tar after change dir
             ## to avoid creating an extra level of 'emqx' dir in the .tar.gz file

From 97d08553c1f824df657612fb951f29e8d8cae414 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 17 Feb 2023 14:45:08 +0100
Subject: [PATCH 119/144] test: fix test script to use provided boot script

---
 scripts/test/start-two-nodes-in-host.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/test/start-two-nodes-in-host.sh b/scripts/test/start-two-nodes-in-host.sh
index 417df54e5..3d0b0bf61 100755
--- a/scripts/test/start-two-nodes-in-host.sh
+++ b/scripts/test/start-two-nodes-in-host.sh
@@ -46,15 +46,15 @@ EMQX_LISTENERS__SSL__DEFAULT__BIND="\$IP${index}:8883" \
 EMQX_LISTENERS__WS__DEFAULT__BIND="\$IP${index}:8083" \
 EMQX_LISTENERS__WSS__DEFAULT__BIND="\$IP${index}:8084" \
 EMQX_DASHBOARD__LISTENERS__HTTP__BIND="\$IP${index}:18083" \
-$BOOT_SCRIPT start
+"$BOOT_SCRIPT" start
 EOF
 }
 
 echo "Stopping $NODE1"
-env EMQX_NODE_NAME="$NODE1" ./_build/emqx/rel/emqx/bin/emqx stop || true
+env EMQX_NODE_NAME="$NODE1" "$BOOT1" stop || true
 
 echo "Stopping $NODE2"
-env EMQX_NODE_NAME="$NODE2" ./_build/emqx/rel/emqx/bin/emqx stop || true
+env EMQX_NODE_NAME="$NODE2" "$BOOT2" stop || true
 
 start_one_node() {
     local index="$1"

From 53ecfb98d099a5691cb15657e781c25b46aca868 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Fri, 17 Feb 2023 15:21:20 +0100
Subject: [PATCH 120/144] chore: fix stale code comments

---
 .../src/kafka/emqx_bridge_impl_kafka_producer.erl               | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
index ac98209ed..cff17b7de 100644
--- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
+++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl
@@ -228,7 +228,7 @@ render_timestamp(Template, Message) ->
 %% Wolff producer never gives up retrying
 %% so there can only be 'ok' results.
 on_kafka_ack(_Partition, Offset, {ReplyFn, Args}) when is_integer(Offset) ->
-    %% the ReplyFn is emqx_resource_worker:handle_async_reply/2
+    %% the ReplyFn is emqx_resource_buffer_worker:handle_async_reply/2
     apply(ReplyFn, Args ++ [ok]);
 on_kafka_ack(_Partition, buffer_overflow_discarded, _Callback) ->
     %% wolff should bump the dropped_queue_full counter

From 9316690c29a595de983dd53715e758967f991470 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Sat, 18 Feb 2023 13:09:03 +0100
Subject: [PATCH 121/144] fix(schema): binary string for default values

A lot of the string value fields had default value defined in
schema as list-string rather than binary-string.
This caused the generated schema dump (in JSON format)
to have raw_default field as an integer array.
---
 .../emqx_limiter/src/emqx_limiter_schema.erl  | 20 ++---
 apps/emqx/src/emqx_schema.erl                 | 88 +++++++++----------
 apps/emqx_authn/src/emqx_authn.app.src        |  2 +-
 .../src/simple_authn/emqx_authn_mysql.erl     |  2 +-
 apps/emqx_authz/src/emqx_authz.app.src        |  2 +-
 apps/emqx_authz/src/emqx_authz_api_schema.erl |  2 +-
 apps/emqx_authz/src/emqx_authz_schema.erl     |  2 +-
 apps/emqx_conf/src/emqx_conf_schema.erl       | 72 +++++++--------
 .../src/emqx_connector_http.erl               |  2 +-
 .../src/mqtt/emqx_connector_mqtt_schema.erl   |  4 +-
 .../src/emqx_dashboard_schema.erl             | 10 +--
 .../test/emqx_swagger_remote_schema.erl       |  6 +-
 .../test/emqx_swagger_requestBody_SUITE.erl   |  2 +-
 .../test/emqx_swagger_response_SUITE.erl      |  2 +-
 apps/emqx_exhook/src/emqx_exhook.app.src      |  2 +-
 apps/emqx_exhook/src/emqx_exhook_api.erl      |  4 +-
 apps/emqx_exhook/src/emqx_exhook_schema.erl   |  4 +-
 apps/emqx_gateway/src/emqx_gateway_schema.erl | 10 +--
 apps/emqx_plugins/src/emqx_plugins.app.src    |  2 +-
 apps/emqx_plugins/src/emqx_plugins_schema.erl |  4 +-
 .../src/emqx_prometheus_schema.erl            |  4 +-
 .../src/emqx_retainer_schema.erl              |  6 +-
 .../src/emqx_rule_engine_schema.erl           |  2 +-
 .../emqx_slow_subs/src/emqx_slow_subs.app.src |  2 +-
 .../src/emqx_slow_subs_schema.erl             |  4 +-
 apps/emqx_statsd/src/emqx_statsd_api.erl      |  6 +-
 apps/emqx_statsd/src/emqx_statsd_schema.erl   |  4 +-
 .../src/emqx_ee_bridge_gcp_pubsub.erl         |  4 +-
 .../src/emqx_ee_bridge_kafka.erl              | 24 ++---
 .../emqx_license/src/emqx_license_schema.erl  |  4 +-
 30 files changed, 151 insertions(+), 151 deletions(-)

diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl
index fa67e1977..ddfc55f7a 100644
--- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl
+++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl
@@ -110,11 +110,11 @@ fields(limiter) ->
         ];
 fields(node_opts) ->
     [
-        {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})},
+        {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})},
         {burst,
             ?HOCON(burst_rate(), #{
                 desc => ?DESC(burst),
-                default => 0
+                default => <<"0">>
             })}
     ];
 fields(client_fields) ->
@@ -128,14 +128,14 @@ fields(client_fields) ->
     ];
 fields(bucket_opts) ->
     [
-        {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})},
-        {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => "infinity"})},
-        {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})}
+        {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})},
+        {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => <<"infinity">>})},
+        {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})}
     ];
 fields(client_opts) ->
     [
-        {rate, ?HOCON(rate(), #{default => "infinity", desc => ?DESC(rate)})},
-        {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})},
+        {rate, ?HOCON(rate(), #{default => <<"infinity">>, desc => ?DESC(rate)})},
+        {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})},
         %% low_watermark add for emqx_channel and emqx_session
         %% both modules consume first and then check
         %% so we need to use this value to prevent excessive consumption
@@ -145,13 +145,13 @@ fields(client_opts) ->
                 initial(),
                 #{
                     desc => ?DESC(low_watermark),
-                    default => "0"
+                    default => <<"0">>
                 }
             )},
         {capacity,
             ?HOCON(capacity(), #{
                 desc => ?DESC(client_bucket_capacity),
-                default => "infinity"
+                default => <<"infinity">>
             })},
         {divisible,
             ?HOCON(
@@ -166,7 +166,7 @@ fields(client_opts) ->
                 emqx_schema:duration(),
                 #{
                     desc => ?DESC(max_retry_time),
-                    default => "10s"
+                    default => <<"10s">>
                 }
             )},
         {failure_strategy,
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 008aa23c9..8d24e6937 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -268,7 +268,7 @@ fields("persistent_session_store") ->
             sc(
                 duration(),
                 #{
-                    default => "1h",
+                    default => <<"1h">>,
                     desc => ?DESC(persistent_session_store_max_retain_undelivered)
                 }
             )},
@@ -276,7 +276,7 @@ fields("persistent_session_store") ->
             sc(
                 duration(),
                 #{
-                    default => "1h",
+                    default => <<"1h">>,
                     desc => ?DESC(persistent_session_store_message_gc_interval)
                 }
             )},
@@ -284,7 +284,7 @@ fields("persistent_session_store") ->
             sc(
                 duration(),
                 #{
-                    default => "1m",
+                    default => <<"1m">>,
                     desc => ?DESC(persistent_session_store_session_message_gc_interval)
                 }
             )}
@@ -352,7 +352,7 @@ fields("authz_cache") ->
             sc(
                 duration(),
                 #{
-                    default => "1m",
+                    default => <<"1m">>,
                     desc => ?DESC(fields_cache_ttl)
                 }
             )}
@@ -363,7 +363,7 @@ fields("mqtt") ->
             sc(
                 hoconsc:union([infinity, duration()]),
                 #{
-                    default => "15s",
+                    default => <<"15s">>,
                     desc => ?DESC(mqtt_idle_timeout)
                 }
             )},
@@ -371,7 +371,7 @@ fields("mqtt") ->
             sc(
                 bytesize(),
                 #{
-                    default => "1MB",
+                    default => <<"1MB">>,
                     desc => ?DESC(mqtt_max_packet_size)
                 }
             )},
@@ -507,7 +507,7 @@ fields("mqtt") ->
             sc(
                 duration(),
                 #{
-                    default => "30s",
+                    default => <<"30s">>,
                     desc => ?DESC(mqtt_retry_interval)
                 }
             )},
@@ -523,7 +523,7 @@ fields("mqtt") ->
             sc(
                 duration(),
                 #{
-                    default => "300s",
+                    default => <<"300s">>,
                     desc => ?DESC(mqtt_await_rel_timeout)
                 }
             )},
@@ -531,7 +531,7 @@ fields("mqtt") ->
             sc(
                 duration(),
                 #{
-                    default => "2h",
+                    default => <<"2h">>,
                     desc => ?DESC(mqtt_session_expiry_interval)
                 }
             )},
@@ -617,7 +617,7 @@ fields("flapping_detect") ->
             sc(
                 duration(),
                 #{
-                    default => "1m",
+                    default => <<"1m">>,
                     desc => ?DESC(flapping_detect_window_time)
                 }
             )},
@@ -625,7 +625,7 @@ fields("flapping_detect") ->
             sc(
                 duration(),
                 #{
-                    default => "5m",
+                    default => <<"5m">>,
                     desc => ?DESC(flapping_detect_ban_time)
                 }
             )}
@@ -652,7 +652,7 @@ fields("force_shutdown") ->
             sc(
                 wordsize(),
                 #{
-                    default => "32MB",
+                    default => <<"32MB">>,
                     desc => ?DESC(force_shutdown_max_heap_size),
                     validator => fun ?MODULE:validate_heap_size/1
                 }
@@ -715,7 +715,7 @@ fields("conn_congestion") ->
             sc(
                 duration(),
                 #{
-                    default => "1m",
+                    default => <<"1m">>,
                     desc => ?DESC(conn_congestion_min_alarm_sustain_duration)
                 }
             )}
@@ -739,7 +739,7 @@ fields("force_gc") ->
             sc(
                 bytesize(),
                 #{
-                    default => "16MB",
+                    default => <<"16MB">>,
                     desc => ?DESC(force_gc_bytes)
                 }
             )}
@@ -874,7 +874,7 @@ fields("mqtt_quic_listener") ->
             sc(
                 duration_ms(),
                 #{
-                    default => "10s",
+                    default => <<"10s">>,
                     desc => ?DESC(fields_mqtt_quic_listener_handshake_idle_timeout)
                 }
             )},
@@ -901,7 +901,7 @@ fields("ws_opts") ->
             sc(
                 string(),
                 #{
-                    default => "/mqtt",
+                    default => <<"/mqtt">>,
                     desc => ?DESC(fields_ws_opts_mqtt_path)
                 }
             )},
@@ -925,7 +925,7 @@ fields("ws_opts") ->
             sc(
                 duration(),
                 #{
-                    default => "7200s",
+                    default => <<"7200s">>,
                     desc => ?DESC(fields_ws_opts_idle_timeout)
                 }
             )},
@@ -949,7 +949,7 @@ fields("ws_opts") ->
             sc(
                 comma_separated_list(),
                 #{
-                    default => "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5",
+                    default => <<"mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5">>,
                     desc => ?DESC(fields_ws_opts_supported_subprotocols)
                 }
             )},
@@ -981,7 +981,7 @@ fields("ws_opts") ->
             sc(
                 string(),
                 #{
-                    default => "x-forwarded-for",
+                    default => <<"x-forwarded-for">>,
                     desc => ?DESC(fields_ws_opts_proxy_address_header)
                 }
             )},
@@ -989,7 +989,7 @@ fields("ws_opts") ->
             sc(
                 string(),
                 #{
-                    default => "x-forwarded-port",
+                    default => <<"x-forwarded-port">>,
                     desc => ?DESC(fields_ws_opts_proxy_port_header)
                 }
             )},
@@ -1021,7 +1021,7 @@ fields("tcp_opts") ->
             sc(
                 duration(),
                 #{
-                    default => "15s",
+                    default => <<"15s">>,
                     desc => ?DESC(fields_tcp_opts_send_timeout)
                 }
             )},
@@ -1062,7 +1062,7 @@ fields("tcp_opts") ->
             sc(
                 bytesize(),
                 #{
-                    default => "1MB",
+                    default => <<"1MB">>,
                     desc => ?DESC(fields_tcp_opts_high_watermark)
                 }
             )},
@@ -1275,7 +1275,7 @@ fields("sys_topics") ->
             sc(
                 hoconsc:union([disabled, duration()]),
                 #{
-                    default => "1m",
+                    default => <<"1m">>,
                     desc => ?DESC(sys_msg_interval)
                 }
             )},
@@ -1283,7 +1283,7 @@ fields("sys_topics") ->
             sc(
                 hoconsc:union([disabled, duration()]),
                 #{
-                    default => "30s",
+                    default => <<"30s">>,
                     desc => ?DESC(sys_heartbeat_interval)
                 }
             )},
@@ -1352,7 +1352,7 @@ fields("sysmon_vm") ->
             sc(
                 duration(),
                 #{
-                    default => "30s",
+                    default => <<"30s">>,
                     desc => ?DESC(sysmon_vm_process_check_interval)
                 }
             )},
@@ -1360,7 +1360,7 @@ fields("sysmon_vm") ->
             sc(
                 percent(),
                 #{
-                    default => "80%",
+                    default => <<"80%">>,
                     desc => ?DESC(sysmon_vm_process_high_watermark)
                 }
             )},
@@ -1368,7 +1368,7 @@ fields("sysmon_vm") ->
             sc(
                 percent(),
                 #{
-                    default => "60%",
+                    default => <<"60%">>,
                     desc => ?DESC(sysmon_vm_process_low_watermark)
                 }
             )},
@@ -1384,7 +1384,7 @@ fields("sysmon_vm") ->
             sc(
                 hoconsc:union([disabled, duration()]),
                 #{
-                    default => "240ms",
+                    default => <<"240ms">>,
                     desc => ?DESC(sysmon_vm_long_schedule)
                 }
             )},
@@ -1392,7 +1392,7 @@ fields("sysmon_vm") ->
             sc(
                 hoconsc:union([disabled, bytesize()]),
                 #{
-                    default => "32MB",
+                    default => <<"32MB">>,
                     desc => ?DESC(sysmon_vm_large_heap)
                 }
             )},
@@ -1419,7 +1419,7 @@ fields("sysmon_os") ->
             sc(
                 duration(),
                 #{
-                    default => "60s",
+                    default => <<"60s">>,
                     desc => ?DESC(sysmon_os_cpu_check_interval)
                 }
             )},
@@ -1427,7 +1427,7 @@ fields("sysmon_os") ->
             sc(
                 percent(),
                 #{
-                    default => "80%",
+                    default => <<"80%">>,
                     desc => ?DESC(sysmon_os_cpu_high_watermark)
                 }
             )},
@@ -1435,7 +1435,7 @@ fields("sysmon_os") ->
             sc(
                 percent(),
                 #{
-                    default => "60%",
+                    default => <<"60%">>,
                     desc => ?DESC(sysmon_os_cpu_low_watermark)
                 }
             )},
@@ -1443,7 +1443,7 @@ fields("sysmon_os") ->
             sc(
                 hoconsc:union([disabled, duration()]),
                 #{
-                    default => "60s",
+                    default => <<"60s">>,
                     desc => ?DESC(sysmon_os_mem_check_interval)
                 }
             )},
@@ -1451,7 +1451,7 @@ fields("sysmon_os") ->
             sc(
                 percent(),
                 #{
-                    default => "70%",
+                    default => <<"70%">>,
                     desc => ?DESC(sysmon_os_sysmem_high_watermark)
                 }
             )},
@@ -1459,7 +1459,7 @@ fields("sysmon_os") ->
             sc(
                 percent(),
                 #{
-                    default => "5%",
+                    default => <<"5%">>,
                     desc => ?DESC(sysmon_os_procmem_high_watermark)
                 }
             )}
@@ -1480,7 +1480,7 @@ fields("sysmon_top") ->
                 emqx_schema:duration(),
                 #{
                     mapping => "system_monitor.top_sample_interval",
-                    default => "2s",
+                    default => <<"2s">>,
                     desc => ?DESC(sysmon_top_sample_interval)
                 }
             )},
@@ -1499,7 +1499,7 @@ fields("sysmon_top") ->
                 #{
                     mapping => "system_monitor.db_hostname",
                     desc => ?DESC(sysmon_top_db_hostname),
-                    default => ""
+                    default => <<>>
                 }
             )},
         {"db_port",
@@ -1516,7 +1516,7 @@ fields("sysmon_top") ->
                 string(),
                 #{
                     mapping => "system_monitor.db_username",
-                    default => "system_monitor",
+                    default => <<"system_monitor">>,
                     desc => ?DESC(sysmon_top_db_username)
                 }
             )},
@@ -1525,7 +1525,7 @@ fields("sysmon_top") ->
                 binary(),
                 #{
                     mapping => "system_monitor.db_password",
-                    default => "system_monitor_password",
+                    default => <<"system_monitor_password">>,
                     desc => ?DESC(sysmon_top_db_password),
                     converter => fun password_converter/2,
                     sensitive => true
@@ -1536,7 +1536,7 @@ fields("sysmon_top") ->
                 string(),
                 #{
                     mapping => "system_monitor.db_name",
-                    default => "postgres",
+                    default => <<"postgres">>,
                     desc => ?DESC(sysmon_top_db_name)
                 }
             )}
@@ -1566,7 +1566,7 @@ fields("alarm") ->
             sc(
                 duration(),
                 #{
-                    default => "24h",
+                    default => <<"24h">>,
                     example => "24h",
                     desc => ?DESC(alarm_validity_period)
                 }
@@ -1605,7 +1605,7 @@ mqtt_listener(Bind) ->
                     duration(),
                     #{
                         desc => ?DESC(mqtt_listener_proxy_protocol_timeout),
-                        default => "3s"
+                        default => <<"3s">>
                     }
                 )},
             {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication(listener)}
@@ -1956,7 +1956,7 @@ common_ssl_opts_schema(Defaults) ->
             sc(
                 duration(),
                 #{
-                    default => Df("hibernate_after", "5s"),
+                    default => Df("hibernate_after", <<"5s">>),
                     desc => ?DESC(common_ssl_opts_schema_hibernate_after)
                 }
             )}
@@ -2006,7 +2006,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) ->
                 sc(
                     duration(),
                     #{
-                        default => Df("handshake_timeout", "15s"),
+                        default => Df("handshake_timeout", <<"15s">>),
                         desc => ?DESC(server_ssl_opts_schema_handshake_timeout)
                     }
                 )}
diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src
index 0b5b0dedc..7fbdf787a 100644
--- a/apps/emqx_authn/src/emqx_authn.app.src
+++ b/apps/emqx_authn/src/emqx_authn.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authn, [
     {description, "EMQX Authentication"},
-    {vsn, "0.1.13"},
+    {vsn, "0.1.14"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
     {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]},
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
index 0e6eeb6af..bedd169e2 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
@@ -74,7 +74,7 @@ query(_) -> undefined.
 
 query_timeout(type) -> emqx_schema:duration_ms();
 query_timeout(desc) -> ?DESC(?FUNCTION_NAME);
-query_timeout(default) -> "5s";
+query_timeout(default) -> <<"5s">>;
 query_timeout(_) -> undefined.
 
 %%------------------------------------------------------------------------------
diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src
index 3fea50147..f016db09a 100644
--- a/apps/emqx_authz/src/emqx_authz.app.src
+++ b/apps/emqx_authz/src/emqx_authz.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.13"},
+    {vsn, "0.1.14"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [
diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl
index 44ec0d28a..4adada182 100644
--- a/apps/emqx_authz/src/emqx_authz_api_schema.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl
@@ -108,7 +108,7 @@ authz_http_common_fields() ->
                 })},
             {request_timeout,
                 mk_duration("Request timeout", #{
-                    required => false, default => "30s", desc => ?DESC(request_timeout)
+                    required => false, default => <<"30s">>, desc => ?DESC(request_timeout)
                 })}
         ] ++
         maps:to_list(
diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl
index 5527c26d6..e68ab3a50 100644
--- a/apps/emqx_authz/src/emqx_authz_schema.erl
+++ b/apps/emqx_authz/src/emqx_authz_schema.erl
@@ -223,7 +223,7 @@ http_common_fields() ->
         {url, fun url/1},
         {request_timeout,
             mk_duration("Request timeout", #{
-                required => false, default => "30s", desc => ?DESC(request_timeout)
+                required => false, default => <<"30s">>, desc => ?DESC(request_timeout)
             })},
         {body, ?HOCON(map(), #{required => false, desc => ?DESC(body)})}
     ] ++
diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl
index 9793e00d0..4862be5fe 100644
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@@ -145,7 +145,7 @@ fields("cluster") ->
                 emqx_schema:duration(),
                 #{
                     mapping => "ekka.cluster_autoclean",
-                    default => "5m",
+                    default => <<"5m">>,
                     desc => ?DESC(cluster_autoclean),
                     'readOnly' => true
                 }
@@ -214,7 +214,7 @@ fields(cluster_mcast) ->
             sc(
                 string(),
                 #{
-                    default => "239.192.0.1",
+                    default => <<"239.192.0.1">>,
                     desc => ?DESC(cluster_mcast_addr),
                     'readOnly' => true
                 }
@@ -232,7 +232,7 @@ fields(cluster_mcast) ->
             sc(
                 string(),
                 #{
-                    default => "0.0.0.0",
+                    default => <<"0.0.0.0">>,
                     desc => ?DESC(cluster_mcast_iface),
                     'readOnly' => true
                 }
@@ -259,7 +259,7 @@ fields(cluster_mcast) ->
             sc(
                 emqx_schema:bytesize(),
                 #{
-                    default => "16KB",
+                    default => <<"16KB">>,
                     desc => ?DESC(cluster_mcast_sndbuf),
                     'readOnly' => true
                 }
@@ -268,7 +268,7 @@ fields(cluster_mcast) ->
             sc(
                 emqx_schema:bytesize(),
                 #{
-                    default => "16KB",
+                    default => <<"16KB">>,
                     desc => ?DESC(cluster_mcast_recbuf),
                     'readOnly' => true
                 }
@@ -277,7 +277,7 @@ fields(cluster_mcast) ->
             sc(
                 emqx_schema:bytesize(),
                 #{
-                    default => "32KB",
+                    default => <<"32KB">>,
                     desc => ?DESC(cluster_mcast_buffer),
                     'readOnly' => true
                 }
@@ -289,7 +289,7 @@ fields(cluster_dns) ->
             sc(
                 string(),
                 #{
-                    default => "localhost",
+                    default => <<"localhost">>,
                     desc => ?DESC(cluster_dns_name),
                     'readOnly' => true
                 }
@@ -318,7 +318,7 @@ fields(cluster_etcd) ->
             sc(
                 string(),
                 #{
-                    default => "emqxcl",
+                    default => <<"emqxcl">>,
                     desc => ?DESC(cluster_etcd_prefix),
                     'readOnly' => true
                 }
@@ -327,7 +327,7 @@ fields(cluster_etcd) ->
             sc(
                 emqx_schema:duration(),
                 #{
-                    default => "1m",
+                    default => <<"1m">>,
                     'readOnly' => true,
                     desc => ?DESC(cluster_etcd_node_ttl)
                 }
@@ -347,7 +347,7 @@ fields(cluster_k8s) ->
             sc(
                 string(),
                 #{
-                    default => "http://10.110.111.204:8080",
+                    default => <<"http://10.110.111.204:8080">>,
                     desc => ?DESC(cluster_k8s_apiserver),
                     'readOnly' => true
                 }
@@ -356,7 +356,7 @@ fields(cluster_k8s) ->
             sc(
                 string(),
                 #{
-                    default => "emqx",
+                    default => <<"emqx">>,
                     desc => ?DESC(cluster_k8s_service_name),
                     'readOnly' => true
                 }
@@ -374,7 +374,7 @@ fields(cluster_k8s) ->
             sc(
                 string(),
                 #{
-                    default => "default",
+                    default => <<"default">>,
                     desc => ?DESC(cluster_k8s_namespace),
                     'readOnly' => true
                 }
@@ -383,7 +383,7 @@ fields(cluster_k8s) ->
             sc(
                 string(),
                 #{
-                    default => "pod.local",
+                    default => <<"pod.local">>,
                     'readOnly' => true,
                     desc => ?DESC(cluster_k8s_suffix)
                 }
@@ -395,7 +395,7 @@ fields("node") ->
             sc(
                 string(),
                 #{
-                    default => "emqx@127.0.0.1",
+                    default => <<"emqx@127.0.0.1">>,
                     'readOnly' => true,
                     desc => ?DESC(node_name)
                 }
@@ -477,7 +477,7 @@ fields("node") ->
                 hoconsc:union([disabled, emqx_schema:duration()]),
                 #{
                     mapping => "emqx_machine.global_gc_interval",
-                    default => "15m",
+                    default => <<"15m">>,
                     desc => ?DESC(node_global_gc_interval),
                     'readOnly' => true
                 }
@@ -497,7 +497,7 @@ fields("node") ->
                 emqx_schema:duration_s(),
                 #{
                     mapping => "vm_args.-env ERL_CRASH_DUMP_SECONDS",
-                    default => "30s",
+                    default => <<"30s">>,
                     desc => ?DESC(node_crash_dump_seconds),
                     'readOnly' => true
                 }
@@ -507,7 +507,7 @@ fields("node") ->
                 emqx_schema:bytesize(),
                 #{
                     mapping => "vm_args.-env ERL_CRASH_DUMP_BYTES",
-                    default => "100MB",
+                    default => <<"100MB">>,
                     desc => ?DESC(node_crash_dump_bytes),
                     'readOnly' => true
                 }
@@ -517,7 +517,7 @@ fields("node") ->
                 emqx_schema:duration_s(),
                 #{
                     mapping => "vm_args.-kernel net_ticktime",
-                    default => "2m",
+                    default => <<"2m">>,
                     'readOnly' => true,
                     desc => ?DESC(node_dist_net_ticktime)
                 }
@@ -624,7 +624,7 @@ fields("cluster_call") ->
                 emqx_schema:duration(),
                 #{
                     desc => ?DESC(cluster_call_retry_interval),
-                    default => "1m"
+                    default => <<"1m">>
                 }
             )},
         {"max_history",
@@ -640,7 +640,7 @@ fields("cluster_call") ->
                 emqx_schema:duration(),
                 #{
                     desc => ?DESC(cluster_call_cleanup_interval),
-                    default => "5m"
+                    default => <<"5m">>
                 }
             )}
     ];
@@ -712,7 +712,7 @@ fields("rpc") ->
                 emqx_schema:duration(),
                 #{
                     mapping => "gen_rpc.connect_timeout",
-                    default => "5s",
+                    default => <<"5s">>,
                     desc => ?DESC(rpc_connect_timeout)
                 }
             )},
@@ -745,7 +745,7 @@ fields("rpc") ->
                 emqx_schema:duration(),
                 #{
                     mapping => "gen_rpc.send_timeout",
-                    default => "5s",
+                    default => <<"5s">>,
                     desc => ?DESC(rpc_send_timeout)
                 }
             )},
@@ -754,7 +754,7 @@ fields("rpc") ->
                 emqx_schema:duration(),
                 #{
                     mapping => "gen_rpc.authentication_timeout",
-                    default => "5s",
+                    default => <<"5s">>,
                     desc => ?DESC(rpc_authentication_timeout)
                 }
             )},
@@ -763,7 +763,7 @@ fields("rpc") ->
                 emqx_schema:duration(),
                 #{
                     mapping => "gen_rpc.call_receive_timeout",
-                    default => "15s",
+                    default => <<"15s">>,
                     desc => ?DESC(rpc_call_receive_timeout)
                 }
             )},
@@ -772,7 +772,7 @@ fields("rpc") ->
                 emqx_schema:duration_s(),
                 #{
                     mapping => "gen_rpc.socket_keepalive_idle",
-                    default => "15m",
+                    default => <<"15m">>,
                     desc => ?DESC(rpc_socket_keepalive_idle)
                 }
             )},
@@ -781,7 +781,7 @@ fields("rpc") ->
                 emqx_schema:duration_s(),
                 #{
                     mapping => "gen_rpc.socket_keepalive_interval",
-                    default => "75s",
+                    default => <<"75s">>,
                     desc => ?DESC(rpc_socket_keepalive_interval)
                 }
             )},
@@ -799,7 +799,7 @@ fields("rpc") ->
                 emqx_schema:bytesize(),
                 #{
                     mapping => "gen_rpc.socket_sndbuf",
-                    default => "1MB",
+                    default => <<"1MB">>,
                     desc => ?DESC(rpc_socket_sndbuf)
                 }
             )},
@@ -808,7 +808,7 @@ fields("rpc") ->
                 emqx_schema:bytesize(),
                 #{
                     mapping => "gen_rpc.socket_recbuf",
-                    default => "1MB",
+                    default => <<"1MB">>,
                     desc => ?DESC(rpc_socket_recbuf)
                 }
             )},
@@ -817,7 +817,7 @@ fields("rpc") ->
                 emqx_schema:bytesize(),
                 #{
                     mapping => "gen_rpc.socket_buffer",
-                    default => "1MB",
+                    default => <<"1MB">>,
                     desc => ?DESC(rpc_socket_buffer)
                 }
             )},
@@ -861,7 +861,7 @@ fields("log_file_handler") ->
             sc(
                 hoconsc:union([infinity, emqx_schema:bytesize()]),
                 #{
-                    default => "50MB",
+                    default => <<"50MB">>,
                     desc => ?DESC("log_file_handler_max_size")
                 }
             )}
@@ -899,7 +899,7 @@ fields("log_overload_kill") ->
             sc(
                 emqx_schema:bytesize(),
                 #{
-                    default => "30MB",
+                    default => <<"30MB">>,
                     desc => ?DESC("log_overload_kill_mem_size")
                 }
             )},
@@ -915,7 +915,7 @@ fields("log_overload_kill") ->
             sc(
                 hoconsc:union([emqx_schema:duration_ms(), infinity]),
                 #{
-                    default => "5s",
+                    default => <<"5s">>,
                     desc => ?DESC("log_overload_kill_restart_after")
                 }
             )}
@@ -942,7 +942,7 @@ fields("log_burst_limit") ->
             sc(
                 emqx_schema:duration(),
                 #{
-                    default => "1s",
+                    default => <<"1s">>,
                     desc => ?DESC("log_burst_limit_window_time")
                 }
             )}
@@ -1092,7 +1092,7 @@ log_handler_common_confs(Enable) ->
             sc(
                 string(),
                 #{
-                    default => "system",
+                    default => <<"system">>,
                     desc => ?DESC("common_handler_time_offset"),
                     validator => fun validate_time_offset/1
                 }
@@ -1169,9 +1169,9 @@ crash_dump_file_default() ->
     case os:getenv("RUNNER_LOG_DIR") of
         false ->
             %% testing, or running emqx app as deps
-            "log/erl_crash.dump";
+            <<"log/erl_crash.dump">>;
         Dir ->
-            [filename:join([Dir, "erl_crash.dump"])]
+            unicode:characters_to_binary(filename:join([Dir, "erl_crash.dump"]), utf8)
     end.
 
 %% utils
diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl
index 7c4a1fcf8..7d91e18b9 100644
--- a/apps/emqx_connector/src/emqx_connector_http.erl
+++ b/apps/emqx_connector/src/emqx_connector_http.erl
@@ -87,7 +87,7 @@ fields(config) ->
             sc(
                 emqx_schema:duration_ms(),
                 #{
-                    default => "15s",
+                    default => <<"15s">>,
                     desc => ?DESC("connect_timeout")
                 }
             )},
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
index 073b75ae8..e08804685 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
@@ -115,12 +115,12 @@ fields("server_configs") ->
                     desc => ?DESC("clean_start")
                 }
             )},
-        {keepalive, mk_duration("MQTT Keepalive.", #{default => "300s"})},
+        {keepalive, mk_duration("MQTT Keepalive.", #{default => <<"300s">>})},
         {retry_interval,
             mk_duration(
                 "Message retry interval. Delay for the MQTT bridge to retry sending the QoS1/QoS2 "
                 "messages in case of ACK not received.",
-                #{default => "15s"}
+                #{default => <<"15s">>}
             )},
         {max_inflight,
             mk(
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl
index ceb2415f8..7df661fb2 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl
@@ -40,7 +40,7 @@ fields("dashboard") ->
             ?HOCON(
                 emqx_schema:duration_s(),
                 #{
-                    default => "10s",
+                    default => <<"10s">>,
                     desc => ?DESC(sample_interval),
                     validator => fun validate_sample_interval/1
                 }
@@ -49,7 +49,7 @@ fields("dashboard") ->
             ?HOCON(
                 emqx_schema:duration(),
                 #{
-                    default => "60m",
+                    default => <<"60m">>,
                     desc => ?DESC(token_expired_time)
                 }
             )},
@@ -141,7 +141,7 @@ common_listener_fields() ->
             ?HOCON(
                 emqx_schema:duration(),
                 #{
-                    default => "10s",
+                    default => <<"10s">>,
                     desc => ?DESC(send_timeout)
                 }
             )},
@@ -206,14 +206,14 @@ desc(_) ->
     undefined.
 
 default_username(type) -> binary();
-default_username(default) -> "admin";
+default_username(default) -> <<"admin">>;
 default_username(required) -> true;
 default_username(desc) -> ?DESC(default_username);
 default_username('readOnly') -> true;
 default_username(_) -> undefined.
 
 default_password(type) -> binary();
-default_password(default) -> "public";
+default_password(default) -> <<"public">>;
 default_password(required) -> true;
 default_password('readOnly') -> true;
 default_password(sensitive) -> true;
diff --git a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl
index a797d3b43..c2266ad5b 100644
--- a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl
+++ b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl
@@ -32,8 +32,8 @@ fields("root") ->
             )},
         {default_username, fun default_username/1},
         {default_password, fun default_password/1},
-        {sample_interval, mk(emqx_schema:duration_s(), #{default => "10s"})},
-        {token_expired_time, mk(emqx_schema:duration(), #{default => "30m"})}
+        {sample_interval, mk(emqx_schema:duration_s(), #{default => <<"10s">>})},
+        {token_expired_time, mk(emqx_schema:duration(), #{default => <<"30m">>})}
     ];
 fields("ref1") ->
     [
@@ -52,7 +52,7 @@ fields("ref3") ->
     ].
 
 default_username(type) -> string();
-default_username(default) -> "admin";
+default_username(default) -> <<"admin">>;
 default_username(required) -> true;
 default_username(_) -> undefined.
 
diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl
index d17725e80..979d01c77 100644
--- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl
@@ -790,7 +790,7 @@ to_schema(Body) ->
 
 fields(good_ref) ->
     [
-        {'webhook-host', mk(emqx_schema:ip_port(), #{default => "127.0.0.1:80"})},
+        {'webhook-host', mk(emqx_schema:ip_port(), #{default => <<"127.0.0.1:80">>})},
         {log_dir, mk(emqx_schema:file(), #{example => "var/log/emqx"})},
         {tag, mk(binary(), #{desc => <<"tag">>})}
     ];
diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl
index 346f4ef71..c9cfba254 100644
--- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl
@@ -689,7 +689,7 @@ to_schema(Object) ->
 
 fields(good_ref) ->
     [
-        {'webhook-host', mk(emqx_schema:ip_port(), #{default => "127.0.0.1:80"})},
+        {'webhook-host', mk(emqx_schema:ip_port(), #{default => <<"127.0.0.1:80">>})},
         {log_dir, mk(emqx_schema:file(), #{example => "var/log/emqx"})},
         {tag, mk(binary(), #{desc => <<"tag">>})}
     ];
diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src
index d81819c98..04e0a57db 100644
--- a/apps/emqx_exhook/src/emqx_exhook.app.src
+++ b/apps/emqx_exhook/src/emqx_exhook.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_exhook, [
     {description, "EMQX Extension for Hook"},
-    {vsn, "5.0.9"},
+    {vsn, "5.0.10"},
     {modules, []},
     {registered, []},
     {mod, {emqx_exhook_app, []}},
diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl
index 4d7de2866..bcfc68269 100644
--- a/apps/emqx_exhook/src/emqx_exhook_api.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_api.erl
@@ -229,9 +229,9 @@ server_conf_schema() ->
             name => "default",
             enable => true,
             url => <<"http://127.0.0.1:8081">>,
-            request_timeout => "5s",
+            request_timeout => <<"5s">>,
             failed_action => deny,
-            auto_reconnect => "60s",
+            auto_reconnect => <<"60s">>,
             pool_size => 8,
             ssl => SSL
         }
diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl
index ce79dddac..07373288d 100644
--- a/apps/emqx_exhook/src/emqx_exhook_schema.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl
@@ -63,7 +63,7 @@ fields(server) ->
             })},
         {request_timeout,
             ?HOCON(emqx_schema:duration(), #{
-                default => "5s",
+                default => <<"5s">>,
                 desc => ?DESC(request_timeout)
             })},
         {failed_action, failed_action()},
@@ -74,7 +74,7 @@ fields(server) ->
             })},
         {auto_reconnect,
             ?HOCON(hoconsc:union([false, emqx_schema:duration()]), #{
-                default => "60s",
+                default => <<"60s">>,
                 desc => ?DESC(auto_reconnect)
             })},
         {pool_size,
diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl
index 4ea845ea1..2034a40eb 100644
--- a/apps/emqx_gateway/src/emqx_gateway_schema.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl
@@ -267,7 +267,7 @@ fields(lwm2m) ->
             sc(
                 duration(),
                 #{
-                    default => "15s",
+                    default => <<"15s">>,
                     desc => ?DESC(lwm2m_lifetime_min)
                 }
             )},
@@ -275,7 +275,7 @@ fields(lwm2m) ->
             sc(
                 duration(),
                 #{
-                    default => "86400s",
+                    default => <<"86400s">>,
                     desc => ?DESC(lwm2m_lifetime_max)
                 }
             )},
@@ -283,7 +283,7 @@ fields(lwm2m) ->
             sc(
                 duration_s(),
                 #{
-                    default => "22s",
+                    default => <<"22s">>,
                     desc => ?DESC(lwm2m_qmode_time_window)
                 }
             )},
@@ -624,7 +624,7 @@ mountpoint(Default) ->
     sc(
         binary(),
         #{
-            default => Default,
+            default => iolist_to_binary(Default),
             desc => ?DESC(gateway_common_mountpoint)
         }
     ).
@@ -707,7 +707,7 @@ proxy_protocol_opts() ->
             sc(
                 duration(),
                 #{
-                    default => "15s",
+                    default => <<"15s">>,
                     desc => ?DESC(tcp_listener_proxy_protocol_timeout)
                 }
             )}
diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src
index de56099ba..ed893c80d 100644
--- a/apps/emqx_plugins/src/emqx_plugins.app.src
+++ b/apps/emqx_plugins/src/emqx_plugins.app.src
@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_plugins, [
     {description, "EMQX Plugin Management"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {modules, []},
     {mod, {emqx_plugins_app, []}},
     {applications, [kernel, stdlib, emqx]},
diff --git a/apps/emqx_plugins/src/emqx_plugins_schema.erl b/apps/emqx_plugins/src/emqx_plugins_schema.erl
index 8b3cca8fd..9d9d045de 100644
--- a/apps/emqx_plugins/src/emqx_plugins_schema.erl
+++ b/apps/emqx_plugins/src/emqx_plugins_schema.erl
@@ -78,11 +78,11 @@ states(_) -> undefined.
 install_dir(type) -> string();
 install_dir(required) -> false;
 %% runner's root dir
-install_dir(default) -> "plugins";
+install_dir(default) -> <<"plugins">>;
 install_dir(T) when T =/= desc -> undefined;
 install_dir(desc) -> ?DESC(install_dir).
 
 check_interval(type) -> emqx_schema:duration();
-check_interval(default) -> "5s";
+check_interval(default) -> <<"5s">>;
 check_interval(T) when T =/= desc -> undefined;
 check_interval(desc) -> ?DESC(check_interval).
diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl
index fcda5dea0..6ced0bf42 100644
--- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl
+++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl
@@ -40,7 +40,7 @@ fields("prometheus") ->
             ?HOCON(
                 string(),
                 #{
-                    default => "http://127.0.0.1:9091",
+                    default => <<"http://127.0.0.1:9091">>,
                     required => true,
                     validator => fun ?MODULE:validate_push_gateway_server/1,
                     desc => ?DESC(push_gateway_server)
@@ -50,7 +50,7 @@ fields("prometheus") ->
             ?HOCON(
                 emqx_schema:duration_ms(),
                 #{
-                    default => "15s",
+                    default => <<"15s">>,
                     required => true,
                     desc => ?DESC(interval)
                 }
diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl
index 472ecc284..dbe1ad9d5 100644
--- a/apps/emqx_retainer/src/emqx_retainer_schema.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl
@@ -41,13 +41,13 @@ fields("retainer") ->
             sc(
                 emqx_schema:duration_ms(),
                 msg_expiry_interval,
-                "0s"
+                <<"0s">>
             )},
         {msg_clear_interval,
             sc(
                 emqx_schema:duration_ms(),
                 msg_clear_interval,
-                "0s"
+                <<"0s">>
             )},
         {flow_control,
             sc(
@@ -59,7 +59,7 @@ fields("retainer") ->
             sc(
                 emqx_schema:bytesize(),
                 max_payload_size,
-                "1MB"
+                <<"1MB">>
             )},
         {stop_publish_clear_msg,
             sc(
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
index d6913cbc6..2281eea53 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
@@ -51,7 +51,7 @@ fields("rule_engine") ->
             ?HOCON(
                 emqx_schema:duration_ms(),
                 #{
-                    default => "10s",
+                    default => <<"10s">>,
                     desc => ?DESC("rule_engine_jq_function_default_timeout")
                 }
             )},
diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src
index 866655b61..170a4bb02 100644
--- a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src
+++ b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src
@@ -1,7 +1,7 @@
 {application, emqx_slow_subs, [
     {description, "EMQX Slow Subscribers Statistics"},
     % strict semver, bump manually!
-    {vsn, "1.0.2"},
+    {vsn, "1.0.3"},
     {modules, []},
     {registered, [emqx_slow_subs_sup]},
     {applications, [kernel, stdlib, emqx]},
diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl
index 8ae015ae4..9e9e6488a 100644
--- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl
+++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl
@@ -30,13 +30,13 @@ fields("slow_subs") ->
         {threshold,
             sc(
                 emqx_schema:duration_ms(),
-                "500ms",
+                <<"500ms">>,
                 threshold
             )},
         {expire_interval,
             sc(
                 emqx_schema:duration_ms(),
-                "300s",
+                <<"300s">>,
                 expire_interval
             )},
         {top_k_num,
diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl
index b1b3601aa..e65c93432 100644
--- a/apps/emqx_statsd/src/emqx_statsd_api.erl
+++ b/apps/emqx_statsd/src/emqx_statsd_api.erl
@@ -77,9 +77,9 @@ statsd_config_schema() ->
 statsd_example() ->
     #{
         enable => true,
-        flush_time_interval => "30s",
-        sample_time_interval => "30s",
-        server => "127.0.0.1:8125",
+        flush_time_interval => <<"30s">>,
+        sample_time_interval => <<"30s">>,
+        server => <<"127.0.0.1:8125">>,
         tags => #{}
     }.
 
diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl
index 1e5aa6e5f..e44f94954 100644
--- a/apps/emqx_statsd/src/emqx_statsd_schema.erl
+++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl
@@ -61,12 +61,12 @@ server() ->
     emqx_schema:servers_sc(Meta, ?SERVER_PARSE_OPTS).
 
 sample_interval(type) -> emqx_schema:duration_ms();
-sample_interval(default) -> "30s";
+sample_interval(default) -> <<"30s">>;
 sample_interval(desc) -> ?DESC(?FUNCTION_NAME);
 sample_interval(_) -> undefined.
 
 flush_interval(type) -> emqx_schema:duration_ms();
-flush_interval(default) -> "30s";
+flush_interval(default) -> <<"30s">>;
 flush_interval(desc) -> ?DESC(?FUNCTION_NAME);
 flush_interval(_) -> undefined.
 
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl
index 1bee9e789..e00483839 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl
@@ -50,7 +50,7 @@ fields(bridge_config) ->
             sc(
                 emqx_schema:duration_ms(),
                 #{
-                    default => "15s",
+                    default => <<"15s">>,
                     desc => ?DESC("connect_timeout")
                 }
             )},
@@ -84,7 +84,7 @@ fields(bridge_config) ->
                 emqx_schema:duration_ms(),
                 #{
                     required => false,
-                    default => "15s",
+                    default => <<"15s">>,
                     desc => ?DESC("request_timeout")
                 }
             )},
diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
index e694f6c15..3983b235c 100644
--- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
+++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl
@@ -84,20 +84,20 @@ fields("config") ->
             )},
         {connect_timeout,
             mk(emqx_schema:duration_ms(), #{
-                default => "5s",
+                default => <<"5s">>,
                 desc => ?DESC(connect_timeout)
             })},
         {min_metadata_refresh_interval,
             mk(
                 emqx_schema:duration_ms(),
                 #{
-                    default => "3s",
+                    default => <<"3s">>,
                     desc => ?DESC(min_metadata_refresh_interval)
                 }
             )},
         {metadata_request_timeout,
             mk(emqx_schema:duration_ms(), #{
-                default => "5s",
+                default => <<"5s">>,
                 desc => ?DESC(metadata_request_timeout)
             })},
         {authentication,
@@ -141,12 +141,12 @@ fields(socket_opts) ->
         {sndbuf,
             mk(
                 emqx_schema:bytesize(),
-                #{default => "1024KB", desc => ?DESC(socket_send_buffer)}
+                #{default => <<"1024KB">>, desc => ?DESC(socket_send_buffer)}
             )},
         {recbuf,
             mk(
                 emqx_schema:bytesize(),
-                #{default => "1024KB", desc => ?DESC(socket_receive_buffer)}
+                #{default => <<"1024KB">>, desc => ?DESC(socket_receive_buffer)}
             )},
         {nodelay,
             mk(
@@ -170,7 +170,7 @@ fields(producer_kafka_opts) ->
         {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})},
         {message, mk(ref(kafka_message), #{required => false, desc => ?DESC(kafka_message)})},
         {max_batch_bytes,
-            mk(emqx_schema:bytesize(), #{default => "896KB", desc => ?DESC(max_batch_bytes)})},
+            mk(emqx_schema:bytesize(), #{default => <<"896KB">>, desc => ?DESC(max_batch_bytes)})},
         {compression,
             mk(enum([no_compression, snappy, gzip]), #{
                 default => no_compression, desc => ?DESC(compression)
@@ -192,7 +192,7 @@ fields(producer_kafka_opts) ->
             mk(
                 emqx_schema:duration_s(),
                 #{
-                    default => "60s",
+                    default => <<"60s">>,
                     desc => ?DESC(partition_count_refresh_interval)
                 }
             )},
@@ -212,11 +212,11 @@ fields(producer_kafka_opts) ->
     ];
 fields(kafka_message) ->
     [
-        {key, mk(string(), #{default => "${.clientid}", desc => ?DESC(kafka_message_key)})},
-        {value, mk(string(), #{default => "${.}", desc => ?DESC(kafka_message_value)})},
+        {key, mk(string(), #{default => <<"${.clientid}">>, desc => ?DESC(kafka_message_key)})},
+        {value, mk(string(), #{default => <<"${.}">>, desc => ?DESC(kafka_message_value)})},
         {timestamp,
             mk(string(), #{
-                default => "${.timestamp}", desc => ?DESC(kafka_message_timestamp)
+                default => <<"${.timestamp}">>, desc => ?DESC(kafka_message_timestamp)
             })}
     ];
 fields(producer_buffer) ->
@@ -229,12 +229,12 @@ fields(producer_buffer) ->
         {per_partition_limit,
             mk(
                 emqx_schema:bytesize(),
-                #{default => "2GB", desc => ?DESC(buffer_per_partition_limit)}
+                #{default => <<"2GB">>, desc => ?DESC(buffer_per_partition_limit)}
             )},
         {segment_bytes,
             mk(
                 emqx_schema:bytesize(),
-                #{default => "100MB", desc => ?DESC(buffer_segment_bytes)}
+                #{default => <<"100MB">>, desc => ?DESC(buffer_segment_bytes)}
             )},
         {memory_overload_protection,
             mk(boolean(), #{
diff --git a/lib-ee/emqx_license/src/emqx_license_schema.erl b/lib-ee/emqx_license/src/emqx_license_schema.erl
index 9d16f697c..7383af92c 100644
--- a/lib-ee/emqx_license/src/emqx_license_schema.erl
+++ b/lib-ee/emqx_license/src/emqx_license_schema.erl
@@ -46,12 +46,12 @@ fields(key_license) ->
         }},
         {connection_low_watermark, #{
             type => emqx_schema:percent(),
-            default => "75%",
+            default => <<"75%">>,
             desc => ?DESC(connection_low_watermark_field)
         }},
         {connection_high_watermark, #{
             type => emqx_schema:percent(),
-            default => "80%",
+            default => <<"80%">>,
             desc => ?DESC(connection_high_watermark_field)
         }}
     ].

From f9895a33b6c545f7528964e3f2802687e5dd67d7 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 15 Feb 2023 14:46:45 +0100
Subject: [PATCH 122/144] refactor: rename max_row_limit to default_row_limit

---
 apps/emqx_management/include/emqx_mgmt.hrl    |  3 ++-
 apps/emqx_management/src/emqx_mgmt.erl        | 26 ++++++++-----------
 apps/emqx_management/src/emqx_mgmt_api.erl    |  6 ++---
 apps/emqx_management/src/emqx_mgmt_util.erl   |  2 +-
 .../test/emqx_mgmt_api_alarms_SUITE.erl       |  2 +-
 .../test/emqx_mgmt_api_clients_SUITE.erl      |  2 +-
 .../test/emqx_mgmt_api_subscription_SUITE.erl |  6 ++---
 .../test/emqx_mgmt_api_topics_SUITE.erl       |  2 +-
 apps/emqx_retainer/src/emqx_retainer_api.erl  |  2 +-
 9 files changed, 24 insertions(+), 27 deletions(-)

diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl
index b68a9a634..12be1df4f 100644
--- a/apps/emqx_management/include/emqx_mgmt.hrl
+++ b/apps/emqx_management/include/emqx_mgmt.hrl
@@ -16,4 +16,5 @@
 
 -define(MANAGEMENT_SHARD, emqx_management_shard).
 
--define(MAX_ROW_LIMIT, 100).
+-define(DEFAULT_ROW_LIMIT, 100).
+-define(MAX_TABLE_SIZE, 100).
diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 2d6fa854e..620c1e3f9 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -21,8 +21,6 @@
 -elvis([{elvis_style, god_modules, disable}]).
 
 -include_lib("stdlib/include/qlc.hrl").
--include_lib("emqx/include/emqx.hrl").
--include_lib("emqx/include/emqx_mqtt.hrl").
 
 %% Nodes and Brokers API
 -export([
@@ -105,12 +103,10 @@
 
 %% Common Table API
 -export([
-    max_row_limit/0,
+    default_row_limit/0,
     vm_stats/0
 ]).
 
--define(APP, emqx_management).
-
 -elvis([{elvis_style, god_modules, disable}]).
 
 %%--------------------------------------------------------------------
@@ -390,10 +386,10 @@ call_client(Node, ClientId, Req) ->
 
 -spec do_list_subscriptions() -> [map()].
 do_list_subscriptions() ->
-    case check_row_limit([mqtt_subproperty]) of
+    case check_max_table_size([mqtt_subproperty]) of
         false ->
             throw(max_row_limit);
-        ok ->
+        true ->
             [
                 #{topic => Topic, clientid => ClientId, options => Options}
              || {{Topic, ClientId}, Options} <- ets:tab2list(mqtt_subproperty)
@@ -556,15 +552,15 @@ unwrap_rpc(Res) ->
 otp_rel() ->
     iolist_to_binary([emqx_vm:get_otp_version(), "/", erlang:system_info(version)]).
 
-check_row_limit(Tables) ->
-    check_row_limit(Tables, max_row_limit()).
+check_max_table_size(Tables) ->
+    check_max_table_size(Tables, ?MAX_TABLE_SIZE).
 
-check_row_limit([], _Limit) ->
-    ok;
-check_row_limit([Tab | Tables], Limit) ->
+check_max_table_size([], _Limit) ->
+    true;
+check_max_table_size([Tab | Tables], Limit) ->
     case table_size(Tab) > Limit of
         true -> false;
-        false -> check_row_limit(Tables, Limit)
+        false -> check_max_table_size(Tables, Limit)
     end.
 
 check_results(Results) ->
@@ -573,7 +569,7 @@ check_results(Results) ->
         false -> unwrap_rpc(lists:last(Results))
     end.
 
-max_row_limit() ->
-    ?MAX_ROW_LIMIT.
+default_row_limit() ->
+    ?DEFAULT_ROW_LIMIT.
 
 table_size(Tab) -> ets:info(Tab, size).
diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl
index 3c4d787d3..a0a40533d 100644
--- a/apps/emqx_management/src/emqx_mgmt_api.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api.erl
@@ -98,8 +98,8 @@ count(Table) ->
 page(Params) ->
     maps:get(<<"page">>, Params, 1).
 
-limit(Params) ->
-    maps:get(<<"limit">>, Params, emqx_mgmt:max_row_limit()).
+limit(Params) when is_map(Params) ->
+    maps:get(<<"limit">>, Params, emqx_mgmt:default_row_limit()).
 
 %%--------------------------------------------------------------------
 %% Node Query
@@ -683,7 +683,7 @@ paginate_test_() ->
     Size = 1000,
     MyLimit = 10,
     ets:insert(?MODULE, [{I, foo} || I <- lists:seq(1, Size)]),
-    DefaultLimit = emqx_mgmt:max_row_limit(),
+    DefaultLimit = emqx_mgmt:default_row_limit(),
     NoParamsResult = paginate(?MODULE, #{}, {?MODULE, paginate_test_format}),
     PaginateResults = [
         paginate(
diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl
index c0d9e6036..b81b39b07 100644
--- a/apps/emqx_management/src/emqx_mgmt_util.erl
+++ b/apps/emqx_management/src/emqx_mgmt_util.erl
@@ -302,7 +302,7 @@ page_params() ->
             name => limit,
             in => query,
             description => <<"Page size">>,
-            schema => #{type => integer, default => emqx_mgmt:max_row_limit()}
+            schema => #{type => integer, default => emqx_mgmt:default_row_limit()}
         }
     ].
 
diff --git a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl
index 2c61651bf..69ace16e8 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl
@@ -62,5 +62,5 @@ get_alarms(AssertCount, Activated) ->
     Limit = maps:get(<<"limit">>, Meta),
     Count = maps:get(<<"count">>, Meta),
     ?assertEqual(Page, 1),
-    ?assertEqual(Limit, emqx_mgmt:max_row_limit()),
+    ?assertEqual(Limit, emqx_mgmt:default_row_limit()),
     ?assert(Count >= AssertCount).
diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
index 1a74d3af6..c7f4c9845 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
@@ -64,7 +64,7 @@ t_clients(_) ->
     ClientsLimit = maps:get(<<"limit">>, ClientsMeta),
     ClientsCount = maps:get(<<"count">>, ClientsMeta),
     ?assertEqual(ClientsPage, 1),
-    ?assertEqual(ClientsLimit, emqx_mgmt:max_row_limit()),
+    ?assertEqual(ClientsLimit, emqx_mgmt:default_row_limit()),
     ?assertEqual(ClientsCount, 2),
 
     %% get /clients/:clientid
diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
index 2ab213e30..ccfa30037 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl
@@ -57,7 +57,7 @@ t_subscription_api(Config) ->
     Data = emqx_json:decode(Response, [return_maps]),
     Meta = maps:get(<<"meta">>, Data),
     ?assertEqual(1, maps:get(<<"page">>, Meta)),
-    ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta)),
+    ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)),
     ?assertEqual(2, maps:get(<<"count">>, Meta)),
     Subscriptions = maps:get(<<"data">>, Data),
     ?assertEqual(length(Subscriptions), 2),
@@ -95,7 +95,7 @@ t_subscription_api(Config) ->
 
     DataTopic2 = #{<<"meta">> := Meta2} = request_json(get, QS, Headers),
     ?assertEqual(1, maps:get(<<"page">>, Meta2)),
-    ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta2)),
+    ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta2)),
     ?assertEqual(1, maps:get(<<"count">>, Meta2)),
     SubscriptionsList2 = maps:get(<<"data">>, DataTopic2),
     ?assertEqual(length(SubscriptionsList2), 1).
@@ -120,7 +120,7 @@ t_subscription_fuzzy_search(Config) ->
 
     MatchData1 = #{<<"meta">> := MatchMeta1} = request_json(get, MatchQs, Headers),
     ?assertEqual(1, maps:get(<<"page">>, MatchMeta1)),
-    ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta1)),
+    ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, MatchMeta1)),
     %% count is undefined in fuzzy searching
     ?assertNot(maps:is_key(<<"count">>, MatchMeta1)),
     ?assertMatch(3, length(maps:get(<<"data">>, MatchData1))),
diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
index 8f9b224ef..0c2e684b4 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl
@@ -52,7 +52,7 @@ t_nodes_api(Config) ->
     RoutesData = emqx_json:decode(Response, [return_maps]),
     Meta = maps:get(<<"meta">>, RoutesData),
     ?assertEqual(1, maps:get(<<"page">>, Meta)),
-    ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta)),
+    ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)),
     ?assertEqual(1, maps:get(<<"count">>, Meta)),
     Data = maps:get(<<"data">>, RoutesData),
     Route = erlang:hd(Data),
diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl
index fa11b00f4..7b1337140 100644
--- a/apps/emqx_retainer/src/emqx_retainer_api.erl
+++ b/apps/emqx_retainer/src/emqx_retainer_api.erl
@@ -166,7 +166,7 @@ config(put, #{body := Body}) ->
 %%------------------------------------------------------------------------------
 lookup_retained(get, #{query_string := Qs}) ->
     Page = maps:get(<<"page">>, Qs, 1),
-    Limit = maps:get(<<"limit">>, Qs, emqx_mgmt:max_row_limit()),
+    Limit = maps:get(<<"limit">>, Qs, emqx_mgmt:default_row_limit()),
     {ok, Msgs} = emqx_retainer_mnesia:page_read(undefined, undefined, Page, Limit),
     {200, #{
         data => [format_message(Msg) || Msg <- Msgs],

From 777ca72ad56e9fbec2b8f440018e43461fa17957 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Wed, 15 Feb 2023 15:54:05 +0100
Subject: [PATCH 123/144] fix: don't crash on broker_info()

---
 apps/emqx_management/src/emqx_mgmt.erl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 620c1e3f9..7a48db0f4 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -190,9 +190,15 @@ lookup_broker(Node) ->
     Broker.
 
 broker_info() ->
-    Info = maps:from_list([{K, iolist_to_binary(V)} || {K, V} <- emqx_sys:info()]),
+    Info = lists:foldl(fun convert_broker_info/2, #{}, emqx_sys:info()),
     Info#{node => node(), otp_release => otp_rel(), node_status => 'Running'}.
 
+convert_broker_info({uptime, Uptime}, M) ->
+    M#{uptime => emqx_datetime:human_readable_duration_string(Uptime)};
+convert_broker_info({K, V}, M) ->
+    M#{K => iolist_to_binary(V)}.
+
+
 broker_info(Nodes) ->
     emqx_rpc:unwrap_erpc(emqx_management_proto_v3:broker_info(Nodes)).
 

From fc33bce40d267688fbe04984ce1403f5716c5d86 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Thu, 16 Feb 2023 09:17:32 +0100
Subject: [PATCH 124/144] test(emqx_mgmt): test list_nodes/0

---
 apps/emqx/test/emqx_common_test_helpers.erl   | 15 +++++
 apps/emqx_management/src/emqx_mgmt.erl        |  2 +-
 apps/emqx_management/test/emqx_mgmt_SUITE.erl | 61 +++++++++++++++++++
 3 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 apps/emqx_management/test/emqx_mgmt_SUITE.erl

diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index fe1dfa35e..5149b8b8a 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -22,6 +22,8 @@
 
 -export([
     all/1,
+    init_per_testcase/3,
+    end_per_testcase/3,
     boot_modules/1,
     start_apps/1,
     start_apps/2,
@@ -150,6 +152,19 @@ all(Suite) ->
         string:substr(atom_to_list(F), 1, 2) == "t_"
     ]).
 
+init_per_testcase(Module, TestCase, Config) ->
+    case erlang:function_exported(Module, TestCase, 2) of
+        true -> Module:TestCase(init, Config);
+        false -> Config
+    end.
+
+end_per_testcase(Module, TestCase, Config) ->
+    case erlang:function_exported(Module, TestCase, 2) of
+        true -> Module:TestCase('end', Config);
+        false -> ok
+    end,
+    Config.
+
 %% set emqx app boot modules
 -spec boot_modules(all | list(atom())) -> ok.
 boot_modules(Mods) ->
diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 7a48db0f4..7f00260b2 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -158,7 +158,7 @@ node_info(Nodes) ->
     emqx_rpc:unwrap_erpc(emqx_management_proto_v3:node_info(Nodes)).
 
 stopped_node_info(Node) ->
-    #{name => Node, node_status => 'stopped'}.
+    {Node, #{node => Node, node_status => 'stopped'}}.
 
 vm_stats() ->
     Idle =
diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
new file mode 100644
index 000000000..68957e891
--- /dev/null
+++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
@@ -0,0 +1,61 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_mgmt_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_management]),
+    Config.
+
+end_per_suite(_) ->
+    emqx_mgmt_api_test_util:end_suite([emqx_management, emqx_conf]).
+
+init_per_testcase(TestCase, Config) ->
+    emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+    emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config).
+
+t_list_nodes(init, Config) ->
+    meck:expect(
+        mria_mnesia,
+        cluster_nodes,
+        fun
+            (running) -> [node()];
+            (stopped) -> ['stopped@node']
+        end
+    ),
+    Config;
+t_list_nodes('end', _Config) ->
+    meck:unload(mria_mnesia).
+
+t_list_nodes(_) ->
+    NodeInfos = emqx_mgmt:list_nodes(),
+    Node = node(),
+    ?assertMatch(
+        [
+            {Node, #{node := Node, node_status := 'running'}},
+            {'stopped@node', #{node := 'stopped@node', node_status := 'stopped'}}
+        ],
+        NodeInfos
+    ).

From a6d88c3caa455d44984d083fc4f7fdaf53713bd7 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Thu, 16 Feb 2023 16:47:44 +0100
Subject: [PATCH 125/144] test: more tests

---
 apps/emqx_management/src/emqx_mgmt.erl        |  2 +-
 apps/emqx_management/test/emqx_mgmt_SUITE.erl | 98 ++++++++++++++++++-
 2 files changed, 98 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 7f00260b2..52c50fda6 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -191,7 +191,7 @@ lookup_broker(Node) ->
 
 broker_info() ->
     Info = lists:foldl(fun convert_broker_info/2, #{}, emqx_sys:info()),
-    Info#{node => node(), otp_release => otp_rel(), node_status => 'Running'}.
+    Info#{node => node(), otp_release => otp_rel(), node_status => 'running'}.
 
 convert_broker_info({uptime, Uptime}, M) ->
     M#{uptime => emqx_datetime:human_readable_duration_string(Uptime)};
diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
index 68957e891..28533cac5 100644
--- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
@@ -19,6 +19,9 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-export([ident/1]).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -31,9 +34,11 @@ end_per_suite(_) ->
     emqx_mgmt_api_test_util:end_suite([emqx_management, emqx_conf]).
 
 init_per_testcase(TestCase, Config) ->
+    meck:expect(mria_mnesia, running_nodes, 0, [node()]),
     emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config).
 
 end_per_testcase(TestCase, Config) ->
+    meck:unload(mria_mnesia),
     emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config).
 
 t_list_nodes(init, Config) ->
@@ -47,7 +52,7 @@ t_list_nodes(init, Config) ->
     ),
     Config;
 t_list_nodes('end', _Config) ->
-    meck:unload(mria_mnesia).
+    ok.
 
 t_list_nodes(_) ->
     NodeInfos = emqx_mgmt:list_nodes(),
@@ -59,3 +64,94 @@ t_list_nodes(_) ->
         ],
         NodeInfos
     ).
+
+t_lookup_node(init, Config) ->
+    meck:new(os, [passthrough, unstick, no_link]),
+    OsType = os:type(),
+    meck:expect(os, type, 0, {win32, winME}),
+    [{os_type, OsType} | Config];
+t_lookup_node('end', Config) ->
+    %% We need to restore the original behavior so that rebar3 doesn't crash. If
+    %% we'd `meck:unload(os)` or not set `no_link` then `ct` crashes calling
+    %% `os` with "The code server called the unloaded module `os'".
+    OsType = ?config(os_type, Config),
+    meck:expect(os, type, 0, OsType),
+    ok.
+
+t_lookup_node(_) ->
+    Node = node(),
+    ?assertMatch(
+        #{node := Node, node_status := 'running', memory_total := 0},
+        emqx_mgmt:lookup_node(node())
+    ),
+    ?assertMatch(
+        {error, _},
+        emqx_mgmt:lookup_node('fake@nohost')
+    ),
+    ok.
+
+t_list_brokers(_) ->
+    Node = node(),
+    ?assertMatch(
+        [{Node, #{node := Node, node_status := running, uptime := _}}],
+        emqx_mgmt:list_brokers()
+    ).
+
+t_lookup_broker(_) ->
+    Node = node(),
+    ?assertMatch(
+        #{node := Node, node_status := running, uptime := _},
+        emqx_mgmt:lookup_broker(Node)
+    ).
+
+t_get_metrics(_) ->
+    Metrics = emqx_mgmt:get_metrics(),
+    ?assert(maps:size(Metrics) > 0),
+    ?assertMatch(
+        Metrics, maps:from_list(emqx_mgmt:get_metrics(node()))
+    ).
+
+t_lookup_client(init, Config) ->
+    setup_clients(Config);
+t_lookup_client('end', Config) ->
+    disconnect_clients(Config).
+
+t_lookup_client(_Config) ->
+    [{Chan, Info, Stats}] = emqx_mgmt:lookup_client({clientid, <<"client1">>}, {?MODULE, ident}),
+    ?assertEqual(
+        [{Chan, Info, Stats}],
+        emqx_mgmt:lookup_client({username, <<"user1">>}, {?MODULE, ident})
+    ),
+    ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, {?MODULE, ident})).
+
+t_kickout_client(init, Config) ->
+    process_flag(trap_exit, true),
+    setup_clients(Config);
+t_kickout_client('end', _Config) ->
+    ok.
+
+t_kickout_client(Config) ->
+    [C | _] = ?config(clients, Config),
+    ok = emqx_mgmt:kickout_client({<<"client1">>, {?MODULE, ident}}),
+    receive
+        {'EXIT', C, Reason} ->
+            ?assertEqual({shutdown, tcp_closed}, Reason);
+        Foo ->
+            error({unexpected, Foo})
+    after 1000 ->
+        error(timeout)
+    end,
+    ?assertEqual({error, not_found}, emqx_mgmt:kickout_client({<<"notfound">>, {?MODULE, ident}})).
+
+%%% helpers
+ident(Arg) ->
+    Arg.
+
+setup_clients(Config) ->
+    {ok, C} = emqtt:start_link([{clientid, <<"client1">>}, {username, <<"user1">>}]),
+    {ok, _} = emqtt:connect(C),
+    [{clients, [C]} | Config].
+
+disconnect_clients(Config) ->
+    Clients = ?config(clients, Config),
+    lists:foreach(fun emqtt:disconnect/1, Clients).

From f3ced5d5eb02c01d21323f9a6f43e4c8a1063684 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Thu, 16 Feb 2023 17:04:15 +0100
Subject: [PATCH 126/144] refactor: kickout_client doesn't need a format fun

---
 apps/emqx_management/src/emqx_mgmt.erl              | 13 +++++++++----
 apps/emqx_management/src/emqx_mgmt_api_clients.erl  |  2 +-
 apps/emqx_management/test/emqx_mgmt_SUITE.erl       | 12 +++++++-----
 .../test/emqx_mgmt_api_clients_SUITE.erl            |  9 ++++++++-
 4 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 52c50fda6..04f7d9ad0 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -267,7 +267,7 @@ lookup_client({username, Username}, FormatFun) ->
      || Node <- mria_mnesia:running_nodes()
     ]).
 
-lookup_client(Node, Key, {M, F}) ->
+lookup_client(Node, Key, FormatFun) ->
     case unwrap_rpc(emqx_cm_proto_v1:lookup_client(Node, Key)) of
         {error, Err} ->
             {error, Err};
@@ -275,14 +275,19 @@ lookup_client(Node, Key, {M, F}) ->
             lists:map(
                 fun({Chan, Info0, Stats}) ->
                     Info = Info0#{node => Node},
-                    M:F({Chan, Info, Stats})
+                    maybe_format(FormatFun, {Chan, Info, Stats})
                 end,
                 L
             )
     end.
 
-kickout_client({ClientID, FormatFun}) ->
-    case lookup_client({clientid, ClientID}, FormatFun) of
+maybe_format(undefined, A) ->
+    A;
+maybe_format({M, F}, A) ->
+    M:F(A).
+
+kickout_client(ClientID) ->
+    case lookup_client({clientid, ClientID}, undefined) of
         [] ->
             {error, not_found};
         _ ->
diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index 571f190f2..0a5488389 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -677,7 +677,7 @@ lookup(#{clientid := ClientID}) ->
     end.
 
 kickout(#{clientid := ClientID}) ->
-    case emqx_mgmt:kickout_client({ClientID, ?FORMAT_FUN}) of
+    case emqx_mgmt:kickout_client(ClientID) of
         {error, not_found} ->
             {404, ?CLIENTID_NOT_FOUND};
         _ ->
diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
index 28533cac5..13acb9fc3 100644
--- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
@@ -23,6 +23,8 @@
 
 -export([ident/1]).
 
+-define(FORMATFUN, {?MODULE, ident}).
+
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
@@ -117,12 +119,12 @@ t_lookup_client('end', Config) ->
     disconnect_clients(Config).
 
 t_lookup_client(_Config) ->
-    [{Chan, Info, Stats}] = emqx_mgmt:lookup_client({clientid, <<"client1">>}, {?MODULE, ident}),
+    [{Chan, Info, Stats}] = emqx_mgmt:lookup_client({clientid, <<"client1">>}, ?FORMATFUN),
     ?assertEqual(
         [{Chan, Info, Stats}],
-        emqx_mgmt:lookup_client({username, <<"user1">>}, {?MODULE, ident})
+        emqx_mgmt:lookup_client({username, <<"user1">>}, ?FORMATFUN)
     ),
-    ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, {?MODULE, ident})).
+    ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, ?FORMATFUN)).
 
 t_kickout_client(init, Config) ->
     process_flag(trap_exit, true),
@@ -132,7 +134,7 @@ t_kickout_client('end', _Config) ->
 
 t_kickout_client(Config) ->
     [C | _] = ?config(clients, Config),
-    ok = emqx_mgmt:kickout_client({<<"client1">>, {?MODULE, ident}}),
+    ok = emqx_mgmt:kickout_client(<<"client1">>),
     receive
         {'EXIT', C, Reason} ->
             ?assertEqual({shutdown, tcp_closed}, Reason);
@@ -141,7 +143,7 @@ t_kickout_client(Config) ->
     after 1000 ->
         error(timeout)
     end,
-    ?assertEqual({error, not_found}, emqx_mgmt:kickout_client({<<"notfound">>, {?MODULE, ident}})).
+    ?assertEqual({error, not_found}, emqx_mgmt:kickout_client(<<"notfound">>)).
 
 %%% helpers
 ident(Arg) ->
diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
index c7f4c9845..d843a2209 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
@@ -78,7 +78,14 @@ t_clients(_) ->
     %% delete /clients/:clientid kickout
     Client2Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId2)]),
     {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client2Path),
-    timer:sleep(300),
+    Kick =
+        receive
+            {'EXIT', C2, _} ->
+                ok
+        after 300 ->
+            timeout
+        end,
+    ?assertEqual(ok, Kick),
     AfterKickoutResponse2 = emqx_mgmt_api_test_util:request_api(get, Client2Path),
     ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse2),
 

From 0d2ce85776c5cba495489090cd7ef5dd31c8c74c Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Thu, 16 Feb 2023 17:39:49 +0100
Subject: [PATCH 127/144] fix: return 'not found' for subscriptions of unknown
 client

---
 apps/emqx_management/src/emqx_mgmt.erl        | 33 +++++++++++--------
 .../src/emqx_mgmt_api_clients.erl             |  9 ++---
 apps/emqx_management/test/emqx_mgmt_SUITE.erl | 21 ++++++++++++
 .../test/emqx_mgmt_api_clients_SUITE.erl      |  2 +-
 4 files changed, 46 insertions(+), 19 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 04f7d9ad0..31436b9a0 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -286,12 +286,12 @@ maybe_format(undefined, A) ->
 maybe_format({M, F}, A) ->
     M:F(A).
 
-kickout_client(ClientID) ->
-    case lookup_client({clientid, ClientID}, undefined) of
+kickout_client(ClientId) ->
+    case lookup_client({clientid, ClientId}, undefined) of
         [] ->
             {error, not_found};
         _ ->
-            Results = [kickout_client(Node, ClientID) || Node <- mria_mnesia:running_nodes()],
+            Results = [kickout_client(Node, ClientId) || Node <- mria_mnesia:running_nodes()],
             check_results(Results)
     end.
 
@@ -302,17 +302,22 @@ list_authz_cache(ClientId) ->
     call_client(ClientId, list_authz_cache).
 
 list_client_subscriptions(ClientId) ->
-    Results = [client_subscriptions(Node, ClientId) || Node <- mria_mnesia:running_nodes()],
-    Filter =
-        fun
-            ({error, _}) ->
-                false;
-            ({_Node, List}) ->
-                erlang:is_list(List) andalso 0 < erlang:length(List)
-        end,
-    case lists:filter(Filter, Results) of
-        [] -> [];
-        [Result | _] -> Result
+    case lookup_client({clientid, ClientId}, undefined) of
+        [] ->
+            {error, not_found};
+        _ ->
+            Results = [client_subscriptions(Node, ClientId) || Node <- mria_mnesia:running_nodes()],
+            Filter =
+                fun
+                    ({error, _}) ->
+                        false;
+                    ({_Node, List}) ->
+                        erlang:is_list(List) andalso 0 < erlang:length(List)
+                end,
+            case lists:filter(Filter, Results) of
+                [] -> [];
+                [Result | _] -> Result
+            end
     end.
 
 client_subscriptions(Node, ClientId) ->
diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
index 0a5488389..cac3edaed 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl
@@ -274,11 +274,10 @@ schema("/clients/:clientid/subscriptions") ->
             responses => #{
                 200 => hoconsc:mk(
                     hoconsc:array(hoconsc:ref(emqx_mgmt_api_subscriptions, subscription)), #{}
+                ),
+                404 => emqx_dashboard_swagger:error_codes(
+                    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
                 )
-                %% returns [] if client not existed in cluster
-                %404 => emqx_dashboard_swagger:error_codes(
-                %    ['CLIENTID_NOT_FOUND'], <<"Client ID not found">>
-                %)
             }
         }
     };
@@ -599,6 +598,8 @@ unsubscribe_batch(post, #{bindings := #{clientid := ClientID}, body := TopicInfo
 
 subscriptions(get, #{bindings := #{clientid := ClientID}}) ->
     case emqx_mgmt:list_client_subscriptions(ClientID) of
+        {error, not_found} ->
+            {404, ?CLIENTID_NOT_FOUND};
         [] ->
             {200, []};
         {Node, Subs} ->
diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
index 13acb9fc3..a9379da8c 100644
--- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
@@ -145,6 +145,27 @@ t_kickout_client(Config) ->
     end,
     ?assertEqual({error, not_found}, emqx_mgmt:kickout_client(<<"notfound">>)).
 
+t_list_authz_cache(init, Config) ->
+    setup_clients(Config);
+t_list_authz_cache('end', Config) ->
+    disconnect_clients(Config).
+
+t_list_authz_cache(_) ->
+    ?assertNotMatch({error, _}, emqx_mgmt:list_authz_cache(<<"client1">>)),
+    ?assertMatch({error, not_found}, emqx_mgmt:list_authz_cache(<<"notfound">>)).
+
+t_list_client_subscriptions(init, Config) ->
+    setup_clients(Config);
+t_list_client_subscriptions('end', Config) ->
+    disconnect_clients(Config).
+
+t_list_client_subscriptions(Config) ->
+    [Client | _] = ?config(clients, Config),
+    ?assertEqual([], emqx_mgmt:list_client_subscriptions(<<"client1">>)),
+    emqtt:subscribe(Client, <<"t/#">>),
+    ?assertMatch({_, [{<<"t/#">>, _Opts}]}, emqx_mgmt:list_client_subscriptions(<<"client1">>)),
+    ?assertEqual({error, not_found}, emqx_mgmt:list_client_subscriptions(<<"notfound">>)).
+
 %%% helpers
 ident(Arg) ->
     Arg.
diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
index d843a2209..b2f3f5655 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
@@ -279,7 +279,7 @@ t_client_id_not_found(_Config) ->
     %% Client kickout
     ?assertMatch({error, {Http, _, Body}}, ReqFun(delete, PathFun([]))),
     %% Client Subscription list
-    ?assertMatch({ok, {{"HTTP/1.1", 200, "OK"}, _, "[]"}}, ReqFun(get, PathFun(["subscriptions"]))),
+    ?assertMatch({error, {Http, _, Body}}, ReqFun(get, PathFun(["subscriptions"]))),
     %% AuthZ Cache lookup
     ?assertMatch({error, {Http, _, Body}}, ReqFun(get, PathFun(["authorization", "cache"]))),
     %% AuthZ Cache clean

From 0d357f7038e8d96a760cff0367a191e3650e958e Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Fri, 17 Feb 2023 14:56:16 +0100
Subject: [PATCH 128/144] refactor: cleanup list subscriptions

---
 apps/emqx_management/src/emqx_mgmt.erl        | 41 +----------
 apps/emqx_management/test/emqx_mgmt_SUITE.erl | 71 +++++++++++++++++++
 .../test/emqx_mgmt_api_clients_SUITE.erl      |  4 +-
 3 files changed, 74 insertions(+), 42 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 31436b9a0..27a3d8b69 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -66,13 +66,8 @@
 
 %% Subscriptions
 -export([
-    list_subscriptions/1,
     list_subscriptions_via_topic/2,
-    list_subscriptions_via_topic/3,
-    lookup_subscriptions/1,
-    lookup_subscriptions/2,
-
-    do_list_subscriptions/0
+    list_subscriptions_via_topic/3
 ]).
 
 %% PubSub
@@ -400,21 +395,6 @@ call_client(Node, ClientId, Req) ->
 %% Subscriptions
 %%--------------------------------------------------------------------
 
--spec do_list_subscriptions() -> [map()].
-do_list_subscriptions() ->
-    case check_max_table_size([mqtt_subproperty]) of
-        false ->
-            throw(max_row_limit);
-        true ->
-            [
-                #{topic => Topic, clientid => ClientId, options => Options}
-             || {{Topic, ClientId}, Options} <- ets:tab2list(mqtt_subproperty)
-            ]
-    end.
-
-list_subscriptions(Node) ->
-    unwrap_rpc(emqx_management_proto_v3:list_subscriptions(Node)).
-
 list_subscriptions_via_topic(Topic, FormatFun) ->
     lists:append([
         list_subscriptions_via_topic(Node, Topic, FormatFun)
@@ -427,12 +407,6 @@ list_subscriptions_via_topic(Node, Topic, _FormatFun = {M, F}) ->
         Result -> M:F(Result)
     end.
 
-lookup_subscriptions(ClientId) ->
-    lists:append([lookup_subscriptions(Node, ClientId) || Node <- mria_mnesia:running_nodes()]).
-
-lookup_subscriptions(Node, ClientId) ->
-    unwrap_rpc(emqx_broker_proto_v1:list_client_subscriptions(Node, ClientId)).
-
 %%--------------------------------------------------------------------
 %% PubSub
 %%--------------------------------------------------------------------
@@ -568,17 +542,6 @@ unwrap_rpc(Res) ->
 otp_rel() ->
     iolist_to_binary([emqx_vm:get_otp_version(), "/", erlang:system_info(version)]).
 
-check_max_table_size(Tables) ->
-    check_max_table_size(Tables, ?MAX_TABLE_SIZE).
-
-check_max_table_size([], _Limit) ->
-    true;
-check_max_table_size([Tab | Tables], Limit) ->
-    case table_size(Tab) > Limit of
-        true -> false;
-        false -> check_max_table_size(Tables, Limit)
-    end.
-
 check_results(Results) ->
     case lists:any(fun(Item) -> Item =:= ok end, Results) of
         true -> ok;
@@ -587,5 +550,3 @@ check_results(Results) ->
 
 default_row_limit() ->
     ?DEFAULT_ROW_LIMIT.
-
-table_size(Tab) -> ets:info(Tab, size).
diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
index a9379da8c..f5f3acd53 100644
--- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
@@ -166,6 +166,77 @@ t_list_client_subscriptions(Config) ->
     ?assertMatch({_, [{<<"t/#">>, _Opts}]}, emqx_mgmt:list_client_subscriptions(<<"client1">>)),
     ?assertEqual({error, not_found}, emqx_mgmt:list_client_subscriptions(<<"notfound">>)).
 
+t_clean_cache(init, Config) ->
+    setup_clients(Config);
+t_clean_cache('end', Config) ->
+    disconnect_clients(Config).
+
+t_clean_cache(_Config) ->
+    ?assertNotMatch(
+        {error, _},
+        emqx_mgmt:clean_authz_cache(<<"client1">>)
+    ),
+    ?assertNotMatch(
+        {error, _},
+        emqx_mgmt:clean_authz_cache_all()
+    ),
+    ?assertNotMatch(
+        {error, _},
+        emqx_mgmt:clean_pem_cache_all()
+    ).
+
+t_set_client_props(init, Config) ->
+    setup_clients(Config);
+t_set_client_props('end', Config) ->
+    disconnect_clients(Config).
+
+t_set_client_props(_Config) ->
+    ?assertEqual(
+        % [FIXME] not implemented at this point?
+        ignored,
+        emqx_mgmt:set_ratelimit_policy(<<"client1">>, foo)
+    ),
+    ?assertEqual(
+        {error, not_found},
+        emqx_mgmt:set_ratelimit_policy(<<"notfound">>, foo)
+    ),
+    ?assertEqual(
+        % [FIXME] not implemented at this point?
+        ignored,
+        emqx_mgmt:set_quota_policy(<<"client1">>, foo)
+    ),
+    ?assertEqual(
+        {error, not_found},
+        emqx_mgmt:set_quota_policy(<<"notfound">>, foo)
+    ),
+    ?assertEqual(
+        ok,
+        emqx_mgmt:set_keepalive(<<"client1">>, 3600)
+    ),
+    ?assertMatch(
+        {error, _},
+        emqx_mgmt:set_keepalive(<<"client1">>, true)
+    ),
+    ?assertEqual(
+        {error, not_found},
+        emqx_mgmt:set_keepalive(<<"notfound">>, 3600)
+    ),
+    ok.
+
+t_list_subscriptions_via_topic(init, Config) ->
+    setup_clients(Config);
+t_list_subscriptions_via_topic('end', Config) ->
+    disconnect_clients(Config).
+
+t_list_subscriptions_via_topic(Config) ->
+    [Client | _] = ?config(clients, Config),
+    ?assertEqual([], emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN)),
+    emqtt:subscribe(Client, <<"t/#">>),
+    ?assertMatch(
+        [{{<<"t/#">>, _SubPid}, _Opts}],
+        emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN)
+    ).
+
 %%% helpers
 ident(Arg) ->
     Arg.
diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
index b2f3f5655..9f26f8542 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl
@@ -114,7 +114,7 @@ t_clients(_) ->
         SubscribeBody
     ),
     timer:sleep(100),
-    [{AfterSubTopic, #{qos := AfterSubQos}}] = emqx_mgmt:lookup_subscriptions(ClientId1),
+    {_, [{AfterSubTopic, #{qos := AfterSubQos}}]} = emqx_mgmt:list_client_subscriptions(ClientId1),
     ?assertEqual(AfterSubTopic, Topic),
     ?assertEqual(AfterSubQos, Qos),
 
@@ -159,7 +159,7 @@ t_clients(_) ->
         UnSubscribeBody
     ),
     timer:sleep(100),
-    ?assertEqual([], emqx_mgmt:lookup_subscriptions(Client1)),
+    ?assertEqual([], emqx_mgmt:list_client_subscriptions(ClientId1)),
 
     %% testcase cleanup, kickout client1
     {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client1Path),

From 0482f438029844a9ff2eba3a7f4147928c0ef3a3 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Fri, 17 Feb 2023 15:51:13 +0100
Subject: [PATCH 129/144] test: add test for banned and alarms

---
 apps/emqx_management/test/emqx_mgmt_SUITE.erl | 65 +++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
index f5f3acd53..993969d48 100644
--- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
@@ -237,10 +237,75 @@ t_list_subscriptions_via_topic(Config) ->
         emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN)
     ).
 
+t_alarms(_) ->
+    Node = node(),
+    ?assertEqual(
+        [{node(), []}],
+        emqx_mgmt:get_alarms(all)
+    ),
+    emqx_alarm:activate(foo),
+    ?assertMatch(
+        [{Node, [#{name := foo, activated := true, duration := _}]}],
+        emqx_mgmt:get_alarms(all)
+    ),
+    emqx_alarm:activate(bar),
+    ?assertMatch(
+        [{Node, [#{name := foo, activated := true}, #{name := bar, activated := true}]}],
+        sort_alarms(emqx_mgmt:get_alarms(all))
+    ),
+    ?assertEqual(
+        ok,
+        emqx_mgmt:deactivate(node(), bar)
+    ),
+    ?assertMatch(
+        [{Node, [#{name := foo, activated := true}, #{name := bar, activated := false}]}],
+        sort_alarms(emqx_mgmt:get_alarms(all))
+    ),
+    ?assertMatch(
+        [{Node, [#{name := foo, activated := true}]}],
+        emqx_mgmt:get_alarms(activated)
+    ),
+    ?assertMatch(
+        [{Node, [#{name := bar, activated := false}]}],
+        emqx_mgmt:get_alarms(deactivated)
+    ),
+    ?assertEqual(
+        [ok],
+        emqx_mgmt:delete_all_deactivated_alarms()
+    ),
+    ?assertMatch(
+        [{Node, [#{name := foo, activated := true}]}],
+        emqx_mgmt:get_alarms(all)
+    ),
+    ?assertEqual(
+        {error, not_found},
+        emqx_mgmt:deactivate(node(), bar)
+    ).
+
+t_banned(_) ->
+    Banned = #{
+        who => {clientid, <<"TestClient">>},
+        by => <<"banned suite">>,
+        reason => <<"test">>,
+        at => erlang:system_time(second),
+        until => erlang:system_time(second) + 1
+    },
+    ?assertMatch(
+        {ok, _},
+        emqx_mgmt:create_banned(Banned)
+    ),
+    ?assertEqual(
+        ok,
+        emqx_mgmt:delete_banned({clientid, <<"TestClient">>})
+    ).
+
 %%% helpers
 ident(Arg) ->
     Arg.
 
+sort_alarms([{Node, Alarms}]) ->
+    [{Node, lists:sort(fun(#{activate_at := A}, #{activate_at := B}) -> A < B end, Alarms)}].
+
 setup_clients(Config) ->
     {ok, C} = emqtt:start_link([{clientid, <<"client1">>}, {username, <<"user1">>}]),
     {ok, _} = emqtt:connect(C),

From c44c7fcbcea921529ad482678a60de4d45846c63 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Fri, 17 Feb 2023 15:55:30 +0100
Subject: [PATCH 130/144] style: remove unused macro

---
 apps/emqx_management/include/emqx_mgmt.hrl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl
index 12be1df4f..7f6b5a675 100644
--- a/apps/emqx_management/include/emqx_mgmt.hrl
+++ b/apps/emqx_management/include/emqx_mgmt.hrl
@@ -17,4 +17,3 @@
 -define(MANAGEMENT_SHARD, emqx_management_shard).
 
 -define(DEFAULT_ROW_LIMIT, 100).
--define(MAX_TABLE_SIZE, 100).

From 71f3efb2ce80c00c94ef286983426684c8d7350c Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Fri, 17 Feb 2023 15:57:53 +0100
Subject: [PATCH 131/144] style: fix empty line

---
 apps/emqx_management/src/emqx_mgmt.erl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index 27a3d8b69..e7c338adf 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -193,7 +193,6 @@ convert_broker_info({uptime, Uptime}, M) ->
 convert_broker_info({K, V}, M) ->
     M#{K => iolist_to_binary(V)}.
 
-
 broker_info(Nodes) ->
     emqx_rpc:unwrap_erpc(emqx_management_proto_v3:broker_info(Nodes)).
 

From 81fad58f12db7d63dd9c3c7562c4b6928d74324b Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 20 Feb 2023 10:47:25 +0100
Subject: [PATCH 132/144] fix: re-add `list_subscriptions/0`

Created a ticket to add an actual working implementation
---
 apps/emqx_management/src/emqx_mgmt.erl | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl
index e7c338adf..efa5a03bd 100644
--- a/apps/emqx_management/src/emqx_mgmt.erl
+++ b/apps/emqx_management/src/emqx_mgmt.erl
@@ -66,8 +66,11 @@
 
 %% Subscriptions
 -export([
+    list_subscriptions/1,
     list_subscriptions_via_topic/2,
-    list_subscriptions_via_topic/3
+    list_subscriptions_via_topic/3,
+
+    do_list_subscriptions/0
 ]).
 
 %% PubSub
@@ -394,6 +397,15 @@ call_client(Node, ClientId, Req) ->
 %% Subscriptions
 %%--------------------------------------------------------------------
 
+-spec do_list_subscriptions() -> no_return().
+do_list_subscriptions() ->
+    %% [FIXME] Add function to `emqx_broker` that returns list of subscriptions
+    %% and either redirect from here or bpapi directly (EMQX-8993).
+    throw(not_implemented).
+
+list_subscriptions(Node) ->
+    unwrap_rpc(emqx_management_proto_v3:list_subscriptions(Node)).
+
 list_subscriptions_via_topic(Topic, FormatFun) ->
     lists:append([
         list_subscriptions_via_topic(Node, Topic, FormatFun)

From 8ae444006122442971952c5fca2d7c1aea54cbce Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 20 Feb 2023 16:51:50 +0100
Subject: [PATCH 133/144] style: fix API description for bytes parameter

---
 apps/emqx_management/src/emqx_mgmt_api_trace.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
index cc4a905a4..624e0bdb8 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
@@ -315,7 +315,7 @@ fields(bytes) ->
             hoconsc:mk(
                 integer(),
                 #{
-                    desc => "Maximum number of bytes to store in request",
+                    desc => "Maximum number of bytes to send in response",
                     in => query,
                     required => false,
                     default => 1000

From 9ecf154a711ca13be0beb879274db45646c68e42 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 20 Feb 2023 17:05:41 +0100
Subject: [PATCH 134/144] fix: limit bytes param to signed 32bit int

We still need to check if chunk we're reading fits in memory
---
 .../src/emqx_mgmt_api_trace.erl               | 30 ++++++++++++++++---
 .../test/emqx_mgmt_api_trace_SUITE.erl        | 11 +++++--
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
index 624e0bdb8..594750d06 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
@@ -47,9 +47,12 @@
     get_trace_size/0
 ]).
 
+-define(MAX_SINT32, 2147483647).
+
 -define(TO_BIN(_B_), iolist_to_binary(_B_)).
 -define(NOT_FOUND(N), {404, #{code => 'NOT_FOUND', message => ?TO_BIN([N, " NOT FOUND"])}}).
 -define(BAD_REQUEST(C, M), {400, #{code => C, message => ?TO_BIN(M)}}).
+-define(SERVICE_UNAVAILABLE(C, M), {503, #{code => C, message => ?TO_BIN(M)}}).
 -define(TAGS, [<<"Trace">>]).
 
 namespace() -> "trace".
@@ -184,8 +187,15 @@ schema("/trace/:name/log") ->
                         {items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})},
                         {meta, fields(bytes) ++ fields(position)}
                     ],
-                400 => emqx_dashboard_swagger:error_codes(['NODE_ERROR'], <<"Trace Log Failed">>),
-                404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Trace Name Not Found">>)
+                400 => emqx_dashboard_swagger:error_codes(
+                    ['BAD_REQUEST', 'NODE_ERROR'], <<"Bad input parameter">>
+                ),
+                404 => emqx_dashboard_swagger:error_codes(
+                    ['NOT_FOUND'], <<"Trace Name Not Found">>
+                ),
+                503 => emqx_dashboard_swagger:error_codes(
+                    ['SERVICE_UNAVAILABLE'], <<"Requested chunk size too big">>
+                )
             }
         }
     }.
@@ -313,12 +323,16 @@ fields(bytes) ->
     [
         {bytes,
             hoconsc:mk(
-                integer(),
+                %% This seems to be the minimum max value we may encounter
+                %% across different OS
+                range(0, ?MAX_SINT32),
                 #{
                     desc => "Maximum number of bytes to send in response",
                     in => query,
                     required => false,
-                    default => 1000
+                    default => 1000,
+                    minimum => 0,
+                    maximum => ?MAX_SINT32
                 }
             )}
     ];
@@ -579,6 +593,14 @@ stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) ->
                     {200, #{meta => Meta, items => <<"">>}};
                 {error, not_found} ->
                     ?NOT_FOUND(Name);
+                {error, enomem} ->
+                    ?SLOG(warning, #{
+                        code => not_enough_mem,
+                        msg => "Requested chunk size too big",
+                        bytes => Bytes,
+                        name => Name
+                    }),
+                    ?SERVICE_UNAVAILABLE('SERVICE_UNAVAILABLE', <<"Requested chunk size too big">>);
                 {badrpc, nodedown} ->
                     ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>)
             end;
diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
index 6962a9043..92b35db99 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
@@ -19,9 +19,7 @@
 -compile(export_all).
 -compile(nowarn_export_all).
 
--include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
--include_lib("emqx/include/emqx.hrl").
 -include_lib("kernel/include/file.hrl").
 -include_lib("stdlib/include/zip.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
@@ -296,6 +294,15 @@ t_stream_log(_Config) ->
     #{<<"meta">> := Meta1, <<"items">> := Bin1} = json(Binary1),
     ?assertEqual(#{<<"position">> => 30, <<"bytes">> => 10}, Meta1),
     ?assertEqual(10, byte_size(Bin1)),
+    ct:pal("~p vs ~p", [Bin, Bin1]),
+    %% in theory they could be the same but we know they shouldn't
+    ?assertNotEqual(Bin, Bin1),
+    BadReqPath = api_path("trace/test_stream_log/log?&bytes=1000000000000"),
+    {error, {_, 400, _}} = request_api(get, BadReqPath),
+    meck:new(file, [passthrough, unstick]),
+    meck:expect(file, read, 2, {error, enomem}),
+    {error, {_, 503, _}} = request_api(get, Path),
+    meck:unload(file),
     {error, {_, 400, _}} =
         request_api(
             get,

From e78c2c2869d77f246e2d840f1b11d6c0a24911fb Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Mon, 20 Feb 2023 17:07:36 +0100
Subject: [PATCH 135/144] fix: return 404 in case node is not found

---
 apps/emqx_management/src/emqx_mgmt_api_trace.erl | 16 ++++++++--------
 .../test/emqx_mgmt_api_trace_SUITE.erl           | 12 ++++++------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
index 594750d06..38ce9dcf2 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl
@@ -51,7 +51,6 @@
 
 -define(TO_BIN(_B_), iolist_to_binary(_B_)).
 -define(NOT_FOUND(N), {404, #{code => 'NOT_FOUND', message => ?TO_BIN([N, " NOT FOUND"])}}).
--define(BAD_REQUEST(C, M), {400, #{code => C, message => ?TO_BIN(M)}}).
 -define(SERVICE_UNAVAILABLE(C, M), {503, #{code => C, message => ?TO_BIN(M)}}).
 -define(TAGS, [<<"Trace">>]).
 
@@ -151,8 +150,9 @@ schema("/trace/:name/download") ->
                                 #{schema => #{type => "string", format => "binary"}}
                         }
                     },
-                400 => emqx_dashboard_swagger:error_codes(['NODE_ERROR'], <<"Node Not Found">>),
-                404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Trace Name Not Found">>)
+                404 => emqx_dashboard_swagger:error_codes(
+                    ['NOT_FOUND', 'NODE_ERROR'], <<"Trace Name or Node Not Found">>
+                )
             }
         }
     };
@@ -188,10 +188,10 @@ schema("/trace/:name/log") ->
                         {meta, fields(bytes) ++ fields(position)}
                     ],
                 400 => emqx_dashboard_swagger:error_codes(
-                    ['BAD_REQUEST', 'NODE_ERROR'], <<"Bad input parameter">>
+                    ['BAD_REQUEST'], <<"Bad input parameter">>
                 ),
                 404 => emqx_dashboard_swagger:error_codes(
-                    ['NOT_FOUND'], <<"Trace Name Not Found">>
+                    ['NOT_FOUND', 'NODE_ERROR'], <<"Trace Name or Node Not Found">>
                 ),
                 503 => emqx_dashboard_swagger:error_codes(
                     ['SERVICE_UNAVAILABLE'], <<"Requested chunk size too big">>
@@ -509,7 +509,7 @@ download_trace_log(get, #{bindings := #{name := Name}, query_string := Query}) -
                     },
                     {200, Headers, {file_binary, ZipName, Binary}};
                 {error, not_found} ->
-                    ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>)
+                    ?NOT_FOUND(<<"Node">>)
             end;
         {error, not_found} ->
             ?NOT_FOUND(Name)
@@ -602,10 +602,10 @@ stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) ->
                     }),
                     ?SERVICE_UNAVAILABLE('SERVICE_UNAVAILABLE', <<"Requested chunk size too big">>);
                 {badrpc, nodedown} ->
-                    ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>)
+                    ?NOT_FOUND(<<"Node">>)
             end;
         {error, not_found} ->
-            ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>)
+            ?NOT_FOUND(<<"Node">>)
     end.
 
 -spec get_trace_size() -> #{{node(), file:name_all()} => non_neg_integer()}.
diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
index 92b35db99..162d07aaa 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl
@@ -223,12 +223,12 @@ t_log_file(_Config) ->
         ]},
         zip:table(Binary2)
     ),
-    {error, {_, 400, _}} =
+    {error, {_, 404, _}} =
         request_api(
             get,
-            api_path("trace/test_client_id/download?node=unknonwn_node")
+            api_path("trace/test_client_id/download?node=unknown_node")
         ),
-    {error, {_, 400, _}} =
+    {error, {_, 404, _}} =
         request_api(
             get,
             % known atom but unknown node
@@ -303,12 +303,12 @@ t_stream_log(_Config) ->
     meck:expect(file, read, 2, {error, enomem}),
     {error, {_, 503, _}} = request_api(get, Path),
     meck:unload(file),
-    {error, {_, 400, _}} =
+    {error, {_, 404, _}} =
         request_api(
             get,
-            api_path("trace/test_stream_log/log?node=unknonwn_node")
+            api_path("trace/test_stream_log/log?node=unknown_node")
         ),
-    {error, {_, 400, _}} =
+    {error, {_, 404, _}} =
         request_api(
             get,
             % known atom but not a node

From 0a207856aa69bca11369764edc71fc52086d808f Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 21 Feb 2023 10:23:16 +0100
Subject: [PATCH 136/144] chore: bump quicer 0.0.111 and emqtt 1.8.2

---
 apps/emqx/rebar.config        | 2 +-
 apps/emqx/rebar.config.script | 2 +-
 mix.exs                       | 4 ++--
 rebar.config                  | 2 +-
 rebar.config.erl              | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index b79d14c54..a3ea4f2e7 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -43,7 +43,7 @@
             {meck, "0.9.2"},
             {proper, "1.4.0"},
             {bbmustache, "1.10.0"},
-            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.1"}}}
+            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.2"}}}
         ]},
         {extra_src_dirs, [{"test", [recursive]}]}
     ]}
diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script
index b2de8a7dd..2025f5ad5 100644
--- a/apps/emqx/rebar.config.script
+++ b/apps/emqx/rebar.config.script
@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.109"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.111"}}}.
 
 Dialyzer = fun(Config) ->
                    {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
diff --git a/mix.exs b/mix.exs
index ef2ed262f..b0608a48c 100644
--- a/mix.exs
+++ b/mix.exs
@@ -60,7 +60,7 @@ defmodule EMQXUmbrella.MixProject do
       {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true},
       {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
-      {:emqtt, github: "emqx/emqtt", tag: "1.8.1", override: true},
+      {:emqtt, github: "emqx/emqtt", tag: "1.8.2", override: true},
       {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
       {:observer_cli, "1.7.1"},
       {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
@@ -645,7 +645,7 @@ defmodule EMQXUmbrella.MixProject do
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
-      do: [{:quicer, github: "emqx/quic", tag: "0.0.109", override: true}],
+      do: [{:quicer, github: "emqx/quic", tag: "0.0.111", override: true}],
       else: []
   end
 
diff --git a/rebar.config b/rebar.config
index bc8362c01..0a163108f 100644
--- a/rebar.config
+++ b/rebar.config
@@ -62,7 +62,7 @@
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}}
     , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
-    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.1"}}}
+    , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.2"}}}
     , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}}
     , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x
     , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}
diff --git a/rebar.config.erl b/rebar.config.erl
index 3be4b70f6..8b36f907a 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -39,7 +39,7 @@ bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}.
 
 quicer() ->
-    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.109"}}}.
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.111"}}}.
 
 jq() ->
     {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}.

From 7502e570668e9efc9b6c9bcb1f0a715341ec42ae Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Feb 2023 10:41:59 +0100
Subject: [PATCH 137/144] chore: add changelog

---
 changes/ce/fix-10009.en.md | 1 +
 changes/ce/fix-10009.zh.md | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changes/ce/fix-10009.en.md
 create mode 100644 changes/ce/fix-10009.zh.md

diff --git a/changes/ce/fix-10009.en.md b/changes/ce/fix-10009.en.md
new file mode 100644
index 000000000..37f33a958
--- /dev/null
+++ b/changes/ce/fix-10009.en.md
@@ -0,0 +1 @@
+Validate `bytes` param to `GET /trace/:name/log` to not exceed signed 32bit integer.
diff --git a/changes/ce/fix-10009.zh.md b/changes/ce/fix-10009.zh.md
new file mode 100644
index 000000000..bb55ea5b9
--- /dev/null
+++ b/changes/ce/fix-10009.zh.md
@@ -0,0 +1 @@
+验证 `GET /trace/:name/log` 的 `bytes` 参数，使其不超过有符号的32位整数。

From 28382ec26bbba048fa79814272d1734aa76422af Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Feb 2023 13:11:34 +0100
Subject: [PATCH 138/144] test: add test for pubsub api

---
 apps/emqx_management/test/emqx_mgmt_SUITE.erl | 73 ++++++++++++++++++-
 1 file changed, 72 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
index 993969d48..4619905cb 100644
--- a/apps/emqx_management/test/emqx_mgmt_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl
@@ -124,7 +124,11 @@ t_lookup_client(_Config) ->
         [{Chan, Info, Stats}],
         emqx_mgmt:lookup_client({username, <<"user1">>}, ?FORMATFUN)
     ),
-    ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, ?FORMATFUN)).
+    ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, ?FORMATFUN)),
+    meck:expect(mria_mnesia, running_nodes, 0, [node(), 'fake@nonode']),
+    ?assertMatch(
+        [_ | {error, nodedown}], emqx_mgmt:lookup_client({clientid, <<"client1">>}, ?FORMATFUN)
+    ).
 
 t_kickout_client(init, Config) ->
     process_flag(trap_exit, true),
@@ -183,6 +187,15 @@ t_clean_cache(_Config) ->
     ?assertNotMatch(
         {error, _},
         emqx_mgmt:clean_pem_cache_all()
+    ),
+    meck:expect(mria_mnesia, running_nodes, 0, [node(), 'fake@nonode']),
+    ?assertMatch(
+        {error, [{'fake@nonode', {error, _}}]},
+        emqx_mgmt:clean_authz_cache_all()
+    ),
+    ?assertMatch(
+        {error, [{'fake@nonode', {error, _}}]},
+        emqx_mgmt:clean_pem_cache_all()
     ).
 
 t_set_client_props(init, Config) ->
@@ -237,6 +250,64 @@ t_list_subscriptions_via_topic(Config) ->
         emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN)
     ).
 
+t_pubsub_api(init, Config) ->
+    setup_clients(Config);
+t_pubsub_api('end', Config) ->
+    disconnect_clients(Config).
+
+-define(TT(Topic), {Topic, #{qos => 0}}).
+
+t_pubsub_api(Config) ->
+    [Client | _] = ?config(clients, Config),
+    ?assertEqual([], emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN)),
+    ?assertMatch(
+        {subscribe, _, _},
+        emqx_mgmt:subscribe(<<"client1">>, [?TT(<<"t/#">>), ?TT(<<"t1/#">>), ?TT(<<"t2/#">>)])
+    ),
+    timer:sleep(100),
+    ?assertMatch(
+        [{{<<"t/#">>, _SubPid}, _Opts}],
+        emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN)
+    ),
+    Message = emqx_message:make(?MODULE, 0, <<"t/foo">>, <<"helloworld">>, #{}, #{}),
+    emqx_mgmt:publish(Message),
+    Recv =
+        receive
+            {publish, #{client_pid := Client, payload := <<"helloworld">>}} ->
+                ok
+        after 100 ->
+            timeout
+        end,
+    ?assertEqual(ok, Recv),
+    ?assertEqual({error, channel_not_found}, emqx_mgmt:subscribe(<<"notfound">>, [?TT(<<"t/#">>)])),
+    ?assertNotMatch({error, _}, emqx_mgmt:unsubscribe(<<"client1">>, <<"t/#">>)),
+    ?assertEqual({error, channel_not_found}, emqx_mgmt:unsubscribe(<<"notfound">>, <<"t/#">>)),
+    Node = node(),
+    ?assertMatch(
+        {Node, [{<<"t1/#">>, _}, {<<"t2/#">>, _}]},
+        emqx_mgmt:list_client_subscriptions(<<"client1">>)
+    ),
+    ?assertMatch(
+        {unsubscribe, [{<<"t1/#">>, _}, {<<"t2/#">>, _}]},
+        emqx_mgmt:unsubscribe_batch(<<"client1">>, [<<"t1/#">>, <<"t2/#">>])
+    ),
+    timer:sleep(100),
+    ?assertMatch([], emqx_mgmt:list_client_subscriptions(<<"client1">>)),
+    ?assertEqual(
+        {error, channel_not_found},
+        emqx_mgmt:unsubscribe_batch(<<"notfound">>, [<<"t1/#">>, <<"t2/#">>])
+    ).
+
+t_alarms(init, Config) ->
+    [
+        emqx_mgmt:deactivate(Node, Name)
+     || {Node, ActiveAlarms} <- emqx_mgmt:get_alarms(activated), #{name := Name} <- ActiveAlarms
+    ],
+    emqx_mgmt:delete_all_deactivated_alarms(),
+    Config;
+t_alarms('end', Config) ->
+    Config.
+
 t_alarms(_) ->
     Node = node(),
     ?assertEqual(

From 965d63f4f52a227ed231935fed70cb87485469cd Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Feb 2023 15:33:56 +0100
Subject: [PATCH 139/144] fix: schema for `/gateways/:name/clients` was missing
 top-level structure

---
 apps/emqx_gateway/src/emqx_gateway_api_clients.erl | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
index ef1c4c386..b30de3a3e 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
@@ -19,7 +19,6 @@
 -include("emqx_gateway_http.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include_lib("emqx/include/emqx_placeholder.hrl").
 -include_lib("emqx/include/logger.hrl").
 
 -behaviour(minirest_api).
@@ -464,7 +463,12 @@ schema("/gateways/:name/clients") ->
                 summary => <<"List Gateway's Clients">>,
                 parameters => params_client_query(),
                 responses =>
-                    ?STANDARD_RESP(#{200 => schema_client_list()})
+                    ?STANDARD_RESP(#{
+                        200 => [
+                            {data, schema_client_list()},
+                            {meta, mk(hoconsc:ref(emqx_dashboard_swagger, meta), #{})}
+                        ]
+                    })
             }
     };
 schema("/gateways/:name/clients/:clientid") ->

From a0589d5b95ab5d6749ad3d9037bfc9202dd0c2b6 Mon Sep 17 00:00:00 2001
From: Stefan Strigler <stefan.strigler@emqx.io>
Date: Tue, 21 Feb 2023 16:23:38 +0100
Subject: [PATCH 140/144] fix: return `404` for unknown node names

---
 .../src/emqx_dashboard_monitor_api.erl        | 45 ++++++++-----------
 .../test/emqx_dashboard_monitor_SUITE.erl     |  6 +--
 2 files changed, 21 insertions(+), 30 deletions(-)

diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
index f8b0918be..69f5bf34e 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl
@@ -55,7 +55,7 @@ schema("/monitor/nodes/:node") ->
             parameters => [parameter_node(), parameter_latest()],
             responses => #{
                 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(sampler)), #{}),
-                400 => emqx_dashboard_swagger:error_codes(['BAD_RPC'], <<"Bad RPC">>)
+                404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Node not found">>)
             }
         }
     };
@@ -79,7 +79,7 @@ schema("/monitor_current/nodes/:node") ->
             parameters => [parameter_node()],
             responses => #{
                 200 => hoconsc:mk(hoconsc:ref(sampler_current), #{}),
-                400 => emqx_dashboard_swagger:error_codes(['BAD_RPC'], <<"Bad RPC">>)
+                404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Node not found">>)
             }
         }
     }.
@@ -122,38 +122,31 @@ fields(sampler_current) ->
 monitor(get, #{query_string := QS, bindings := Bindings}) ->
     Latest = maps:get(<<"latest">>, QS, infinity),
     RawNode = maps:get(node, Bindings, all),
-    case emqx_misc:safe_to_existing_atom(RawNode, utf8) of
-        {ok, Node} ->
-            case emqx_dashboard_monitor:samplers(Node, Latest) of
-                {badrpc, {Node, Reason}} ->
-                    Message = list_to_binary(
-                        io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason])
-                    ),
-                    {400, 'BAD_RPC', Message};
-                Samplers ->
-                    {200, Samplers}
-            end;
-        _ ->
-            Message = list_to_binary(io_lib:format("Bad node ~p", [RawNode])),
-            {400, 'BAD_RPC', Message}
+    with_node(RawNode, dashboard_samplers_fun(Latest)).
+
+dashboard_samplers_fun(Latest) ->
+    fun(NodeOrCluster) ->
+        case emqx_dashboard_monitor:samplers(NodeOrCluster, Latest) of
+            {badrpc, _} = Error -> Error;
+            Samplers -> {ok, Samplers}
+        end
     end.
 
 monitor_current(get, #{bindings := Bindings}) ->
     RawNode = maps:get(node, Bindings, all),
+    with_node(RawNode, fun emqx_dashboard_monitor:current_rate/1).
+
+with_node(RawNode, Fun) ->
     case emqx_misc:safe_to_existing_atom(RawNode, utf8) of
         {ok, NodeOrCluster} ->
-            case emqx_dashboard_monitor:current_rate(NodeOrCluster) of
-                {ok, CurrentRate} ->
-                    {200, CurrentRate};
+            case Fun(NodeOrCluster) of
                 {badrpc, {Node, Reason}} ->
-                    Message = list_to_binary(
-                        io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason])
-                    ),
-                    {400, 'BAD_RPC', Message}
+                    {404, 'NOT_FOUND', io_lib:format("Node not found: ~p (~p)", [Node, Reason])};
+                {ok, Result} ->
+                    {200, Result}
             end;
-        {error, _} ->
-            Message = list_to_binary(io_lib:format("Bad node ~p", [RawNode])),
-            {400, 'BAD_RPC', Message}
+        _Error ->
+            {404, 'NOT_FOUND', io_lib:format("Node not found: ~p", [RawNode])}
     end.
 
 %% -------------------------------------------------------------------------------------------------
diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
index 74c6d9cc1..bfbd9b973 100644
--- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl
@@ -22,8 +22,6 @@
 -import(emqx_dashboard_SUITE, [auth_header_/0]).
 
 -include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
--include_lib("emqx/include/emqx.hrl").
 -include("emqx_dashboard.hrl").
 
 -define(SERVER, "http://127.0.0.1:18083").
@@ -114,9 +112,9 @@ t_monitor_reset(_) ->
     ok.
 
 t_monitor_api_error(_) ->
-    {error, {400, #{<<"code">> := <<"BAD_RPC">>}}} =
+    {error, {404, #{<<"code">> := <<"NOT_FOUND">>}}} =
         request(["monitor", "nodes", 'emqx@127.0.0.2']),
-    {error, {400, #{<<"code">> := <<"BAD_RPC">>}}} =
+    {error, {404, #{<<"code">> := <<"NOT_FOUND">>}}} =
         request(["monitor_current", "nodes", 'emqx@127.0.0.2']),
     {error, {400, #{<<"code">> := <<"BAD_REQUEST">>}}} =
         request(["monitor"], "latest=0"),

From 2d67c96553a05816b63e2f8b7c7c3de481d06b5b Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Tue, 21 Feb 2023 11:38:34 +0100
Subject: [PATCH 141/144] feat(charts): add pod disruption budget to ee

---
 .../charts/emqx-enterprise/templates/pdb.yaml  | 18 ++++++++++++++++++
 .../emqx-enterprise/templates/service.yaml     |  2 +-
 deploy/charts/emqx-enterprise/values.yaml      |  9 ++++++++-
 deploy/charts/emqx/templates/service.yaml      |  2 +-
 4 files changed, 28 insertions(+), 3 deletions(-)
 create mode 100644 deploy/charts/emqx-enterprise/templates/pdb.yaml

diff --git a/deploy/charts/emqx-enterprise/templates/pdb.yaml b/deploy/charts/emqx-enterprise/templates/pdb.yaml
new file mode 100644
index 000000000..a3f233064
--- /dev/null
+++ b/deploy/charts/emqx-enterprise/templates/pdb.yaml
@@ -0,0 +1,18 @@
+{{- if and (.Values.pdb.enabled) (.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "emqx.fullname" . }}-pdb
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  maxUnavailable: {{ .Values.pdb.maxUnavailable }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "emqx.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
diff --git a/deploy/charts/emqx-enterprise/templates/service.yaml b/deploy/charts/emqx-enterprise/templates/service.yaml
index 0fe3dc411..233e69b10 100644
--- a/deploy/charts/emqx-enterprise/templates/service.yaml
+++ b/deploy/charts/emqx-enterprise/templates/service.yaml
@@ -121,7 +121,7 @@ spec:
     port: {{ .Values.service.mqtt | default 1883 }}
     protocol: TCP
     targetPort: mqtt
-    {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }}
+    {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }}
   - name: internalmqtt
     port: {{ .Values.service.internalmqtt | default 11883 }}
     protocol: TCP
diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml
index b3a77682f..9ae863219 100644
--- a/deploy/charts/emqx-enterprise/values.yaml
+++ b/deploy/charts/emqx-enterprise/values.yaml
@@ -102,7 +102,7 @@ initContainers: {}
   #       sysctl -w net.netfilter.nf_conntrack_max=1000000
   #       sysctl -w net.netfilter.nf_conntrack_tcp_timeout_time_wait=30
 
-## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
+## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx-enterprise)
 emqxConfig:
   EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns"
   EMQX_DASHBOARD__DEFAULT_USERNAME: "admin"
@@ -229,3 +229,10 @@ ssl:
   issuer:
     name: letsencrypt-dns
     kind: ClusterIssuer
+
+## Setting PodDisruptionBudget.
+## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb
+##
+pdb:
+  enabled: false
+  maxUnavailable: 1
diff --git a/deploy/charts/emqx/templates/service.yaml b/deploy/charts/emqx/templates/service.yaml
index 5b6376a85..233e69b10 100644
--- a/deploy/charts/emqx/templates/service.yaml
+++ b/deploy/charts/emqx/templates/service.yaml
@@ -38,7 +38,7 @@ spec:
     {{- else if eq .Values.service.type "ClusterIP" }}
     nodePort: null
     {{- end }}
-    {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }}
+    {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }}
   - name: internalmqtt
     port: {{ .Values.service.internalmqtt | default 11883 }}
     protocol: TCP

From beb5a238e76eab44eea6a796534c54c1be6196c8 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Tue, 21 Feb 2023 11:41:06 +0100
Subject: [PATCH 142/144] chore(charts): add readme and a script to sync
 changes

---
 deploy/charts/README.md          | 3 +++
 deploy/charts/sync-enterprise.sh | 6 ++++++
 2 files changed, 9 insertions(+)
 create mode 100644 deploy/charts/README.md
 create mode 100755 deploy/charts/sync-enterprise.sh

diff --git a/deploy/charts/README.md b/deploy/charts/README.md
new file mode 100644
index 000000000..4b8829056
--- /dev/null
+++ b/deploy/charts/README.md
@@ -0,0 +1,3 @@
+# Sync changes to emqx-enterprise
+
+When making changes in charts, please update `emqx` charts and run `./sync-enterprise.sh`.
diff --git a/deploy/charts/sync-enterprise.sh b/deploy/charts/sync-enterprise.sh
new file mode 100755
index 000000000..587871c0d
--- /dev/null
+++ b/deploy/charts/sync-enterprise.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+sed 's|emqx/emqx|emqx/emqx-enterprise|' < emqx/values.yaml > emqx-enterprise/values.yaml
+cp emqx/templates/* emqx-enterprise/templates

From 6fd11904ee534a75cea2ea27a1fdbcc061bd1c08 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Tue, 21 Feb 2023 11:43:25 +0100
Subject: [PATCH 143/144] chore: add changelog for #9213

---
 changes/ce/feat-9213.en.md | 1 +
 changes/ce/feat-9213.zh.md | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changes/ce/feat-9213.en.md
 create mode 100644 changes/ce/feat-9213.zh.md

diff --git a/changes/ce/feat-9213.en.md b/changes/ce/feat-9213.en.md
new file mode 100644
index 000000000..3266ed836
--- /dev/null
+++ b/changes/ce/feat-9213.en.md
@@ -0,0 +1 @@
+Add pod disruption budget to helm chart
diff --git a/changes/ce/feat-9213.zh.md b/changes/ce/feat-9213.zh.md
new file mode 100644
index 000000000..509b1e01c
--- /dev/null
+++ b/changes/ce/feat-9213.zh.md
@@ -0,0 +1 @@
+在舵手图中添加吊舱干扰预算。

From 1744b8bb7bf789eaa8a464bc9e9eaa69f85057f1 Mon Sep 17 00:00:00 2001
From: Ivan Dyachkov <dev@dyachkov.org>
Date: Tue, 21 Feb 2023 11:46:33 +0100
Subject: [PATCH 144/144] chore: add changelog to ee

---
 changes/ee/feat-10011.en.md | 1 +
 changes/ee/feat-10011.zh.md | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 changes/ee/feat-10011.en.md
 create mode 100644 changes/ee/feat-10011.zh.md

diff --git a/changes/ee/feat-10011.en.md b/changes/ee/feat-10011.en.md
new file mode 100644
index 000000000..3266ed836
--- /dev/null
+++ b/changes/ee/feat-10011.en.md
@@ -0,0 +1 @@
+Add pod disruption budget to helm chart
diff --git a/changes/ee/feat-10011.zh.md b/changes/ee/feat-10011.zh.md
new file mode 100644
index 000000000..509b1e01c
--- /dev/null
+++ b/changes/ee/feat-10011.zh.md
@@ -0,0 +1 @@
+在舵手图中添加吊舱干扰预算。