From 6349b5e5e035a88a96bfdd672da09676f4064bdf Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 14 Jun 2022 10:24:41 +0200 Subject: [PATCH 1/5] feat(quic): bump to quicer 0.0.11 --- apps/emqx/rebar.config | 2 +- apps/emqx/rebar.config.script | 31 ++++++++++--------- apps/emqx/src/emqx_listeners.erl | 5 ++- apps/emqx/src/emqx_quic_connection.erl | 3 +- apps/emqx/src/emqx_quic_stream.erl | 25 +++++++-------- .../emqx/test/emqx_mqtt_protocol_v5_SUITE.erl | 3 +- mix.exs | 2 +- rebar.config | 2 +- rebar.config.erl | 2 +- 9 files changed, 40 insertions(+), 35 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 011204b95..9834563a7 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -43,7 +43,7 @@ {meck, "0.9.2"}, {proper, "1.4.0"}, {bbmustache, "1.10.0"}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.5.0"}}} + {emqtt, {git, "https://github.com/qzhuyan/emqtt", {branch, "dev/william/bump-quicer"}}} ]}, {extra_src_dirs, [{"test", [recursive]}]} ]} diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index afcaff941..888e911f3 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -1,16 +1,16 @@ %% -*- mode: erlang -*- IsCentos6 = fun() -> - case file:read_file("/etc/centos-release") of - {ok, <<"CentOS release 6", _/binary >>} -> - true; - _ -> - false - end - end, + case file:read_file("/etc/centos-release") of + {ok, <<"CentOS release 6", _/binary>>} -> + true; + _ -> + false + end +end, IsWin32 = fun() -> - win32 =:= element(1, os:type()) - end, + win32 =:= element(1, os:type()) +end, IsMacOS = fun() -> {unix, darwin} =:= os:type() @@ -25,13 +25,14 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.9"}}}, +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.11"}}}. ExtraDeps = fun(C) -> - {deps, Deps0} = lists:keyfind(deps, 1, C), - Deps = Deps0 ++ [Bcrypt || not IsWin32()] ++ - [ Quicer || IsQuicSupp()], - lists:keystore(deps, 1, C, {deps, Deps}) - end, + {deps, Deps0} = lists:keyfind(deps, 1, C), + Deps = + Deps0 ++ [Bcrypt || not IsWin32()] ++ + [Quicer || IsQuicSupp()], + lists:keystore(deps, 1, C, {deps, Deps}) +end, ExtraDeps(CONFIG). diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 087555b1c..9d46fe975 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -324,15 +324,18 @@ do_start_listener(quic, ListenerName, #{bind := ListenOn} = Opts) -> case [A || {quicer, _, _} = A <- application:which_applications()] of [_] -> DefAcceptors = erlang:system_info(schedulers_online) * 8, + IdleTimeout = timer:seconds(maps:get(idle_timeout, Opts)), ListenOpts = [ {cert, maps:get(certfile, Opts)}, {key, maps:get(keyfile, Opts)}, {alpn, ["mqtt"]}, {conn_acceptors, lists:max([DefAcceptors, maps:get(acceptors, Opts, 0)])}, + {keep_alive_interval_ms, ceil(IdleTimeout / 3)}, + {server_resumption_level, 2}, {idle_timeout_ms, lists:max([ emqx_config:get_zone_conf(zone(Opts), [mqtt, idle_timeout]) * 3, - timer:seconds(maps:get(idle_timeout, Opts)) + IdleTimeout ])} ], ConnectionOpts = #{ diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index f5c281242..b44a4f1c2 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -15,6 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_quic_connection). +-include_lib("quicer/include/quicer.hrl"). %% Callbacks -export([ @@ -59,5 +60,5 @@ connected(_Conn, S) -> -spec shutdown(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}. shutdown(Conn, S) -> - quicer:async_close_connection(Conn), + quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), {ok, S}. diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index e98ea0137..bed5805e2 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -16,6 +16,7 @@ %% MQTT/QUIC Stream -module(emqx_quic_stream). +-include_lib("quicer/include/quicer.hrl"). %% emqx transport Callbacks -export([ @@ -38,7 +39,7 @@ wait({ConnOwner, Conn}) -> receive %% from msquic {quic, new_stream, Stream} -> - {ok, Stream}; + {ok, {quic, Conn, Stream}}; {'EXIT', ConnOwner, _Reason} -> {error, enotconn} end. @@ -46,18 +47,18 @@ wait({ConnOwner, Conn}) -> type(_) -> quic. -peername(S) -> - quicer:peername(S). +peername({quic, Conn, _Stream}) -> + quicer:peername(Conn). -sockname(S) -> - quicer:sockname(S). +sockname({quic, Conn, _Stream}) -> + quicer:sockname(Conn). peercert(_S) -> %% @todo but unsupported by msquic nossl. -getstat(Socket, Stats) -> - case quicer:getstat(Socket, Stats) of +getstat({quic, Conn, _Stream}, Stats) -> + case quicer:getstat(Conn, Stats) of {error, _} -> {error, closed}; Res -> Res end. @@ -84,9 +85,9 @@ getopts(_Socket, _Opts) -> {buffer, 80000} ]}. -fast_close(Stream) -> - %% Stream might be closed already. - _ = quicer:async_close_stream(Stream), +fast_close({quic, _Conn, Stream}) -> + %% Flush send buffer, gracefully shutdown + quicer:async_shutdown_stream(Stream), ok. -spec ensure_ok_or_exit(atom(), list(term())) -> term(). @@ -102,9 +103,7 @@ ensure_ok_or_exit(Fun, Args = [Sock | _]) when is_atom(Fun), is_list(Args) -> Result end. -async_send(Stream, Data, Options) when is_list(Data) -> - async_send(Stream, iolist_to_binary(Data), Options); -async_send(Stream, Data, _Options) when is_binary(Data) -> +async_send({quic, _Conn, Stream}, Data, _Options) -> case quicer:send(Stream, Data) of {ok, _Len} -> ok; Other -> Other diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index 5ef43e738..88ccda452 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -252,7 +252,8 @@ t_connect_will_message(Config) -> {ok, _, [2]} = emqtt:subscribe(Client4, Topic, qos2), ok = emqtt:disconnect(Client3), %% [MQTT-3.1.2-10] - ?assertEqual(0, length(receive_messages(1))), + MsgRecv = receive_messages(1), + ?assertEqual([], MsgRecv), ok = emqtt:disconnect(Client4). t_batch_subscribe(init, Config) -> diff --git a/mix.exs b/mix.exs index 0c3a4ff37..8125ee9f1 100644 --- a/mix.exs +++ b/mix.exs @@ -600,7 +600,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.9", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.11", override: true}], else: [] end diff --git a/rebar.config b/rebar.config index 744feb8de..56c427678 100644 --- a/rebar.config +++ b/rebar.config @@ -60,7 +60,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} , {replayq, "0.3.4"} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.5.1"}}} + , {emqtt, {git, "https://github.com/qzhuyan/emqtt", {branch, "dev/william/bump-quicer"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 0700a0bad..1d04d773f 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -38,7 +38,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.9"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.11"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.4"}}}. From 96a510fbf132d1973669f66d9c7d98f4e056c16d Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 9 Jun 2022 17:52:27 +0200 Subject: [PATCH 2/5] feat(quic): fix some listener --- apps/emqx/src/emqx_listeners.erl | 10 +++++++--- apps/emqx/test/emqx_common_test_helpers.erl | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 9d46fe975..e5b7e4293 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -143,9 +143,13 @@ is_running(Type, ListenerId, _Conf) when Type =:= ws; Type =:= wss -> _:_ -> false end; -is_running(quic, _ListenerId, _Conf) -> - %% TODO: quic support - false. +is_running(quic, ListenerId, _Conf) -> + case quicer:listener(ListenerId) of + {ok, Pid} when is_pid(Pid) -> + true; + _ -> + false + end. current_conns(ID, ListenOn) -> {ok, #{type := Type, name := Name}} = parse_listener_id(ID), diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index dd9a56841..89687905d 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -526,7 +526,6 @@ ensure_dashboard_listeners_started(_App) -> -spec ensure_quic_listener(Name :: atom(), UdpPort :: inet:port_number()) -> ok. ensure_quic_listener(Name, UdpPort) -> application:ensure_all_started(quicer), - emqx_config:put([listeners, quic, Name, mountpoint], <<>>), Conf = #{ acceptors => 16, bind => {{0, 0, 0, 0}, UdpPort}, @@ -545,6 +544,7 @@ ensure_quic_listener(Name, UdpPort) -> mountpoint => <<>>, zone => default }, + emqx_config:put([listeners, quic, Name], Conf), case emqx_listeners:start_listener(quic, Name, Conf) of ok -> ok; {error, {already_started, _Pid}} -> ok From 45e2605485e2f7a525e7561d1d668144d5ad696e Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 15 Jun 2022 11:18:24 +0200 Subject: [PATCH 3/5] build: workaround for windows build --- apps/emqx/rebar.config.script | 32 ++++++++++++++++---------- apps/emqx/src/emqx_quic_connection.erl | 5 ++++ apps/emqx/src/emqx_quic_stream.erl | 1 - apps/emqx_connector/rebar.config | 2 +- mix.exs | 3 ++- rebar.config.erl | 3 ++- 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 888e911f3..136e79190 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -13,26 +13,34 @@ IsWin32 = fun() -> end, IsMacOS = fun() -> - {unix, darwin} =:= os:type() - end, + {unix, darwin} =:= os:type() +end, IsQuicSupp = fun() -> - not (IsCentos6() orelse IsWin32() - orelse IsMacOS() orelse - false =/= os:getenv("BUILD_WITHOUT_QUIC") - ) - orelse "1" == os:getenv("BUILD_WITH_QUIC") - end, + not (IsCentos6() orelse IsWin32() orelse + IsMacOS() orelse + false =/= os:getenv("BUILD_WITHOUT_QUIC")) orelse + "1" == os:getenv("BUILD_WITH_QUIC") +end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.11"}}}. ExtraDeps = fun(C) -> {deps, Deps0} = lists:keyfind(deps, 1, C), - Deps = - Deps0 ++ [Bcrypt || not IsWin32()] ++ - [Quicer || IsQuicSupp()], - lists:keystore(deps, 1, C, {deps, Deps}) + {erl_opts, ErlOpts0} = lists:keyfind(erl_opts, 1, C), + IsQuic = IsQuicSupp(), + New = [ + {deps, Deps0 ++ [Bcrypt || not IsWin32()] ++ [Quicer || IsQuic]}, + {erl_opts, ErlOpts0 ++ [{d, 'BUILD_WITHOUT_QUIC'} || not IsQuic]} + ], + lists:foldl( + fun({Key, _Val} = KV, Acc) -> + lists:keystore(Key, 1, Acc, KV) + end, + C, + New + ) end, ExtraDeps(CONFIG). diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index b44a4f1c2..e9a392187 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -15,7 +15,12 @@ %%-------------------------------------------------------------------- -module(emqx_quic_connection). + +-ifndef(BUILD_WITHOUT_QUIC). -include_lib("quicer/include/quicer.hrl"). +-else. +-define(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0). +-endif. %% Callbacks -export([ diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index bed5805e2..4f40902ab 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -16,7 +16,6 @@ %% MQTT/QUIC Stream -module(emqx_quic_stream). --include_lib("quicer/include/quicer.hrl"). %% emqx transport Callbacks -export([ diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index eb1a62773..b114e0884 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -21,7 +21,7 @@ %% eredis_cluster's dependency getting resolved earlier. %% Here we pin 1.5.2 to avoid surprises in the future. {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.5.1"}}} + {emqtt, {git, "https://github.com/qzhuyan/emqtt", {branch, "dev/william/bump-quicer"}}} ]}. {shell, [ diff --git a/mix.exs b/mix.exs index 8125ee9f1..372e79d94 100644 --- a/mix.exs +++ b/mix.exs @@ -58,7 +58,8 @@ defmodule EMQXUmbrella.MixProject do {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, {:replayq, "0.3.4", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, - {:emqtt, github: "emqx/emqtt", tag: "1.5.1", override: true}, + # {:emqtt, github: "emqx/emqtt", tag: "1.5.1", override: true}, + {:emqtt, github: "qzhuyan/emqtt", branch: "dev/william/bump-quicer", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, diff --git a/rebar.config.erl b/rebar.config.erl index 1d04d773f..58ebb24ef 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -138,7 +138,8 @@ common_compile_opts(Edition, Vsn) -> {compile_info, [{emqx_vsn, Vsn}]}, {d, 'EMQX_RELEASE_EDITION', Edition} ] ++ - [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"]. + [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"] ++ + [{d, 'BUILD_WITHOUT_QUIC'} || not is_quicer_supported()]. prod_compile_opts(Edition, Vsn) -> [ From 2908c06c549aee6529140d8be744569c0151cc5c Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 15 Jun 2022 13:53:36 +0200 Subject: [PATCH 4/5] fix(quic): default quic listener --- apps/emqx/etc/emqx.conf | 2 +- apps/emqx/src/emqx_app.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index bd3ae4af0..43dcfd411 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -31,7 +31,7 @@ listeners.wss.default { } # listeners.quic.default { -# enabled = false +# enabled = true # bind = "0.0.0.0:14567" # max_connections = 1024000 # keyfile = "{{ platform_etc_dir }}/certs/key.pem" diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index ce09cf460..b6194eb31 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -113,7 +113,7 @@ is_quicer_app_present() -> end. is_quic_listener_configured() -> - emqx_listeners:has_enabled_listener_conf_by_type(quic). + maps:is_key(quic, emqx:get_config([listeners])). get_description() -> emqx_release:description(). From a911c27962c8155de2c7232f8751b5b0c20b005b Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 16 Jun 2022 10:38:46 +0200 Subject: [PATCH 5/5] feat(quic): bump emqtt to 1.6.0 with quic 0.0.11 --- apps/emqx/rebar.config | 2 +- apps/emqx_connector/rebar.config | 2 +- mix.exs | 3 +-- rebar.config | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 9834563a7..33f30dce0 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -43,7 +43,7 @@ {meck, "0.9.2"}, {proper, "1.4.0"}, {bbmustache, "1.10.0"}, - {emqtt, {git, "https://github.com/qzhuyan/emqtt", {branch, "dev/william/bump-quicer"}}} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.6.0"}}} ]}, {extra_src_dirs, [{"test", [recursive]}]} ]} diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index b114e0884..0ac2d0da8 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -21,7 +21,7 @@ %% eredis_cluster's dependency getting resolved earlier. %% Here we pin 1.5.2 to avoid surprises in the future. {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}}, - {emqtt, {git, "https://github.com/qzhuyan/emqtt", {branch, "dev/william/bump-quicer"}}} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.6.0"}}} ]}. {shell, [ diff --git a/mix.exs b/mix.exs index 372e79d94..939d3f418 100644 --- a/mix.exs +++ b/mix.exs @@ -58,8 +58,7 @@ defmodule EMQXUmbrella.MixProject do {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, {:replayq, "0.3.4", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, - # {:emqtt, github: "emqx/emqtt", tag: "1.5.1", override: true}, - {:emqtt, github: "qzhuyan/emqtt", branch: "dev/william/bump-quicer", override: true}, + {:emqtt, github: "emqx/emqtt", tag: "1.6.0", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, diff --git a/rebar.config b/rebar.config index 56c427678..1d583a7de 100644 --- a/rebar.config +++ b/rebar.config @@ -60,7 +60,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} , {replayq, "0.3.4"} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/qzhuyan/emqtt", {branch, "dev/william/bump-quicer"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.6.0"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}}