From 899ba579fc55a04e95b780ffec5d20d0942dbda2 Mon Sep 17 00:00:00 2001 From: William Yang Date: Sat, 27 Mar 2021 14:03:10 +0100 Subject: [PATCH 01/23] feat(quic): compile and start quicer listener. --- apps/emqx/etc/emqx.conf | 306 +++++++++++++++++++++++++++++++ apps/emqx/src/emqx_listeners.erl | 17 +- rebar.config | 1 + rebar.config.erl | 1 + 4 files changed, 324 insertions(+), 1 deletion(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index f6627bf1c..6be808264 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -2155,6 +2155,312 @@ listener.wss.external.allow_origin_absence = true ## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 listener.wss.external.check_origins = "https://localhost:8084, https://127.0.0.1:8084" + +##-------------------------------------------------------------------- +## External QUIC listener for MQTT Protocol + +## listener.quic.$name is the IP address and port that the MQTT/QUIC +## listener will bind. +## +## Value: IP:Port | Port +## +## Examples: 8084, 127.0.0.1:8084, ::1:8084 +listener.quic.external = 4567 + +## The path of WebSocket MQTT endpoint +## +## Value: URL Path +listener.quic.external.mqtt_path = /mqtt + +## The acceptor pool for external MQTT/QUIC listener. +## +## Value: Number +listener.quic.external.acceptors = 4 + +## Maximum number of concurrent MQTT/Webwocket/SSL connections. +## +## Value: Number +listener.quic.external.max_connections = 16 + +## Maximum MQTT/QUIC connections per second. +## +## See: listener.tcp.$name.max_conn_rate +## +## Value: Number +listener.quic.external.max_conn_rate = 1000 + +## Simulate the {active, N} option for the MQTT/QUIC connections. +## +## Value: Number +listener.quic.external.active_n = 100 + +## Zone of the external MQTT/QUIC listener belonged to. +## +## Value: String +listener.quic.external.zone = external + +## The access control rules for the MQTT/QUIC listener. +## +## See: listener.tcp.$name.access. +## +## Value: ACL Rule +listener.quic.external.access.1 = allow all + +## If set to true, the server fails if the client does not have a Sec-WebSocket-Protocol to send. +## Set to false for WeChat MiniApp. +## +## Value: true | false +## listener.quic.external.fail_if_no_subprotocol = true + +## Supported subprotocols +## +## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 +## listener.quic.external.supported_subprotocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 + +## Enable the Proxy Protocol V1/2 support. +## +## See: listener.tcp.$name.proxy_protocol +## +## Value: on | off +## listener.quic.external.proxy_protocol = on + +## Sets the timeout for proxy protocol. +## +## See: listener.tcp.$name.proxy_protocol_timeout +## +## Value: Duration +## listener.quic.external.proxy_protocol_timeout = 3s + +## TLS versions only to protect from POODLE attack. +## +## See: listener.ssl.$name.tls_versions +## +## Value: String, seperated by ',' +## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier +## listener.quic.external.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 + +## Path to the file containing the user's private PEM-encoded key. +## +## See: listener.ssl.$name.keyfile +## +## Value: File +listener.quic.external.keyfile = {{ platform_etc_dir }}/certs/key.pem + +## Path to a file containing the user certificate. +## +## See: listener.ssl.$name.certfile +## +## Value: File +listener.quic.external.certfile = {{ platform_etc_dir }}/certs/cert.pem + +## Path to the file containing PEM-encoded CA certificates. +## +## See: listener.ssl.$name.cacert +## +## Value: File +## listener.quic.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem + +## Maximum number of non-self-issued intermediate certificates that +## can follow the peer certificate in a valid certification path. +## +## See: listener.ssl.external.depth +## +## Value: Number +## listener.quic.external.depth = 10 + +## String containing the user's password. Only used if the private keyfile +## is password-protected. +## +## See: listener.ssl.$name.key_password +## +## Value: String +## listener.quic.external.key_password = yourpass + +## See: listener.ssl.$name.dhfile +## +## Value: File +## listener.ssl.external.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem + +## See: listener.ssl.$name.verify +## +## Value: verify_peer | verify_none +## listener.quic.external.verify = verify_peer + +## See: listener.ssl.$name.fail_if_no_peer_cert +## +## Value: false | true +## listener.quic.external.fail_if_no_peer_cert = true + +## See: listener.ssl.$name.ciphers +## +## Value: Ciphers +listener.quic.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA + +## Ciphers for TLS PSK. +## Note that 'listener.quic.external.ciphers' and 'listener.quic.external.psk_ciphers' cannot +## be configured at the same time. +## See 'https://tools.ietf.org/html/rfc4279#section-2'. +## listener.quic.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA + +## See: listener.ssl.$name.secure_renegotiate +## +## Value: on | off +## listener.quic.external.secure_renegotiate = off + +## See: listener.ssl.$name.reuse_sessions +## +## Value: on | off +## listener.quic.external.reuse_sessions = on + +## See: listener.ssl.$name.honor_cipher_order +## +## Value: on | off +## listener.quic.external.honor_cipher_order = on + +## See: listener.ssl.$name.peer_cert_as_username +## +## Value: cn | dn | crt | pem | md5 +## listener.quic.external.peer_cert_as_username = cn + +## See: listener.ssl.$name.peer_cert_as_clientid +## +## Value: cn | dn | crt | pem | md5 +## listener.quic.external.peer_cert_as_clientid = cn + +## TCP backlog for the QUIC connection. +## +## See: listener.tcp.$name.backlog +## +## Value: Number >= 0 +listener.quic.external.backlog = 1024 + +## The TCP send timeout for the QUIC connection. +## +## See: listener.tcp.$name.send_timeout +## +## Value: Duration +listener.quic.external.send_timeout = 15s + +## Close the QUIC connection if send timeout. +## +## See: listener.tcp.$name.send_timeout_close +## +## Value: on | off +listener.quic.external.send_timeout_close = on + +## The TCP receive buffer(os kernel) for the QUIC connections. +## +## See: listener.tcp.$name.recbuf +## +## Value: Bytes +## listener.quic.external.recbuf = 4KB + +## The TCP send buffer(os kernel) for the QUIC connections. +## +## See: listener.tcp.$name.sndbuf +## +## Value: Bytes +## listener.quic.external.sndbuf = 4KB + +## The size of the user-level software buffer used by the driver. +## +## See: listener.tcp.$name.buffer +## +## Value: Bytes +## listener.quic.external.buffer = 4KB + +## The TCP_NODELAY flag for QUIC connections. +## +## See: listener.tcp.$name.nodelay +## +## Value: true | false +## listener.quic.external.nodelay = true + +## The compress flag for external QUIC connections. +## +## If this Value is set true,the websocket message would be compressed +## +## Value: true | false +## listener.quic.external.compress = true + +## The level of deflate options for external QUIC connections. +## +## See: listener.quic.$name.deflate_opts.level +## +## Value: none | default | best_compression | best_speed +## listener.quic.external.deflate_opts.level = default + +## The mem_level of deflate options for external QUIC connections. +## +## See: listener.quic.$name.deflate_opts.mem_level +## +## Valid range is 1-9 +## listener.quic.external.deflate_opts.mem_level = 8 + +## The strategy of deflate options for external QUIC connections. +## +## See: listener.quic.$name.deflate_opts.strategy +## +## Value: default | filtered | huffman_only | rle +## listener.quic.external.deflate_opts.strategy = default + +## The deflate option for external QUIC connections. +## +## See: listener.quic.$name.deflate_opts.server_context_takeover +## +## Value: takeover | no_takeover +## listener.quic.external.deflate_opts.server_context_takeover = takeover + +## The deflate option for external QUIC connections. +## +## See: listener.quic.$name.deflate_opts.client_context_takeover +## +## Value: takeover | no_takeover +## listener.quic.external.deflate_opts.client_context_takeover = takeover + +## The deflate options for external QUIC connections. +## +## See: listener.quic.$name.deflate_opts.server_max_window_bits +## +## Valid range is 8-15 +## listener.quic.external.deflate_opts.server_max_window_bits = 15 + +## The deflate options for external QUIC connections. +## +## See: listener.quic.$name.deflate_opts.client_max_window_bits +## +## Valid range is 8-15 +## listener.quic.external.deflate_opts.client_max_window_bits = 15 + +## The idle timeout for external QUIC connections. +## +## See: listener.quic.$name.idle_timeout +## +## Value: Duration +## listener.quic.external.idle_timeout = 60s + +## The max frame size for external QUIC connections. +## +## Value: Number +## listener.quic.external.max_frame_size = 0 + +## Whether a WebSocket message is allowed to contain multiple MQTT packets +## +## Value: single | multiple +listener.quic.external.mqtt_piggyback = multiple +## Enable origin check in header for secure websocket connection +## +## Value: true | false (default false) +listener.quic.external.check_origin_enable = false +## Allow origin to be absent in header in secure websocket connection when check_origin_enable is true +## +## Value: true | false (default true) +listener.quic.external.allow_origin_absence = true +## Comma separated list of allowed origin in header for secure websocket connection +## +## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 +listener.quic.external.check_origins = https://localhost:8084, https://127.0.0.1:8084 + ## CONFIG_SECTION_END=listeners ================================================ ## CONFIG_SECTION_BGN=modules ================================================== diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 1f3d1776b..d97fe32ed 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -139,7 +139,22 @@ start_listener(Proto, ListenOn, Options) when Proto == http; Proto == ws -> %% Start MQTT/WSS listener start_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss -> start_http_listener(fun cowboy:start_tls/3, 'mqtt:wss', ListenOn, - ranch_opts(Options), ws_opts(Options)). + ranch_opts(Options), ws_opts(Options)); + +%% MQTT over QUIC +start_listener(quic, ListenOn, Options) -> + SSLOpts = proplists:get_value(ssl_options, Options), + ListenOpts = [ {cert, proplists:get_value(certfile, SSLOpts)} + , {key, proplists:get_value(keyfile, SSLOpts)} + , {alpn, ["mqtt"]} + , {conn_acceptors, 32} + ], + ConnectionOpts = [ {conn_callback, quicer_server_conn_callback} + , {idle_timeout_ms, 5000} + , {peer_unidi_stream_count, 1} + , {peer_bidi_stream_count, 10}], + StreamOpts = [{stream_callback, quicer_echo_server_stream_callback}], + quicer:start_listener('mqtt:quic', ListenOn, {ListenOpts, ConnectionOpts, StreamOpts}). replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)]. diff --git a/rebar.config b/rebar.config index 109b680c5..eaa0ea6cf 100644 --- a/rebar.config +++ b/rebar.config @@ -54,6 +54,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} + , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "quicer_application"}}} ]}. {xref_ignores, diff --git a/rebar.config.erl b/rebar.config.erl index 0248e6dab..a05b09686 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -241,6 +241,7 @@ relx_apps(ReleaseType) -> , compiler , runtime_tools , cuttlefish + , quicer , emqx , {mnesia, load} , {ekka, load} From 70f22d2c1b980e885f96354fea35dae57fbd75ee Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 30 Mar 2021 23:39:09 +0200 Subject: [PATCH 02/23] feat(quic): reuse emqx_connection module for quic. --- apps/emqx/src/emqx_connection.erl | 7 +++ apps/emqx/src/emqx_listeners.erl | 4 +- apps/emqx/src/emqx_quic_connection.erl | 26 ++++++++ apps/emqx/src/emqx_quic_stream.erl | 83 ++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 apps/emqx/src/emqx_quic_connection.erl create mode 100644 apps/emqx/src/emqx_quic_stream.erl diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index ab91c02b4..6900e4f1e 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -416,6 +416,13 @@ handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> ok = emqx_metrics:inc('bytes.received', Oct), parse_incoming(Data, State); +handle_msg({quic, Data, _Sock, _, _, _}, State) -> + ?LOG(debug, "RECV ~0p", [Data]), + Oct = iolist_size(Data), + inc_counter(incoming_bytes, Oct), + ok = emqx_metrics:inc('bytes.received', Oct), + parse_incoming(Data, State); + handle_msg({incoming, Packet = ?CONNECT_PACKET(ConnPkt)}, State = #state{idle_timer = IdleTimer}) -> ok = emqx_misc:cancel_timer(IdleTimer), diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index d97fe32ed..b3d6bf319 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -149,11 +149,11 @@ start_listener(quic, ListenOn, Options) -> , {alpn, ["mqtt"]} , {conn_acceptors, 32} ], - ConnectionOpts = [ {conn_callback, quicer_server_conn_callback} + ConnectionOpts = [ {conn_callback, emqx_quic_connection} , {idle_timeout_ms, 5000} , {peer_unidi_stream_count, 1} , {peer_bidi_stream_count, 10}], - StreamOpts = [{stream_callback, quicer_echo_server_stream_callback}], + StreamOpts = [], quicer:start_listener('mqtt:quic', ListenOn, {ListenOpts, ConnectionOpts, StreamOpts}). replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)]. diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl new file mode 100644 index 000000000..b83522c6e --- /dev/null +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -0,0 +1,26 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_quic_connection). + +%% Callbacks +-export([ new_conn/2 + ]). + +new_conn(Conn, {_L, COpts, _S}) when is_map(COpts) -> + new_conn(Conn, maps:to_list(COpts)); +new_conn(Conn, COpts) -> + emqx_connection:start_link(emqx_quic_stream, Conn, COpts). diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl new file mode 100644 index 000000000..e12d95f30 --- /dev/null +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -0,0 +1,83 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% MQTT/QUIC Stream +-module(emqx_quic_stream). + +%% emqx transport Callbacks +-export([ type/1 + , wait/1 + , getstat/2 + , fast_close/1 + , ensure_ok_or_exit/2 + , async_send/3 + , setopts/2 + , getopts/2 + , peername/1 + , sockname/1 + , peercert/1 + ]). + +wait(Conn) -> + quicer:accept_stream(Conn, []). + +type(_) -> + quic. + +peername(S) -> + quicer:peername(S). + +sockname(S) -> + quicer:sockname(S). + +peercert(_S) -> + nossl. + +getstat(Socket, Stats) -> + Res = quicer:getstats(Socket, Stats), + {ok, lists:keyreplace(send_pend, 1, Res, {send_pend, 0})}. + +setopts(_Socket, _Opts) -> + ok. + +getopts(_Socket, _Opts) -> + %% todo + { ok, [{high_watermark, 0}, + {high_msgq_watermark, 0}, + {sndbuf, 0}, + {recbuf, 0}, + {buffer,80000}]}. + +fast_close(Stream) -> + quicer:close_stream(Stream). + +-spec(ensure_ok_or_exit(atom(), list(term())) -> term()). +ensure_ok_or_exit(Fun, Args = [Sock|_]) when is_atom(Fun), is_list(Args) -> + case erlang:apply(?MODULE, Fun, Args) of + {error, Reason} when Reason =:= enotconn; Reason =:= closed -> + fast_close(Sock), + exit(normal); + {error, Reason} -> + fast_close(Sock), + exit({shutdown, Reason}); + Result -> Result + end. + +async_send(Stream, Data, Options) when is_list(Data) -> + async_send(Stream, iolist_to_binary(Data), Options); +async_send(Stream, Data, _Options) when is_binary(Data) -> + {ok, _Len} = quicer:send(Stream, Data), + ok. From 087aa1dd53bb87f45b997e6b07e9caecb73f252c Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 31 Mar 2021 13:45:40 +0200 Subject: [PATCH 03/23] feat(quic): handle stream close. --- apps/emqx/etc/emqx.conf | 4 ++-- apps/emqx/src/emqx_connection.erl | 3 +++ apps/emqx/src/emqx_quic_stream.erl | 4 +++- rebar.config | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 6be808264..19100c41e 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -2164,8 +2164,8 @@ listener.wss.external.check_origins = "https://localhost:8084, https://127.0.0.1 ## ## Value: IP:Port | Port ## -## Examples: 8084, 127.0.0.1:8084, ::1:8084 -listener.quic.external = 4567 +## Examples: 14567, 127.0.0.1:14567, ::1:14567 +listener.quic.external = 14567 ## The path of WebSocket MQTT endpoint ## diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 6900e4f1e..8e3ee400b 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -738,6 +738,9 @@ handle_info({sock_error, Reason}, State) -> end, handle_info({sock_closed, Reason}, close_socket(State)); +handle_info({quic, closed, _Channel, ReasonFlag}, State) -> + handle_info({sock_closed, ReasonFlag}, State); + handle_info(Info, State) -> with_channel(handle_info, [Info], State). diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index e12d95f30..a80af643a 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -62,7 +62,9 @@ getopts(_Socket, _Opts) -> {buffer,80000}]}. fast_close(Stream) -> - quicer:close_stream(Stream). + quicer:close_stream(Stream), + %% Stream might be closed already. + ok. -spec(ensure_ok_or_exit(atom(), list(term())) -> term()). ensure_ok_or_exit(Fun, Args = [Sock|_]) when is_atom(Fun), is_list(Args) -> diff --git a/rebar.config b/rebar.config index eaa0ea6cf..ac0604728 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} - , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "quicer_application"}}} + , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "fix/getopt3-free-bin"}}} ]}. {xref_ignores, From 570e096b5690ece5aba7ad01304b589b8909f639 Mon Sep 17 00:00:00 2001 From: William Yang Date: Sat, 3 Apr 2021 11:22:47 +0200 Subject: [PATCH 04/23] fix(quic): return empty list for dead 'Socket' --- apps/emqx/src/emqx_quic_stream.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index a80af643a..1c99ad7da 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -47,8 +47,10 @@ peercert(_S) -> nossl. getstat(Socket, Stats) -> - Res = quicer:getstats(Socket, Stats), - {ok, lists:keyreplace(send_pend, 1, Res, {send_pend, 0})}. + case quicer:getstats(Socket, Stats) of + {error, _} -> []; + Res -> {ok, Res} + end. setopts(_Socket, _Opts) -> ok. From 9570d01792fbdd91301dddf511037a9d8e1dcc83 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 5 Apr 2021 14:28:00 +0200 Subject: [PATCH 05/23] fix(quic): error handling for getstats. - return {error, closed} instead - quicer demo/3 branch. --- apps/emqx/src/emqx_quic_stream.erl | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 1c99ad7da..056c5dd24 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -48,7 +48,7 @@ peercert(_S) -> getstat(Socket, Stats) -> case quicer:getstats(Socket, Stats) of - {error, _} -> []; + {error, _} -> {error, closed}; Res -> {ok, Res} end. diff --git a/rebar.config b/rebar.config index ac0604728..67efca3f6 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} - , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "fix/getopt3-free-bin"}}} + , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "demo/3"}}} ]}. {xref_ignores, From 6dc4088f7e169744056cb63c8683c3f054055100 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 5 May 2021 12:41:12 +0200 Subject: [PATCH 06/23] chore(ci): disable centos7 build --- .github/workflows/build_slim_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 6c9bbf04a..53bbba702 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -18,7 +18,7 @@ jobs: - erl23.2.7.2-emqx-2 os: - ubuntu20.04 - - centos7 + #- centos7 container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} From 06f9674ce3ce57d1fc0496729eacee744a8b46b7 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 5 May 2021 13:30:50 +0200 Subject: [PATCH 07/23] feat(quic): add quicer to application deps list. --- apps/emqx/src/emqx.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index e909702ae..a6984370e 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -4,7 +4,7 @@ {vsn, "5.0.0"}, % strict semver, bump manually! {modules, []}, {registered, []}, - {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]}, + {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon,quicer]}, {mod, {emqx_app,[]}}, {env, []}, {licenses, ["Apache-2.0"]}, From 14057c033d5d5d1bc6746b5e60f70cda17fe1169 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 7 May 2021 13:43:48 +0200 Subject: [PATCH 08/23] feat(quic): support stop/start quic listeners. --- apps/emqx/src/emqx.erl | 4 ++-- apps/emqx/src/emqx_app.erl | 3 +++ apps/emqx/src/emqx_listeners.erl | 4 +++- rebar.config | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 2d2e4eb52..449116738 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -239,10 +239,10 @@ reboot() -> -ifdef(EMQX_ENTERPRISE). default_started_applications() -> - [gproc, esockd, ranch, cowboy, ekka, emqx]. + [gproc, esockd, ranch, cowboy, ekka, quicer, emqx]. -else. default_started_applications() -> - [gproc, esockd, ranch, cowboy, ekka, emqx, emqx_modules]. + [gproc, esockd, ranch, cowboy, ekka, quicer, emqx, emqx_modules]. -endif. %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 26b81c8e7..d786a42b9 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -49,6 +49,9 @@ start(_Type, _Args) -> ok = emqx_plugins:init(), _ = emqx_plugins:load(), _ = start_ce_modules(), + %% @fixme unsure why we need this. + quicer_nif:open_lib(), + quicer_nif:reg_open(), emqx_boot:is_enabled(listeners) andalso (ok = emqx_listeners:start()), register(emqx, self()), ok = emqx_alarm_handler:load(), diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index b3d6bf319..2d7a77e65 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -141,7 +141,7 @@ start_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss -> start_http_listener(fun cowboy:start_tls/3, 'mqtt:wss', ListenOn, ranch_opts(Options), ws_opts(Options)); -%% MQTT over QUIC +%% Start MQTT/QUIC listener start_listener(quic, ListenOn, Options) -> SSLOpts = proplists:get_value(ssl_options, Options), ListenOpts = [ {cert, proplists:get_value(certfile, SSLOpts)} @@ -253,6 +253,8 @@ stop_listener(Proto, ListenOn, _Opts) when Proto == http; Proto == ws -> cowboy:stop_listener(ws_name('mqtt:ws', ListenOn)); stop_listener(Proto, ListenOn, _Opts) when Proto == https; Proto == wss -> cowboy:stop_listener(ws_name('mqtt:wss', ListenOn)); +stop_listener(quic, _ListenOn, _Opts) -> + quicer:stop_listener('mqtt:quic'); stop_listener(Proto, ListenOn, _Opts) -> esockd:close(Proto, ListenOn). diff --git a/rebar.config b/rebar.config index 67efca3f6..b602005f6 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} - , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "demo/3"}}} + , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/main-prepare-emqx"}}} ]}. {xref_ignores, From e062be2b0e1e4835358b6aa21d14b9d77cf372bf Mon Sep 17 00:00:00 2001 From: William Yang Date: Sat, 8 May 2021 23:31:29 +0200 Subject: [PATCH 09/23] feat(quic): reload quicer lib before start listener --- apps/emqx/src/emqx_app.erl | 3 --- apps/emqx/src/emqx_listeners.erl | 3 +++ apps/emqx/test/emqx_listeners_SUITE.erl | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index d786a42b9..26b81c8e7 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -49,9 +49,6 @@ start(_Type, _Args) -> ok = emqx_plugins:init(), _ = emqx_plugins:load(), _ = start_ce_modules(), - %% @fixme unsure why we need this. - quicer_nif:open_lib(), - quicer_nif:reg_open(), emqx_boot:is_enabled(listeners) andalso (ok = emqx_listeners:start()), register(emqx, self()), ok = emqx_alarm_handler:load(), diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 2d7a77e65..d66606895 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -143,6 +143,9 @@ start_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss -> %% Start MQTT/QUIC listener start_listener(quic, ListenOn, Options) -> + %% @fixme unsure why we need reopen lib and reopen config. + quicer_nif:open_lib(), + quicer_nif:reg_open(), SSLOpts = proplists:get_value(ssl_options, Options), ListenOpts = [ {cert, proplists:get_value(certfile, SSLOpts)} , {key, proplists:get_value(keyfile, SSLOpts)} diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 53f388dfa..41b9126b0 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -28,6 +28,7 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Config) -> NewConfig = generate_config(), application:ensure_all_started(esockd), + application:ensure_all_started(quicer), application:ensure_all_started(cowboy), lists:foreach(fun set_app_env/1, NewConfig), Config. From f9a113477e059831655ba9c1a57a683a667a434f Mon Sep 17 00:00:00 2001 From: William Yang Date: Sun, 9 May 2021 00:00:40 +0200 Subject: [PATCH 10/23] feat(quic): use quicer:getstat instead. --- apps/emqx/src/emqx_quic_stream.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 056c5dd24..22d194632 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -47,7 +47,7 @@ peercert(_S) -> nossl. getstat(Socket, Stats) -> - case quicer:getstats(Socket, Stats) of + case quicer:getstat(Socket, Stats) of {error, _} -> {error, closed}; Res -> {ok, Res} end. From 1ffd2cf2459479ed2e8c1465a9184999e4f6c904 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 11 May 2021 08:28:11 +0200 Subject: [PATCH 11/23] chore(config): adapt to new config format --- apps/emqx/etc/emqx.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 19100c41e..bb4c76534 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -2170,7 +2170,7 @@ listener.quic.external = 14567 ## The path of WebSocket MQTT endpoint ## ## Value: URL Path -listener.quic.external.mqtt_path = /mqtt +listener.quic.external.mqtt_path = "/mqtt" ## The acceptor pool for external MQTT/QUIC listener. ## @@ -2244,14 +2244,14 @@ listener.quic.external.access.1 = allow all ## See: listener.ssl.$name.keyfile ## ## Value: File -listener.quic.external.keyfile = {{ platform_etc_dir }}/certs/key.pem +listener.quic.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Path to a file containing the user certificate. ## ## See: listener.ssl.$name.certfile ## ## Value: File -listener.quic.external.certfile = {{ platform_etc_dir }}/certs/cert.pem +listener.quic.external.certfile = "{{ platform_etc_dir }}/certs/cert.pem" ## Path to the file containing PEM-encoded CA certificates. ## @@ -2294,7 +2294,7 @@ listener.quic.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## See: listener.ssl.$name.ciphers ## ## Value: Ciphers -listener.quic.external.ciphers = TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +listener.quic.external.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" ## Ciphers for TLS PSK. ## Note that 'listener.quic.external.ciphers' and 'listener.quic.external.psk_ciphers' cannot @@ -2459,7 +2459,7 @@ listener.quic.external.allow_origin_absence = true ## Comma separated list of allowed origin in header for secure websocket connection ## ## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 -listener.quic.external.check_origins = https://localhost:8084, https://127.0.0.1:8084 +listener.quic.external.check_origins = "https://localhost:8084, https://127.0.0.1:8084" ## CONFIG_SECTION_END=listeners ================================================ From 5356668eaccb33a6e689455cf6dd9ca680df65b2 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 10 Jun 2021 16:18:16 +0200 Subject: [PATCH 12/23] feat(quic): adapt to hocon schema --- apps/emqx/etc/emqx.conf | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index bb4c76534..fd6de554e 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -1841,7 +1841,7 @@ listener.ws.external.check_origins = "http://localhost:18083, http://127.0.0.1:1 ##-------------------------------------------------------------------- ## External WebSocket/SSL listener for MQTT Protocol -## listener.wss.$name is the IP address and port that the MQTT/WebSocket/SSL +## listener.wss.$name.endpoint is the IP address and port that the MQTT/WebSocket/SSL ## listener will bind. ## ## Value: IP:Port | Port @@ -2159,18 +2159,13 @@ listener.wss.external.check_origins = "https://localhost:8084, https://127.0.0.1 ##-------------------------------------------------------------------- ## External QUIC listener for MQTT Protocol -## listener.quic.$name is the IP address and port that the MQTT/QUIC +## listener.quic.$name.endpoint is the IP address and port that the MQTT/QUIC ## listener will bind. ## ## Value: IP:Port | Port ## ## Examples: 14567, 127.0.0.1:14567, ::1:14567 -listener.quic.external = 14567 - -## The path of WebSocket MQTT endpoint -## -## Value: URL Path -listener.quic.external.mqtt_path = "/mqtt" +listener.quic.external.endpoint = 14567 ## The acceptor pool for external MQTT/QUIC listener. ## @@ -2204,25 +2199,7 @@ listener.quic.external.zone = external ## See: listener.tcp.$name.access. ## ## Value: ACL Rule -listener.quic.external.access.1 = allow all - -## If set to true, the server fails if the client does not have a Sec-WebSocket-Protocol to send. -## Set to false for WeChat MiniApp. -## -## Value: true | false -## listener.quic.external.fail_if_no_subprotocol = true - -## Supported subprotocols -## -## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## listener.quic.external.supported_subprotocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 - -## Enable the Proxy Protocol V1/2 support. -## -## See: listener.tcp.$name.proxy_protocol -## -## Value: on | off -## listener.quic.external.proxy_protocol = on +listener.quic.external.access.1 = "allow all" ## Sets the timeout for proxy protocol. ## From 14614fbe33ee27a26e705945261994f8534374b9 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 11 May 2021 22:52:25 +0200 Subject: [PATCH 13/23] feat(quic): adapt to new quicer API. --- apps/emqx/src/emqx_listeners.erl | 2 ++ apps/emqx/src/emqx_quic_stream.erl | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index d66606895..6ee07e17b 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -150,6 +150,8 @@ start_listener(quic, ListenOn, Options) -> ListenOpts = [ {cert, proplists:get_value(certfile, SSLOpts)} , {key, proplists:get_value(keyfile, SSLOpts)} , {alpn, ["mqtt"]} + , {peer_unidi_stream_count, 1} + , {peer_bidi_stream_count, 10} , {conn_acceptors, 32} ], ConnectionOpts = [ {conn_callback, emqx_quic_connection} diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 22d194632..4fbe2ed65 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -49,7 +49,7 @@ peercert(_S) -> getstat(Socket, Stats) -> case quicer:getstat(Socket, Stats) of {error, _} -> {error, closed}; - Res -> {ok, Res} + Res -> Res end. setopts(_Socket, _Opts) -> From e63f86e5f0882266a0d35ff1e0737609195c157f Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 12 May 2021 12:17:04 +0200 Subject: [PATCH 14/23] Revert "chore(ci): disable centos7 build" This reverts commit 22e8da1b37492af7def46329ba97dd4cc7741bdf. --- .github/workflows/build_slim_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 53bbba702..6c9bbf04a 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -18,7 +18,7 @@ jobs: - erl23.2.7.2-emqx-2 os: - ubuntu20.04 - #- centos7 + - centos7 container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} From 3200bbb301d816ed7112c676e99b56f3cea0d25b Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 1 Jun 2021 09:53:48 +0200 Subject: [PATCH 15/23] fix(build): test with centos branch --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index b602005f6..394296e23 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} - , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/main-prepare-emqx"}}} + , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/main-prepare-emqx-centos7"}}} ]}. {xref_ignores, From bb6459ba3ada82dc1f74be8c1db78ee6a9928a01 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 11 Jun 2021 08:20:23 +0200 Subject: [PATCH 16/23] build: add quic dep in app/emqx/rebar.config --- apps/emqx/rebar.config | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 7e649ae80..d30946a02 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -20,6 +20,7 @@ , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} + , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/main-prepare-emqx-centos7"}}} ]}. {plugins, [rebar3_proper]}. From 68844cefd9a119420a0dc5f73fc8d76109ebc082 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 11 Jun 2021 11:12:07 +0200 Subject: [PATCH 17/23] feat(quic): update emqx_schema for quic --- apps/emqx/etc/emqx.conf | 220 +++++++++++++++++----------------- apps/emqx/src/emqx_schema.erl | 34 +++++- 2 files changed, 143 insertions(+), 111 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index fd6de554e..b115964cf 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -2184,43 +2184,43 @@ listener.quic.external.max_connections = 16 ## Value: Number listener.quic.external.max_conn_rate = 1000 -## Simulate the {active, N} option for the MQTT/QUIC connections. -## -## Value: Number -listener.quic.external.active_n = 100 +# ## Simulate the {active, N} option for the MQTT/QUIC connections. +# ## +# ## Value: Number +# listener.quic.external.active_n = 100 ## Zone of the external MQTT/QUIC listener belonged to. ## ## Value: String listener.quic.external.zone = external -## The access control rules for the MQTT/QUIC listener. -## -## See: listener.tcp.$name.access. -## -## Value: ACL Rule -listener.quic.external.access.1 = "allow all" +# ## The access control rules for the MQTT/QUIC listener. +# ## +# ## See: listener.tcp.$name.access. +# ## +# ## Value: ACL Rule +# listener.quic.external.access.1 = "allow all" -## Sets the timeout for proxy protocol. -## -## See: listener.tcp.$name.proxy_protocol_timeout -## -## Value: Duration -## listener.quic.external.proxy_protocol_timeout = 3s +# ## Sets the timeout for proxy protocol. +# ## +# ## See: listener.tcp.$name.proxy_protocol_timeout +# ## +# ## Value: Duration +# ## listener.quic.external.proxy_protocol_timeout = 3s -## TLS versions only to protect from POODLE attack. -## -## See: listener.ssl.$name.tls_versions -## -## Value: String, seperated by ',' -## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## listener.quic.external.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 +# ## TLS versions only to protect from POODLE attack. +# ## +# ## See: listener.ssl.$name.tls_versions +# ## +# ## Value: String, seperated by ',' +# ## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier +# ## listener.quic.external.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 -## Path to the file containing the user's private PEM-encoded key. -## -## See: listener.ssl.$name.keyfile -## -## Value: File +# ## Path to the file containing the user's private PEM-encoded key. +# ## +# ## See: listener.ssl.$name.keyfile +# ## +# ## Value: File listener.quic.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Path to a file containing the user certificate. @@ -2230,100 +2230,100 @@ listener.quic.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Value: File listener.quic.external.certfile = "{{ platform_etc_dir }}/certs/cert.pem" -## Path to the file containing PEM-encoded CA certificates. -## -## See: listener.ssl.$name.cacert -## -## Value: File -## listener.quic.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +# ## Path to the file containing PEM-encoded CA certificates. +# ## +# ## See: listener.ssl.$name.cacert +# ## +# ## Value: File +# ## listener.quic.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem -## Maximum number of non-self-issued intermediate certificates that -## can follow the peer certificate in a valid certification path. -## -## See: listener.ssl.external.depth -## -## Value: Number -## listener.quic.external.depth = 10 +# ## Maximum number of non-self-issued intermediate certificates that +# ## can follow the peer certificate in a valid certification path. +# ## +# ## See: listener.ssl.external.depth +# ## +# ## Value: Number +# ## listener.quic.external.depth = 10 -## String containing the user's password. Only used if the private keyfile -## is password-protected. -## -## See: listener.ssl.$name.key_password -## -## Value: String -## listener.quic.external.key_password = yourpass +# ## String containing the user's password. Only used if the private keyfile +# ## is password-protected. +# ## +# ## See: listener.ssl.$name.key_password +# ## +# ## Value: String +# ## listener.quic.external.key_password = yourpass -## See: listener.ssl.$name.dhfile -## -## Value: File -## listener.ssl.external.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem +# ## See: listener.ssl.$name.dhfile +# ## +# ## Value: File +# ## listener.ssl.external.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem -## See: listener.ssl.$name.verify -## -## Value: verify_peer | verify_none -## listener.quic.external.verify = verify_peer +# ## See: listener.ssl.$name.verify +# ## +# ## Value: verify_peer | verify_none +# ## listener.quic.external.verify = verify_peer -## See: listener.ssl.$name.fail_if_no_peer_cert -## -## Value: false | true -## listener.quic.external.fail_if_no_peer_cert = true +# ## See: listener.ssl.$name.fail_if_no_peer_cert +# ## +# ## Value: false | true +# ## listener.quic.external.fail_if_no_peer_cert = true -## See: listener.ssl.$name.ciphers -## -## Value: Ciphers +# ## See: listener.ssl.$name.ciphers +# ## +# ## Value: Ciphers listener.quic.external.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" -## Ciphers for TLS PSK. -## Note that 'listener.quic.external.ciphers' and 'listener.quic.external.psk_ciphers' cannot -## be configured at the same time. -## See 'https://tools.ietf.org/html/rfc4279#section-2'. -## listener.quic.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA +# ## Ciphers for TLS PSK. +# ## Note that 'listener.quic.external.ciphers' and 'listener.quic.external.psk_ciphers' cannot +# ## be configured at the same time. +# ## See 'https://tools.ietf.org/html/rfc4279#section-2'. +# ## listener.quic.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA -## See: listener.ssl.$name.secure_renegotiate -## -## Value: on | off -## listener.quic.external.secure_renegotiate = off +# ## See: listener.ssl.$name.secure_renegotiate +# ## +# ## Value: on | off +# ## listener.quic.external.secure_renegotiate = off -## See: listener.ssl.$name.reuse_sessions -## -## Value: on | off -## listener.quic.external.reuse_sessions = on +# ## See: listener.ssl.$name.reuse_sessions +# ## +# ## Value: on | off +# ## listener.quic.external.reuse_sessions = on -## See: listener.ssl.$name.honor_cipher_order -## -## Value: on | off -## listener.quic.external.honor_cipher_order = on +# ## See: listener.ssl.$name.honor_cipher_order +# ## +# ## Value: on | off +# ## listener.quic.external.honor_cipher_order = on -## See: listener.ssl.$name.peer_cert_as_username -## -## Value: cn | dn | crt | pem | md5 -## listener.quic.external.peer_cert_as_username = cn +# ## See: listener.ssl.$name.peer_cert_as_username +# ## +# ## Value: cn | dn | crt | pem | md5 +# ## listener.quic.external.peer_cert_as_username = cn -## See: listener.ssl.$name.peer_cert_as_clientid -## -## Value: cn | dn | crt | pem | md5 -## listener.quic.external.peer_cert_as_clientid = cn +# ## See: listener.ssl.$name.peer_cert_as_clientid +# ## +# ## Value: cn | dn | crt | pem | md5 +# ## listener.quic.external.peer_cert_as_clientid = cn -## TCP backlog for the QUIC connection. -## -## See: listener.tcp.$name.backlog -## -## Value: Number >= 0 -listener.quic.external.backlog = 1024 +# ## TCP backlog for the QUIC connection. +# ## +# ## See: listener.tcp.$name.backlog +# ## +# ## Value: Number >= 0 +# listener.quic.external.backlog = 1024 -## The TCP send timeout for the QUIC connection. -## -## See: listener.tcp.$name.send_timeout -## -## Value: Duration -listener.quic.external.send_timeout = 15s +# ## The TCP send timeout for the QUIC connection. +# ## +# ## See: listener.tcp.$name.send_timeout +# ## +# ## Value: Duration +# listener.quic.external.send_timeout = 15s -## Close the QUIC connection if send timeout. -## -## See: listener.tcp.$name.send_timeout_close -## -## Value: on | off -listener.quic.external.send_timeout_close = on +# ## Close the QUIC connection if send timeout. +# ## +# ## See: listener.tcp.$name.send_timeout_close +# ## +# ## Value: on | off +# listener.quic.external.send_timeout_close = on ## The TCP receive buffer(os kernel) for the QUIC connections. ## @@ -2424,19 +2424,19 @@ listener.quic.external.send_timeout_close = on ## Whether a WebSocket message is allowed to contain multiple MQTT packets ## ## Value: single | multiple -listener.quic.external.mqtt_piggyback = multiple +#listener.quic.external.mqtt_piggyback = multiple ## Enable origin check in header for secure websocket connection ## ## Value: true | false (default false) -listener.quic.external.check_origin_enable = false +#listener.quic.external.check_origin_enable = false ## Allow origin to be absent in header in secure websocket connection when check_origin_enable is true ## ## Value: true | false (default true) -listener.quic.external.allow_origin_absence = true +#listener.quic.external.allow_origin_absence = true ## Comma separated list of allowed origin in header for secure websocket connection ## ## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 -listener.quic.external.check_origins = "https://localhost:8084, https://127.0.0.1:8084" +#listener.quic.external.check_origins = "https://localhost:8084, https://127.0.0.1:8084" ## CONFIG_SECTION_END=listeners ================================================ diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 316f4b77c..d1c163e58 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -282,6 +282,7 @@ fields("listener") -> , {"ssl", ref("ssl_listener")} , {"ws", ref("ws_listener")} , {"wss", ref("wss_listener")} + , {"quic", ref("quic_listener")} ]; fields("tcp_listener") -> @@ -296,6 +297,9 @@ fields("ws_listener") -> fields("wss_listener") -> [ {"$name", ref("wss_listener_settings")}]; +fields("quic_listener") -> + [ {"$name", ref("quic_listener_settings")}]; + fields("listener_settings") -> [ {"endpoint", t(union(ip_port(), integer()))} , {"acceptors", t(integer(), undefined, 8)} @@ -356,6 +360,32 @@ fields("wss_listener_settings") -> Settings = lists:ukeymerge(1, Ssl, fields("ws_listener_settings")), lists:keydelete("high_watermark", 1, Settings); +fields("quic_listener_settings") -> + Unsupported = [ "max_connections" + , "max_conn_rate" + , "active_n" + , "access" + , "proxy_protocol" + , "proxy_protocol_timeout" + , "backlog" + , "send_timeout" + , "send_timeout_close" + , "recvbuf" + , "sndbuf" + , "buffer" + , "high_watermark" + , "tune_buffer" + , "nodelay" + , "reuseaddr" + ], + lists:foldl(fun(K, Acc) -> + lists:keydelete(K, 1, Acc) + end, + [ {"certfile", t(string(), "emqx.certfile", undefined)} + , {"keyfile", t(string(), "emqx.keyfile", undefined)} + | fields("listener_settings")], + Unsupported); + fields("access") -> [ {"$id", t(string(), undefined, undefined)}]; @@ -772,7 +802,9 @@ tr_listeners(Conf) -> lists:flatten([TcpListeners("tcp", Name) || Name <- keys("listener.tcp", Conf)] ++ [TcpListeners("ws", Name) || Name <- keys("listener.ws", Conf)] ++ [SslListeners("ssl", Name) || Name <- keys("listener.ssl", Conf)] - ++ [SslListeners("wss", Name) || Name <- keys("listener.wss", Conf)]). + ++ [SslListeners("wss", Name) || Name <- keys("listener.wss", Conf)] + ++ [SslListeners("quic", Name) || Name <- keys("listener.quic", Conf)] + ). tr_modules(Conf) -> Subscriptions = fun() -> From af2faed10740b14071b78a8ab1fc5a4f52247890 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 14 Jun 2021 11:29:23 +0200 Subject: [PATCH 18/23] feat(quic): switch to deps on emqx quicer repo --- apps/emqx/rebar.config | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index d30946a02..5719ec0af 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -20,7 +20,7 @@ , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} - , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/main-prepare-emqx-centos7"}}} + , {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.2"}}} ]}. {plugins, [rebar3_proper]}. diff --git a/rebar.config b/rebar.config index 394296e23..413158842 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} - , {quicer, {git, "https://github.com/qzhuyan/quic.git", {branch, "dev/william/main-prepare-emqx-centos7"}}} + , {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.2"}}} ]}. {xref_ignores, From e34470f9f2e045f76ee4012788c16a6b4b130870 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 15 Jun 2021 09:23:11 +0200 Subject: [PATCH 19/23] feat(quic): remove unsupported configs. --- apps/emqx/etc/emqx.conf | 262 +++++++++------------------------------- 1 file changed, 60 insertions(+), 202 deletions(-) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index b115964cf..f9c1d26ff 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -2184,43 +2184,21 @@ listener.quic.external.max_connections = 16 ## Value: Number listener.quic.external.max_conn_rate = 1000 -# ## Simulate the {active, N} option for the MQTT/QUIC connections. -# ## -# ## Value: Number -# listener.quic.external.active_n = 100 +## Simulate the {active, N} option for the MQTT/QUIC connections. +## @todo +## Value: Number +## listener.quic.external.active_n = 100 ## Zone of the external MQTT/QUIC listener belonged to. ## ## Value: String listener.quic.external.zone = external -# ## The access control rules for the MQTT/QUIC listener. -# ## -# ## See: listener.tcp.$name.access. -# ## -# ## Value: ACL Rule -# listener.quic.external.access.1 = "allow all" - -# ## Sets the timeout for proxy protocol. -# ## -# ## See: listener.tcp.$name.proxy_protocol_timeout -# ## -# ## Value: Duration -# ## listener.quic.external.proxy_protocol_timeout = 3s - -# ## TLS versions only to protect from POODLE attack. -# ## -# ## See: listener.ssl.$name.tls_versions -# ## -# ## Value: String, seperated by ',' -# ## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -# ## listener.quic.external.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1 - -# ## Path to the file containing the user's private PEM-encoded key. -# ## -# ## See: listener.ssl.$name.keyfile -# ## -# ## Value: File +## Path to the file containing the user's private PEM-encoded key. +## +## See: listener.ssl.$name.keyfile +## +## Value: File listener.quic.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Path to a file containing the user certificate. @@ -2230,214 +2208,94 @@ listener.quic.external.keyfile = "{{ platform_etc_dir }}/certs/key.pem" ## Value: File listener.quic.external.certfile = "{{ platform_etc_dir }}/certs/cert.pem" -# ## Path to the file containing PEM-encoded CA certificates. -# ## -# ## See: listener.ssl.$name.cacert -# ## -# ## Value: File -# ## listener.quic.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem +## Path to the file containing PEM-encoded CA certificates. +## @todo +## See: listener.ssl.$name.cacert +## +## Value: File +## listener.quic.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem -# ## Maximum number of non-self-issued intermediate certificates that -# ## can follow the peer certificate in a valid certification path. -# ## -# ## See: listener.ssl.external.depth -# ## -# ## Value: Number -# ## listener.quic.external.depth = 10 +## String containing the user's password. Only used if the private keyfile +## is password-protected. +## @todo +## See: listener.ssl.$name.key_password +## +## Value: String +## listener.quic.external.key_password = yourpass -# ## String containing the user's password. Only used if the private keyfile -# ## is password-protected. -# ## -# ## See: listener.ssl.$name.key_password -# ## -# ## Value: String -# ## listener.quic.external.key_password = yourpass +## See: listener.ssl.$name.verify +## @todo +## Value: verify_peer | verify_none +## listener.quic.external.verify = verify_peer -# ## See: listener.ssl.$name.dhfile -# ## -# ## Value: File -# ## listener.ssl.external.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem +## See: listener.ssl.$name.fail_if_no_peer_cert +## @todo +## Value: false | true +## listener.quic.external.fail_if_no_peer_cert = true -# ## See: listener.ssl.$name.verify -# ## -# ## Value: verify_peer | verify_none -# ## listener.quic.external.verify = verify_peer +## See: listener.ssl.$name.ciphers +## @todo +## Value: Ciphers +listener.quic.external.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256" -# ## See: listener.ssl.$name.fail_if_no_peer_cert -# ## -# ## Value: false | true -# ## listener.quic.external.fail_if_no_peer_cert = true +## Ciphers for TLS PSK. +## @todo +## Note that 'listener.quic.external.ciphers' and 'listener.quic.external.psk_ciphers' cannot +## be configured at the same time. +## See 'https://tools.ietf.org/html/rfc4279#section-2'. +## listener.quic.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA -# ## See: listener.ssl.$name.ciphers -# ## -# ## Value: Ciphers -listener.quic.external.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" +## See: listener.ssl.$name.honor_cipher_order +## @todo +## Value: on | off +## listener.quic.external.honor_cipher_order = on -# ## Ciphers for TLS PSK. -# ## Note that 'listener.quic.external.ciphers' and 'listener.quic.external.psk_ciphers' cannot -# ## be configured at the same time. -# ## See 'https://tools.ietf.org/html/rfc4279#section-2'. -# ## listener.quic.external.psk_ciphers = PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA - -# ## See: listener.ssl.$name.secure_renegotiate -# ## -# ## Value: on | off -# ## listener.quic.external.secure_renegotiate = off - -# ## See: listener.ssl.$name.reuse_sessions -# ## -# ## Value: on | off -# ## listener.quic.external.reuse_sessions = on - -# ## See: listener.ssl.$name.honor_cipher_order -# ## -# ## Value: on | off -# ## listener.quic.external.honor_cipher_order = on - -# ## See: listener.ssl.$name.peer_cert_as_username -# ## -# ## Value: cn | dn | crt | pem | md5 -# ## listener.quic.external.peer_cert_as_username = cn - -# ## See: listener.ssl.$name.peer_cert_as_clientid -# ## -# ## Value: cn | dn | crt | pem | md5 -# ## listener.quic.external.peer_cert_as_clientid = cn - -# ## TCP backlog for the QUIC connection. -# ## -# ## See: listener.tcp.$name.backlog -# ## -# ## Value: Number >= 0 -# listener.quic.external.backlog = 1024 - -# ## The TCP send timeout for the QUIC connection. -# ## -# ## See: listener.tcp.$name.send_timeout -# ## -# ## Value: Duration +## The send timeout for the QUIC stream. +## @todo +## +## Value: Duration # listener.quic.external.send_timeout = 15s -# ## Close the QUIC connection if send timeout. -# ## -# ## See: listener.tcp.$name.send_timeout_close -# ## -# ## Value: on | off -# listener.quic.external.send_timeout_close = on - -## The TCP receive buffer(os kernel) for the QUIC connections. +## Close the QUIC connection if send timeout. +## @todo +## See: listener.tcp.$name.send_timeout_close ## +## Value: on | off +## listener.quic.external.send_timeout_close = on + +## The receive buffer for the QUIC connections. +## @todo ## See: listener.tcp.$name.recbuf ## ## Value: Bytes ## listener.quic.external.recbuf = 4KB ## The TCP send buffer(os kernel) for the QUIC connections. -## +## @todo ## See: listener.tcp.$name.sndbuf ## ## Value: Bytes ## listener.quic.external.sndbuf = 4KB ## The size of the user-level software buffer used by the driver. -## +## @todo ## See: listener.tcp.$name.buffer ## ## Value: Bytes ## listener.quic.external.buffer = 4KB -## The TCP_NODELAY flag for QUIC connections. -## -## See: listener.tcp.$name.nodelay -## -## Value: true | false -## listener.quic.external.nodelay = true - -## The compress flag for external QUIC connections. -## -## If this Value is set true,the websocket message would be compressed -## -## Value: true | false -## listener.quic.external.compress = true - -## The level of deflate options for external QUIC connections. -## -## See: listener.quic.$name.deflate_opts.level -## -## Value: none | default | best_compression | best_speed -## listener.quic.external.deflate_opts.level = default - -## The mem_level of deflate options for external QUIC connections. -## -## See: listener.quic.$name.deflate_opts.mem_level -## -## Valid range is 1-9 -## listener.quic.external.deflate_opts.mem_level = 8 - -## The strategy of deflate options for external QUIC connections. -## -## See: listener.quic.$name.deflate_opts.strategy -## -## Value: default | filtered | huffman_only | rle -## listener.quic.external.deflate_opts.strategy = default - -## The deflate option for external QUIC connections. -## -## See: listener.quic.$name.deflate_opts.server_context_takeover -## -## Value: takeover | no_takeover -## listener.quic.external.deflate_opts.server_context_takeover = takeover - -## The deflate option for external QUIC connections. -## -## See: listener.quic.$name.deflate_opts.client_context_takeover -## -## Value: takeover | no_takeover -## listener.quic.external.deflate_opts.client_context_takeover = takeover - -## The deflate options for external QUIC connections. -## -## See: listener.quic.$name.deflate_opts.server_max_window_bits -## -## Valid range is 8-15 -## listener.quic.external.deflate_opts.server_max_window_bits = 15 - -## The deflate options for external QUIC connections. -## -## See: listener.quic.$name.deflate_opts.client_max_window_bits -## -## Valid range is 8-15 -## listener.quic.external.deflate_opts.client_max_window_bits = 15 - ## The idle timeout for external QUIC connections. -## +## @todo ## See: listener.quic.$name.idle_timeout ## ## Value: Duration ## listener.quic.external.idle_timeout = 60s ## The max frame size for external QUIC connections. -## +## @todo ## Value: Number ## listener.quic.external.max_frame_size = 0 -## Whether a WebSocket message is allowed to contain multiple MQTT packets -## -## Value: single | multiple -#listener.quic.external.mqtt_piggyback = multiple -## Enable origin check in header for secure websocket connection -## -## Value: true | false (default false) -#listener.quic.external.check_origin_enable = false -## Allow origin to be absent in header in secure websocket connection when check_origin_enable is true -## -## Value: true | false (default true) -#listener.quic.external.allow_origin_absence = true -## Comma separated list of allowed origin in header for secure websocket connection -## -## Value: http://url eg. https://localhost:8084, https://127.0.0.1:8084 -#listener.quic.external.check_origins = "https://localhost:8084, https://127.0.0.1:8084" - ## CONFIG_SECTION_END=listeners ================================================ ## CONFIG_SECTION_BGN=modules ================================================== From fd785240f51a7b32ac46556aec261a15328a6838 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 15 Jun 2021 14:07:57 +0200 Subject: [PATCH 20/23] feat(quic): bump quicer to 0.0.3 --- apps/emqx/rebar.config | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 5719ec0af..1a58efd5b 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -20,7 +20,7 @@ , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} - , {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.2"}}} + , {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.3"}}} ]}. {plugins, [rebar3_proper]}. diff --git a/rebar.config b/rebar.config index 413158842..d8cee22b5 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} - , {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.2"}}} + , {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.3"}}} ]}. {xref_ignores, From 4e2e2d5635237b8626b124119a0655700edea729 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 15 Jun 2021 15:35:42 +0200 Subject: [PATCH 21/23] feat(quic): update emqx_schema for quic --- apps/emqx/src/emqx_schema.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index d1c163e58..f8a5de183 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -361,9 +361,7 @@ fields("wss_listener_settings") -> lists:keydelete("high_watermark", 1, Settings); fields("quic_listener_settings") -> - Unsupported = [ "max_connections" - , "max_conn_rate" - , "active_n" + Unsupported = [ "active_n" , "access" , "proxy_protocol" , "proxy_protocol_timeout" @@ -381,8 +379,9 @@ fields("quic_listener_settings") -> lists:foldl(fun(K, Acc) -> lists:keydelete(K, 1, Acc) end, - [ {"certfile", t(string(), "emqx.certfile", undefined)} - , {"keyfile", t(string(), "emqx.keyfile", undefined)} + [ {"certfile", t(string(), undefined, undefined)} + , {"keyfile", t(string(), undefined, undefined)} + , {"ciphers", t(comma_separated_list(), undefined, "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256")} | fields("listener_settings")], Unsupported); From b4a9d663aee800141cbfbcaba0fe7d5935b269c3 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 16 Jun 2021 15:08:56 +0200 Subject: [PATCH 22/23] feat(quic): quic conn idle_timeout default 1min --- apps/emqx/src/emqx_listeners.erl | 10 +++++----- apps/emqx/src/emqx_schema.erl | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 6ee07e17b..c7d42e2e4 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -147,17 +147,17 @@ start_listener(quic, ListenOn, Options) -> quicer_nif:open_lib(), quicer_nif:reg_open(), SSLOpts = proplists:get_value(ssl_options, Options), + DefAcceptors = erlang:system_info(schedulers_online) * 8, ListenOpts = [ {cert, proplists:get_value(certfile, SSLOpts)} , {key, proplists:get_value(keyfile, SSLOpts)} , {alpn, ["mqtt"]} - , {peer_unidi_stream_count, 1} - , {peer_bidi_stream_count, 10} - , {conn_acceptors, 32} + , {conn_acceptors, proplists:get_value(acceptors, Options, DefAcceptors)} + , {idle_timeout_ms, proplists:get_value(idle_timeout, Options, 60000)} ], ConnectionOpts = [ {conn_callback, emqx_quic_connection} - , {idle_timeout_ms, 5000} , {peer_unidi_stream_count, 1} - , {peer_bidi_stream_count, 10}], + , {peer_bidi_stream_count, 10} + ], StreamOpts = [], quicer:start_listener('mqtt:quic', ListenOn, {ListenOpts, ConnectionOpts, StreamOpts}). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index f8a5de183..2660fa668 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -382,6 +382,7 @@ fields("quic_listener_settings") -> [ {"certfile", t(string(), undefined, undefined)} , {"keyfile", t(string(), undefined, undefined)} , {"ciphers", t(comma_separated_list(), undefined, "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256")} + , {"idle_timeout", t(duration(), undefined, 60000)} | fields("listener_settings")], Unsupported); From d1978aaaf278d9524e5456ac36758e23c4c582fb Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 17 Jun 2021 09:02:34 +0200 Subject: [PATCH 23/23] chore(quic): fix format --- apps/emqx/src/emqx_quic_stream.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 4fbe2ed65..e5cd4c3fc 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -56,12 +56,12 @@ setopts(_Socket, _Opts) -> ok. getopts(_Socket, _Opts) -> - %% todo - { ok, [{high_watermark, 0}, - {high_msgq_watermark, 0}, - {sndbuf, 0}, - {recbuf, 0}, - {buffer,80000}]}. + %% @todo + {ok, [{high_watermark, 0}, + {high_msgq_watermark, 0}, + {sndbuf, 0}, + {recbuf, 0}, + {buffer,80000}]}. fast_close(Stream) -> quicer:close_stream(Stream), @@ -77,7 +77,7 @@ ensure_ok_or_exit(Fun, Args = [Sock|_]) when is_atom(Fun), is_list(Args) -> {error, Reason} -> fast_close(Sock), exit({shutdown, Reason}); - Result -> Result + Result -> Result end. async_send(Stream, Data, Options) when is_list(Data) ->