From 88731fd145303576a1a2a41eafed0c1387adbf3f Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 6 Mar 2023 14:03:28 +0100 Subject: [PATCH 1/6] feat(quic): support TLS password protected keyfile --- apps/emqx/src/emqx_listeners.erl | 6 +- apps/emqx/test/emqx_common_test_helpers.erl | 106 ++++++++++++++++++++ apps/emqx/test/emqx_listeners_SUITE.erl | 36 ++++++- 3 files changed, 142 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index b351212a7..4e5843166 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -388,7 +388,11 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> ] ++ case maps:get(cacertfile, SSLOpts, undefined) of undefined -> []; - CaCertFile -> [{cacertfile, binary_to_list(CaCertFile)}] + CaCertFile -> [{cacertfile, str(CaCertFile)}] + end ++ + case maps:get(password, SSLOpts, undefined) of + undefined -> []; + Password -> [{password, str(Password)}] end ++ optional_quic_listener_opts(Opts), ConnectionOpts = #{ diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index d08812075..67edd58a7 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -85,6 +85,13 @@ reset_proxy/2 ]). +%% TLS certs API +-export([ + gen_ca/2, + gen_host_cert/3, + gen_host_cert/4 +]). + -define(CERTS_PATH(CertName), filename:join(["etc", "certs", CertName])). -define(MQTT_SSL_CLIENT_CERTS, [ @@ -561,6 +568,7 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> mountpoint => <<>>, zone => default }, + Conf2 = maps:merge(Conf, ExtraSettings), emqx_config:put([listeners, quic, Name], Conf2), case emqx_listeners:start_listener(emqx_listeners:listener_id(quic, Name)) of @@ -1073,6 +1081,104 @@ latency_up_proxy(off, Name, ProxyHost, ProxyPort) -> ). %%------------------------------------------------------------------------------- +%% TLS certs +%%------------------------------------------------------------------------------- +gen_ca(Path, Name) -> + %% Generate ca.pem and ca.key which will be used to generate certs + %% for hosts server and clients + ECKeyFile = filename(Path, "~s-ec.key", [Name]), + filelib:ensure_dir(ECKeyFile), + os:cmd("openssl ecparam -name secp256r1 > " ++ ECKeyFile), + Cmd = lists:flatten( + io_lib:format( + "openssl req -new -x509 -nodes " + "-newkey ec:~s " + "-keyout ~s -out ~s -days 3650 " + "-subj \"/C=SE/O=Internet Widgits Pty Ltd CA\"", + [ + ECKeyFile, + ca_key_name(Path, Name), + ca_cert_name(Path, Name) + ] + ) + ), + os:cmd(Cmd). + +ca_cert_name(Path, Name) -> + filename(Path, "~s.pem", [Name]). +ca_key_name(Path, Name) -> + filename(Path, "~s.key", [Name]). + +gen_host_cert(H, CaName, Path) -> + gen_host_cert(H, CaName, Path, #{}). + +gen_host_cert(H, CaName, Path, Opts) -> + ECKeyFile = filename(Path, "~s-ec.key", [CaName]), + CN = str(H), + HKey = filename(Path, "~s.key", [H]), + HCSR = filename(Path, "~s.csr", [H]), + HPEM = filename(Path, "~s.pem", [H]), + HEXT = filename(Path, "~s.extfile", [H]), + PasswordArg = + case maps:get(password, Opts, undefined) of + undefined -> + " -nodes "; + Password -> + io_lib:format(" -passout pass:'~s' ", [Password]) + end, + CSR_Cmd = + lists:flatten( + io_lib:format( + "openssl req -new ~s -newkey ec:~s " + "-keyout ~s -out ~s " + "-addext \"subjectAltName=DNS:~s\" " + "-addext keyUsage=digitalSignature,keyAgreement " + "-subj \"/C=SE/O=Internet Widgits Pty Ltd/CN=~s\"", + [PasswordArg, ECKeyFile, HKey, HCSR, CN, CN] + ) + ), + create_file( + HEXT, + "keyUsage=digitalSignature,keyAgreement\n" + "subjectAltName=DNS:~s\n", + [CN] + ), + CERT_Cmd = + lists:flatten( + io_lib:format( + "openssl x509 -req " + "-extfile ~s " + "-in ~s -CA ~s -CAkey ~s -CAcreateserial " + "-out ~s -days 500", + [ + HEXT, + HCSR, + ca_cert_name(Path, CaName), + ca_key_name(Path, CaName), + HPEM + ] + ) + ), + os:cmd(CSR_Cmd), + os:cmd(CERT_Cmd), + file:delete(HEXT). + +filename(Path, F, A) -> + filename:join(Path, str(io_lib:format(F, A))). + +str(Arg) -> + binary_to_list(iolist_to_binary(Arg)). + +create_file(Filename, Fmt, Args) -> + filelib:ensure_dir(Filename), + {ok, F} = file:open(Filename, [write]), + try + io:format(F, Fmt, Args) + after + file:close(F) + end, + ok. +%%------------------------------------------------------------------------------- %% Testcase teardown utilities %%------------------------------------------------------------------------------- diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 015439587..ac4bf6c76 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -26,6 +26,8 @@ -define(CERTS_PATH(CertName), filename:join(["../../lib/emqx/etc/certs/", CertName])). +-define(SERVER_KEY_PASSWORD, "sErve7r8Key$!"). + all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> @@ -45,11 +47,6 @@ init_per_testcase(Case, Config) when -> catch emqx_config_handler:stop(), {ok, _} = emqx_config_handler:start_link(), - case emqx_config:get([listeners], undefined) of - undefined -> ok; - Listeners -> emqx_config:put([listeners], maps:remove(quic, Listeners)) - end, - PrevListeners = emqx_config:get([listeners], #{}), PureListeners = remove_default_limiter(PrevListeners), PureListeners2 = PureListeners#{ @@ -185,6 +182,28 @@ t_wss_conn(_) -> {ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000), ok = ssl:close(Socket). +t_quic_conn(Config) -> + DataDir = ?config(data_dir, Config), + generate_quic_tls_certs(Config), + SSLOpts = #{ + password => ?SERVER_KEY_PASSWORD, + certfile => filename:join(DataDir, "server-password.pem"), + cacertfile => filename:join(DataDir, "ca.pem"), + keyfile => filename:join(DataDir, "server-password.key") + }, + emqx_common_test_helpers:ensure_quic_listener(?FUNCTION_NAME, 24568, #{ssl_options => SSLOpts}), + ct:pal("~p", [emqx_listeners:list()]), + {ok, Conn} = quicer:connect( + {127, 0, 0, 1}, + 24568, + [ + {verify, verify_none}, + {alpn, ["mqtt"]} + ], + 1000 + ), + ok = quicer:close_connection(Conn). + t_format_bind(_) -> ?assertEqual( ":1883", @@ -269,3 +288,10 @@ remove_default_limiter(Listeners) -> end, Listeners ). + +generate_quic_tls_certs(Config) -> + DataDir = ?config(data_dir, Config), + emqx_common_test_helpers:gen_ca(DataDir, "ca"), + emqx_common_test_helpers:gen_host_cert("server-password", "ca", DataDir, #{ + password => ?SERVER_KEY_PASSWORD + }). From 7e15f90bf5cfa824faad63ffdef02e5eab063ff6 Mon Sep 17 00:00:00 2001 From: William Yang Date: Sun, 19 Mar 2023 21:48:05 +0100 Subject: [PATCH 2/6] chore(test): check openssl cmd returns --- apps/emqx/test/emqx_common_test_helpers.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 67edd58a7..bda8d167b 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -1159,8 +1159,8 @@ gen_host_cert(H, CaName, Path, Opts) -> ] ) ), - os:cmd(CSR_Cmd), - os:cmd(CERT_Cmd), + ct:pal(os:cmd(CSR_Cmd)), + ct:pal(os:cmd(CERT_Cmd)), file:delete(HEXT). filename(Path, F, A) -> From 70a1c25d0fa9288007925fe98aa1e24ce956bce6 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 21 Mar 2023 12:06:18 +0100 Subject: [PATCH 3/6] docs: add changelogs feat-10077 Co-authored-by: Thales Macedo Garitezi --- changes/ce/feat-10077.en.md | 2 ++ changes/ce/feat-10077.zh.md | 1 + 2 files changed, 3 insertions(+) create mode 100644 changes/ce/feat-10077.en.md create mode 100644 changes/ce/feat-10077.zh.md diff --git a/changes/ce/feat-10077.en.md b/changes/ce/feat-10077.en.md new file mode 100644 index 000000000..923e21fa1 --- /dev/null +++ b/changes/ce/feat-10077.en.md @@ -0,0 +1,2 @@ +Add support for QUIC TLS password protected certificate file. + diff --git a/changes/ce/feat-10077.zh.md b/changes/ce/feat-10077.zh.md new file mode 100644 index 000000000..e9c7b5625 --- /dev/null +++ b/changes/ce/feat-10077.zh.md @@ -0,0 +1 @@ +增加对 QUIC TLS 密码保护证书文件的支持。 From 169cc9f822885d44a168f2c06175769419143cbd Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 21 Mar 2023 12:11:03 +0100 Subject: [PATCH 4/6] chore(quic): unhide TLS certfile password --- apps/emqx/src/emqx_schema.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 433fb20e5..b7a8aed64 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -3019,9 +3019,9 @@ is_quic_ssl_opts(Name) -> "cacertfile", "certfile", "keyfile", - "verify" + "verify", + "password" %% Followings are planned - %% , "password" %% , "hibernate_after" %% , "fail_if_no_peer_cert" %% , "handshake_timeout" From cec77c2b6529a332add7af71852a966abec25c88 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 22 Mar 2023 15:14:39 +0100 Subject: [PATCH 5/6] test(quic): chasing flaky tc. --- apps/emqx/test/emqx_quic_multistreams_SUITE.erl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl index 642b5468c..4afd965bd 100644 --- a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -1569,7 +1569,7 @@ t_multi_streams_remote_shutdown(Config) -> ok = stop_emqx(), %% Client should be closed - assert_client_die(C). + assert_client_die(C, 100, 50). t_multi_streams_remote_shutdown_with_reconnect(Config) -> erlang:process_flag(trap_exit, true), @@ -2047,14 +2047,15 @@ via_stream({quic, _Conn, Stream}) -> assert_client_die(C) -> assert_client_die(C, 100, 10). assert_client_die(C, _, 0) -> - ct:fail("Client ~p did not die", [C]); + ct:fail("Client ~p did not die: stacktrace: ~p", [C, process_info(C, current_stacktrace)]); assert_client_die(C, Delay, Retries) -> - case catch emqtt:info(C) of - {'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}} -> - ok; - _Other -> + try emqtt:info(C) of + Info when is_list(Info) -> timer:sleep(Delay), assert_client_die(C, Delay, Retries - 1) + catch + exit:Error -> + ct:comment("client die with ~p", [Error]) end. %% BUILD_WITHOUT_QUIC From ecc2cd1a949e7ed84484d7f2e1f1ddddfaa5c29f Mon Sep 17 00:00:00 2001 From: William Yang Date: Sat, 25 Mar 2023 10:34:00 +0100 Subject: [PATCH 6/6] test: password cert for SSL listener --- apps/emqx/test/emqx_listeners_SUITE.erl | 33 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index ac4bf6c76..107f3d4e7 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -35,6 +35,7 @@ init_per_suite(Config) -> application:ensure_all_started(esockd), application:ensure_all_started(quicer), application:ensure_all_started(cowboy), + generate_tls_certs(Config), lists:foreach(fun set_app_env/1, NewConfig), Config. @@ -183,26 +184,48 @@ t_wss_conn(_) -> ok = ssl:close(Socket). t_quic_conn(Config) -> + Port = 24568, DataDir = ?config(data_dir, Config), - generate_quic_tls_certs(Config), SSLOpts = #{ password => ?SERVER_KEY_PASSWORD, certfile => filename:join(DataDir, "server-password.pem"), cacertfile => filename:join(DataDir, "ca.pem"), keyfile => filename:join(DataDir, "server-password.key") }, - emqx_common_test_helpers:ensure_quic_listener(?FUNCTION_NAME, 24568, #{ssl_options => SSLOpts}), + emqx_common_test_helpers:ensure_quic_listener(?FUNCTION_NAME, Port, #{ssl_options => SSLOpts}), ct:pal("~p", [emqx_listeners:list()]), {ok, Conn} = quicer:connect( {127, 0, 0, 1}, - 24568, + Port, [ {verify, verify_none}, {alpn, ["mqtt"]} ], 1000 ), - ok = quicer:close_connection(Conn). + ok = quicer:close_connection(Conn), + emqx_listeners:stop_listener(quic, ?FUNCTION_NAME, #{bind => Port}). + +t_ssl_password_cert(Config) -> + Port = 24568, + DataDir = ?config(data_dir, Config), + SSLOptsPWD = #{ + password => ?SERVER_KEY_PASSWORD, + certfile => filename:join(DataDir, "server-password.pem"), + cacertfile => filename:join(DataDir, "ca.pem"), + keyfile => filename:join(DataDir, "server-password.key") + }, + LConf = #{ + enabled => true, + bind => {{127, 0, 0, 1}, Port}, + mountpoint => <<>>, + zone => default, + ssl_options => SSLOptsPWD + }, + ok = emqx_listeners:start_listener(ssl, ?FUNCTION_NAME, LConf), + {ok, SSLSocket} = ssl:connect("127.0.0.1", Port, [{verify, verify_none}]), + ssl:close(SSLSocket), + emqx_listeners:stop_listener(ssl, ?FUNCTION_NAME, LConf). t_format_bind(_) -> ?assertEqual( @@ -289,7 +312,7 @@ remove_default_limiter(Listeners) -> Listeners ). -generate_quic_tls_certs(Config) -> +generate_tls_certs(Config) -> DataDir = ?config(data_dir, Config), emqx_common_test_helpers:gen_ca(DataDir, "ca"), emqx_common_test_helpers:gen_host_cert("server-password", "ca", DataDir, #{