From 5c3f5d808568fca7c94e0a06c57606a57f09a77a Mon Sep 17 00:00:00 2001 From: zmstone Date: Tue, 4 Jun 2024 14:33:13 +0200 Subject: [PATCH 01/39] perf: do not call inet getstat before each and every send In a stress test environment, when alarm is enabled, there were a lot of long_schedule warnings with the stacktrace pointing to congestion alarm based inet:getstat function. When alarm is disabled, the one single client is able to hold 40,000 QoS 1 messages per second (inflight=32) throughput. When alarm is enabled, its mqueue overflows. This commit removes the call before each and every data send, so to rely on emit_stats timer to trigger the congestion check. --- apps/emqx/src/emqx_connection.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index e0baab238..3708593e7 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -914,7 +914,6 @@ send(IoData, #state{transport = Transport, socket = Socket, channel = Channel}) Oct = iolist_size(IoData), ok = emqx_metrics:inc('bytes.sent', Oct), inc_counter(outgoing_bytes, Oct), - emqx_congestion:maybe_alarm_conn_congestion(Socket, Transport, Channel), case Transport:async_send(Socket, IoData, []) of ok -> ok; From 2525bd0c169aa890b2de1bb736a49c399e0a6f64 Mon Sep 17 00:00:00 2001 From: zmstone Date: Tue, 4 Jun 2024 15:00:06 +0200 Subject: [PATCH 02/39] perf: eliminate the loop-back fake async inet_reply message Now that EMQX is on OTP 26 by default, the fake inet_reply message when the send call returns ok only adds overhead. For TCP (gen_tcp module), OTP 26 no longer supports direct port_command so we had to swith to gen_tcp:send, then fake an async reply: see https://github.com/emqx/esockd/pull/181 For TLS (ssl module), it has never supported async send anyway. This commit switches to call Transport:send/2 directly, this should result in one less loop-back message for each message. For OTP 25 on which EMQX can still be compiled, one will have to suffer the performance penalty for gen_tcp:send function to do a non-optimized 'receive' for the send result. --- apps/emqx/src/emqx_connection.erl | 79 +++++++------------ apps/emqx/src/emqx_quic_stream.erl | 4 +- apps/emqx/test/emqx_connection_SUITE.erl | 22 +++--- .../src/bhvrs/emqx_gateway_conn.erl | 56 ++++++------- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- 5 files changed, 71 insertions(+), 92 deletions(-) diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 3708593e7..ed62fb63c 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -158,31 +158,6 @@ -define(ENABLED(X), (X =/= undefined)). --define(ALARM_TCP_CONGEST(Channel), - list_to_binary( - io_lib:format( - "mqtt_conn/congested/~ts/~ts", - [ - emqx_channel:info(clientid, Channel), - emqx_channel:info(username, Channel) - ] - ) - ) -). - --define(ALARM_CONN_INFO_KEYS, [ - socktype, - sockname, - peername, - clientid, - username, - proto_name, - proto_ver, - connected_at -]). --define(ALARM_SOCK_STATS_KEYS, [send_pend, recv_cnt, recv_oct, send_cnt, send_oct]). --define(ALARM_SOCK_OPTS_KEYS, [high_watermark, high_msgq_watermark, sndbuf, recbuf, buffer]). - -define(LIMITER_BYTES_IN, bytes). -define(LIMITER_MESSAGE_IN, messages). @@ -603,17 +578,6 @@ handle_msg( ActiveN = get_active_n(Type, Listener), Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); -%% Something sent -handle_msg({inet_reply, _Sock, ok}, State = #state{listener = {Type, Listener}}) -> - case emqx_pd:get_counter(outgoing_pubs) > get_active_n(Type, Listener) of - true -> - Pubs = emqx_pd:reset_counter(outgoing_pubs), - Bytes = emqx_pd:reset_counter(outgoing_bytes), - OutStats = #{cnt => Pubs, oct => Bytes}, - {ok, check_oom(run_gc(OutStats, State))}; - false -> - ok - end; handle_msg({inet_reply, _Sock, {error, Reason}}, State) -> handle_info({sock_error, Reason}, State); handle_msg({connack, ConnAck}, State) -> @@ -729,9 +693,9 @@ handle_call(_From, Req, State = #state{channel = Channel}) -> shutdown(Reason, Reply, State#state{channel = NChannel}); {shutdown, Reason, Reply, OutPacket, NChannel} -> NState = State#state{channel = NChannel}, - ok = handle_outgoing(OutPacket, NState), - NState2 = graceful_shutdown_transport(Reason, NState), - shutdown(Reason, Reply, NState2) + {ok, NState2} = handle_outgoing(OutPacket, NState), + NState3 = graceful_shutdown_transport(Reason, NState2), + shutdown(Reason, Reply, NState3) end. %%-------------------------------------------------------------------- @@ -854,8 +818,8 @@ with_channel(Fun, Args, State = #state{channel = Channel}) -> shutdown(Reason, State#state{channel = NChannel}); {shutdown, Reason, Packet, NChannel} -> NState = State#state{channel = NChannel}, - ok = handle_outgoing(Packet, NState), - shutdown(Reason, NState) + {ok, NState2} = handle_outgoing(Packet, NState), + shutdown(Reason, NState2) end. %%-------------------------------------------------------------------- @@ -909,19 +873,36 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) -> %%-------------------------------------------------------------------- %% Send data --spec send(iodata(), state()) -> ok. -send(IoData, #state{transport = Transport, socket = Socket, channel = Channel}) -> +-spec send(iodata(), state()) -> {ok, state()}. +send(IoData, #state{transport = Transport, socket = Socket} = State) -> Oct = iolist_size(IoData), - ok = emqx_metrics:inc('bytes.sent', Oct), + emqx_metrics:inc('bytes.sent', Oct), inc_counter(outgoing_bytes, Oct), - case Transport:async_send(Socket, IoData, []) of + case Transport:send(Socket, IoData) of ok -> - ok; + %% NOTE: for Transport=emqx_quic_stream, it's actually an + %% async_send, sent/1 should technically be called when + %% {quic, send_complete, _Stream, true | false} is received, + %% but it is handled early for simplicity + sent(State); Error = {error, _Reason} -> - %% Send an inet_reply to postpone handling the error - %% @FIXME: why not just return error? + %% Defer error handling + %% so it's handled the same as tcp_closed or ssl_closed self() ! {inet_reply, Socket, Error}, - ok + {ok, State} + end. + +%% Some bytes sent +sent(#state{listener = {Type, Listener}} = State) -> + %% Run GC and check OOM after certain amount of messages or bytes sent. + case emqx_pd:get_counter(outgoing_pubs) > get_active_n(Type, Listener) of + true -> + Pubs = emqx_pd:reset_counter(outgoing_pubs), + Bytes = emqx_pd:reset_counter(outgoing_bytes), + OutStats = #{cnt => Pubs, oct => Bytes}, + {ok, check_oom(run_gc(OutStats, State))}; + false -> + {ok, State} end. %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 43f1eebfe..ca4134c25 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -34,7 +34,7 @@ fast_close/1, shutdown/2, ensure_ok_or_exit/2, - async_send/3, + send/2, setopts/2, getopts/2, peername/1, @@ -165,7 +165,7 @@ ensure_ok_or_exit(Fun, Args = [Sock | _]) when is_atom(Fun), is_list(Args) -> Result end. -async_send({quic, _Conn, Stream, _Info}, Data, _Options) -> +send({quic, _Conn, Stream, _Info}, Data) -> case quicer:async_send(Stream, Data, ?QUICER_SEND_FLAG_SYNC) of {ok, _Len} -> ok; {error, X, Y} -> {error, {X, Y}}; diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 7fffa3374..3e2db9e36 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -94,8 +94,7 @@ init_per_testcase(TestCase, Config) when ok = meck:expect(emqx_transport, getstat, fun(_Sock, Options) -> {ok, [{K, 0} || K <- Options]} end), - ok = meck:expect(emqx_transport, async_send, fun(_Sock, _Data) -> ok end), - ok = meck:expect(emqx_transport, async_send, fun(_Sock, _Data, _Opts) -> ok end), + ok = meck:expect(emqx_transport, send, fun(_Sock, _Data) -> ok end), ok = meck:expect(emqx_transport, fast_close, fun(_Sock) -> ok end), case erlang:function_exported(?MODULE, TestCase, 2) of true -> ?MODULE:TestCase(init, Config); @@ -234,9 +233,11 @@ t_handle_msg_incoming(_) -> ?assertMatch({ok, _St}, handle_msg({incoming, undefined}, st())). t_handle_msg_outgoing(_) -> - ?assertEqual(ok, handle_msg({outgoing, ?PUBLISH_PACKET(?QOS_2, <<"Topic">>, 1, <<>>)}, st())), - ?assertEqual(ok, handle_msg({outgoing, ?PUBREL_PACKET(1)}, st())), - ?assertEqual(ok, handle_msg({outgoing, ?PUBCOMP_PACKET(1)}, st())). + ?assertMatch( + {ok, _}, handle_msg({outgoing, ?PUBLISH_PACKET(?QOS_2, <<"Topic">>, 1, <<>>)}, st()) + ), + ?assertMatch({ok, _}, handle_msg({outgoing, ?PUBREL_PACKET(1)}, st())), + ?assertMatch({ok, _}, handle_msg({outgoing, ?PUBCOMP_PACKET(1)}, st())). t_handle_msg_tcp_error(_) -> ?assertMatch( @@ -255,18 +256,13 @@ t_handle_msg_deliver(_) -> ?assertMatch({ok, _St}, handle_msg({deliver, topic, msg}, st())). t_handle_msg_inet_reply(_) -> - ok = meck:expect(emqx_pd, get_counter, fun(_) -> 10 end), - emqx_config:put_listener_conf(tcp, default, [tcp_options, active_n], 0), - ?assertMatch({ok, _St}, handle_msg({inet_reply, for_testing, ok}, st())), - emqx_config:put_listener_conf(tcp, default, [tcp_options, active_n], 100), - ?assertEqual(ok, handle_msg({inet_reply, for_testing, ok}, st())), ?assertMatch( {stop, {shutdown, for_testing}, _St}, handle_msg({inet_reply, for_testing, {error, for_testing}}, st()) ). t_handle_msg_connack(_) -> - ?assertEqual(ok, handle_msg({connack, ?CONNACK_PACKET(?CONNACK_ACCEPT)}, st())). + ?assertMatch({ok, _}, handle_msg({connack, ?CONNACK_PACKET(?CONNACK_ACCEPT)}, st())). t_handle_msg_close(_) -> ?assertMatch({stop, {shutdown, normal}, _St}, handle_msg({close, normal}, st())). @@ -388,8 +384,8 @@ t_with_channel(_) -> meck:unload(emqx_channel). t_handle_outgoing(_) -> - ?assertEqual(ok, emqx_connection:handle_outgoing(?PACKET(?PINGRESP), st())), - ?assertEqual(ok, emqx_connection:handle_outgoing([?PACKET(?PINGRESP)], st())). + ?assertMatch({ok, _}, emqx_connection:handle_outgoing(?PACKET(?PINGRESP), st())), + ?assertMatch({ok, _}, emqx_connection:handle_outgoing([?PACKET(?PINGRESP)], st())). t_handle_info(_) -> ?assertMatch( diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 84dfe44a2..710148b94 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -242,7 +242,7 @@ esockd_send(Data, #state{ }) -> gen_udp:send(Sock, Ip, Port, Data); esockd_send(Data, #state{socket = {esockd_transport, Sock}}) -> - esockd_transport:async_send(Sock, Data). + esockd_transport:send(Sock, Data). keepalive_stats(recv) -> emqx_pd:get_counter(recv_pkt); @@ -503,18 +503,6 @@ handle_msg( ) -> Delivers = [Deliver | emqx_utils:drain_deliver(ActiveN)], with_channel(handle_deliver, [Delivers], State); -%% Something sent -%% TODO: Who will deliver this message? -handle_msg({inet_reply, _Sock, ok}, State = #state{active_n = ActiveN}) -> - case emqx_pd:get_counter(outgoing_pkt) > ActiveN of - true -> - Pubs = emqx_pd:reset_counter(outgoing_pkt), - Bytes = emqx_pd:reset_counter(outgoing_bytes), - OutStats = #{cnt => Pubs, oct => Bytes}, - {ok, check_oom(run_gc(OutStats, State))}; - false -> - ok - end; handle_msg({inet_reply, _Sock, {error, Reason}}, State) -> handle_info({sock_error, Reason}, State); handle_msg({close, Reason}, State) -> @@ -630,8 +618,8 @@ handle_call( shutdown(Reason, Reply, State#state{channel = NChannel}); {shutdown, Reason, Reply, Packet, NChannel} -> NState = State#state{channel = NChannel}, - ok = handle_outgoing(Packet, NState), - shutdown(Reason, Reply, NState) + {ok, NState1} = handle_outgoing(Packet, NState), + shutdown(Reason, Reply, NState1) end. %%-------------------------------------------------------------------- @@ -772,15 +760,15 @@ with_channel( shutdown(Reason, State#state{channel = NChannel}); {shutdown, Reason, Packet, NChannel} -> NState = State#state{channel = NChannel}, - ok = handle_outgoing(Packet, NState), - shutdown(Reason, NState) + {ok, NState1} = handle_outgoing(Packet, NState), + shutdown(Reason, NState1) end. %%-------------------------------------------------------------------- %% Handle outgoing packets -handle_outgoing(_Packets = [], _State) -> - ok; +handle_outgoing(_Packets = [], State) -> + {ok, State}; handle_outgoing( Packets, State = #state{socket = Socket} @@ -792,12 +780,15 @@ handle_outgoing( State ); _ -> - lists:foreach( - fun(Packet) -> - handle_outgoing(Packet, State) + NState = lists:foldl( + fun(Packet, State0) -> + {ok, State1} = handle_outgoing(Packet, State0), + State1 end, + State, Packets - ) + ), + {ok, NState} end; handle_outgoing(Packet, State) -> send((serialize_and_inc_stats_fun(State))(Packet), State). @@ -842,7 +833,7 @@ serialize_and_inc_stats_fun(#state{ %%-------------------------------------------------------------------- %% Send data --spec send(iodata(), state()) -> ok. +-spec send(iodata(), state()) -> {ok, state()}. send( IoData, State = #state{ @@ -858,11 +849,22 @@ send( inc_counter(outgoing_bytes, Oct), case esockd_send(IoData, State) of ok -> - ok; + sent(State); Error = {error, _Reason} -> - %% Send an inet_reply to postpone handling the error + %% Send an inet_reply to defer handling the error self() ! {inet_reply, Socket, Error}, - ok + {ok, State} + end. + +sent(#state{active_n = ActiveN} = State) -> + case emqx_pd:get_counter(outgoing_pkt) > ActiveN of + true -> + Pubs = emqx_pd:reset_counter(outgoing_pkt), + Bytes = emqx_pd:reset_counter(outgoing_bytes), + OutStats = #{cnt => Pubs, oct => Bytes}, + {ok, check_oom(run_gc(OutStats, State))}; + false -> + {ok, State} end. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 3c6634edc..fa8a774ed 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.32"}, + {vsn, "0.1.33"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_auth, emqx_ctl]}, From 69caf4b4ebe0d3b482d62469fe1d689805f0a1c5 Mon Sep 17 00:00:00 2001 From: zmstone Date: Wed, 5 Jun 2024 11:40:29 +0200 Subject: [PATCH 03/39] docs: add changelog for PR 13180 --- changes/ce/feat-13180.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-13180.en.md diff --git a/changes/ce/feat-13180.en.md b/changes/ce/feat-13180.en.md new file mode 100644 index 000000000..255230d0d --- /dev/null +++ b/changes/ce/feat-13180.en.md @@ -0,0 +1 @@ +Improve client message handling performance when running on OTP 26. From 1ce13242a899a58f2650e8f2fa6e34bd006eb200 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 5 Oct 2023 14:51:37 +0200 Subject: [PATCH 04/39] feat(tls): port partial_chain, part 1 --- apps/emqx/src/emqx_const_v2.erl | 126 +++++++++++++++++++++++++++++++ apps/emqx/src/emqx_listeners.erl | 14 +++- apps/emqx/src/emqx_tls_lib.erl | 54 +++++++++++++ 3 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 apps/emqx/src/emqx_const_v2.erl diff --git a/apps/emqx/src/emqx_const_v2.erl b/apps/emqx/src/emqx_const_v2.erl new file mode 100644 index 000000000..9fb2b7fa7 --- /dev/null +++ b/apps/emqx/src/emqx_const_v2.erl @@ -0,0 +1,126 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @doc Never update this module, create a v3 instead. +%%-------------------------------------------------------------------- + +-module(emqx_const_v2). + +-export([ + make_tls_root_fun/2, + make_tls_verify_fun/2 +]). + +-include_lib("public_key/include/public_key.hrl"). +%% @doc Build a root fun for verify TLS partial_chain. +%% The `InputChain' is composed by OTP SSL with local cert store +%% AND the cert (chain if any) from the client. +%% @end +make_tls_root_fun(cacert_from_cacertfile, [Trusted]) -> + %% Allow only one trusted ca cert, and just return the defined trusted CA cert, + fun(_InputChain) -> + %% Note, returing `trusted_ca` doesn't really mean it accepts the connection + %% OTP SSL app will do the path validation, signature validation subsequently. + {trusted_ca, Trusted} + end; +make_tls_root_fun(cacert_from_cacertfile, [TrustedOne, TrustedTwo]) -> + %% Allow two trusted CA certs in case of CA cert renewal + %% This is a little expensive call as it compares the binaries. + fun(InputChain) -> + case lists:member(TrustedOne, InputChain) of + true -> + {trusted_ca, TrustedOne}; + false -> + {trusted_ca, TrustedTwo} + end + end. + +make_tls_verify_fun(verify_cert_extKeyUsage, KeyUsages) -> + AllowedKeyUsages = ext_key_opts(KeyUsages), + {fun verify_fun_peer_extKeyUsage/3, AllowedKeyUsages}. + +verify_fun_peer_extKeyUsage(_, {bad_cert, invalid_ext_key_usage}, UserState) -> + %% !! Override OTP verify peer default + %% OTP SSL is unhappy with the ext_key_usage but we will check on our own. + {unknown, UserState}; +verify_fun_peer_extKeyUsage(_, {bad_cert, _} = Reason, _UserState) -> + %% OTP verify_peer default + {fail, Reason}; +verify_fun_peer_extKeyUsage(_, {extension, _}, UserState) -> + %% OTP verify_peer default + {unknown, UserState}; +verify_fun_peer_extKeyUsage(_, valid, UserState) -> + %% OTP verify_peer default + {valid, UserState}; +verify_fun_peer_extKeyUsage( + #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{extensions = ExtL}}, + %% valid peer cert + valid_peer, + AllowedKeyUsages +) -> + %% override OTP verify_peer default + %% must have id-ce-extKeyUsage + case lists:keyfind(?'id-ce-extKeyUsage', 2, ExtL) of + #'Extension'{extnID = ?'id-ce-extKeyUsage', extnValue = VL} -> + case do_verify_ext_key_usage(VL, AllowedKeyUsages) of + true -> + %% pass the check, + %% fallback to OTP verify_peer default + {valid, AllowedKeyUsages}; + false -> + {fail, extKeyUsage_unmatched} + end; + _ -> + {fail, extKeyUsage_not_set} + end. + +%% @doc check required extkeyUsages are presented in the cert +do_verify_ext_key_usage(_, []) -> + %% Verify finished + true; +do_verify_ext_key_usage(CertExtL, [Usage | T] = _Required) -> + case lists:member(Usage, CertExtL) of + true -> + do_verify_ext_key_usage(CertExtL, T); + false -> + false + end. + +%% @doc Helper tls cert extension +-spec ext_key_opts + (string()) -> [OidString :: string() | public_key:oid()]; + (undefined) -> undefined. +ext_key_opts(Str) -> + Usages = string:tokens(Str, ","), + lists:map( + fun + ("clientAuth") -> + ?'id-kp-clientAuth'; + ("serverAuth") -> + ?'id-kp-serverAuth'; + ("codeSigning") -> + ?'id-kp-codeSigning'; + ("emailProtection") -> + ?'id-kp-emailProtection'; + ("timeStamping") -> + ?'id-kp-timeStamping'; + ("ocspSigning") -> + ?'id-kp-OCSPSigning'; + ([$O, $I, $D, $: | OidStr]) -> + OidList = string:tokens(OidStr, "."), + list_to_tuple(lists:map(fun list_to_integer/1, OidList)) + end, + Usages + ). diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index dd9024fef..4e8d6274f 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -611,7 +611,9 @@ esockd_opts(ListenerId, Type, Name, Opts0) -> ssl -> OptsWithCRL = inject_crl_config(Opts0), OptsWithSNI = inject_sni_fun(ListenerId, OptsWithCRL), - SSLOpts = ssl_opts(OptsWithSNI), + OptsWithRootFun = inject_root_fun(OptsWithSNI), + OptsWithVerifyFun = inject_verify_fun(OptsWithRootFun), + SSLOpts = ssl_opts(OptsWithVerifyFun), Opts3#{ssl_options => SSLOpts, tcp_options => tcp_opts(Opts0)} end ). @@ -962,6 +964,16 @@ quic_listener_optional_settings() -> stateless_operation_expiration_ms ]. +inject_root_fun(#{ssl_options := SslOpts} = Opts) -> + Opts#{ssl_options := emqx_tls_lib:opt_partial_chain(SslOpts)}; +inject_root_fun(Opts) -> + Opts. + +inject_verify_fun(#{ssl_options := SslOpts} = Opts) -> + Opts#{ssl_options := emqx_tls_lib:opt_verify_fun(SslOpts)}; +inject_verify_fun(Opts) -> + Opts. + inject_sni_fun(ListenerId, Conf = #{ssl_options := #{ocsp := #{enable_ocsp_stapling := true}}}) -> emqx_ocsp_cache:inject_sni_fun(ListenerId, Conf); inject_sni_fun(_ListenerId, Conf) -> diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index c524381ad..8ab4a7a5d 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -23,6 +23,8 @@ default_ciphers/0, selected_ciphers/1, integral_ciphers/2, + opt_partial_chain/1, + opt_verify_fun/1, all_ciphers_set_cached/0 ]). @@ -685,3 +687,55 @@ ensure_ssl_file_key(SSL, RequiredKeyPaths) -> [] -> ok; Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} end. + +%% @doc enable TLS partial_chain validation if set. +-spec opt_partial_chain(SslOpts :: map()) -> NewSslOpts :: map(). +opt_partial_chain(#{partial_chain := false} = SslOpts) -> + maps:remove(partial_chain, SslOpts); +opt_partial_chain(#{partial_chain := true} = SslOpts) -> + SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(1, SslOpts)}; +opt_partial_chain(#{partial_chain := cacert_from_cacertfile} = SslOpts) -> + SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(1, SslOpts)}; +opt_partial_chain(#{partial_chain := two_cacerts_from_cacertfile} = SslOpts) -> + SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(2, SslOpts)}; +opt_partial_chain(SslOpts) -> + SslOpts. + +%% @doc make verify_fun if set. +-spec opt_verify_fun(SslOpts :: map()) -> NewSslOpts :: map(). +opt_verify_fun(#{verify_peer_ext_key_usage := V} = SslOpts) -> + SslOpts#{verify_fun => emqx_const_v2:make_tls_verify_fun(verify_cert_extKeyUsage, V)}; +opt_verify_fun(SslOpts) -> + SslOpts. + +%% @doc Helper, make TLS root_fun +rootfun_trusted_ca_from_cacertfile(NumOfCerts, #{cacertfile := Cacertfile}) -> + case file:read_file(Cacertfile) of + {ok, PemBin} -> + try + do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, PemBin) + catch + _Error:_Info:ST -> + %% The cacertfile will be checked by OTP SSL as well and OTP choice to be silent on this. + %% We are touching security sutffs, don't leak extra info.. + ?SLOG(error, #{ + msg => "trusted_cacert_not_found_in_cacertfile", stacktrace => ST + }), + throw({error, ?FUNCTION_NAME}) + end; + {error, Reason} -> + throw({error, {read_cacertfile_error, Cacertfile, Reason}}) + end; +rootfun_trusted_ca_from_cacertfile(_NumOfCerts, _SslOpts) -> + throw({error, cacertfile_unset}). + +do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, PemBin) -> + %% The last one or two should be the top parent in the chain if it is a chain + Certs = public_key:pem_decode(PemBin), + Pos = length(Certs) - NumOfCerts + 1, + Trusted = [ + CADer + || {'Certificate', CADer, _} <- + lists:sublist(public_key:pem_decode(PemBin), Pos, NumOfCerts) + ], + emqx_const_v2:make_tls_root_fun(cacert_from_cacertfile, Trusted). From f7ff9496e6a0c00cc1dcda1c688a0ca5f99914ef Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 5 Oct 2023 14:54:14 +0200 Subject: [PATCH 05/39] test: port listener tls partial_chain --- .../emqx_listener_tls_verify_chain_SUITE.erl | 248 +++++++ ...mqx_listener_tls_verify_keyusage_SUITE.erl | 360 ++++++++++ ...istener_tls_verify_partial_chain_SUITE.erl | 668 ++++++++++++++++++ apps/emqx/test/emqx_test_tls_certs_helper.erl | 311 ++++++++ 4 files changed, 1587 insertions(+) create mode 100644 apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl create mode 100644 apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl create mode 100644 apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl create mode 100644 apps/emqx/test/emqx_test_tls_certs_helper.erl diff --git a/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl new file mode 100644 index 000000000..c38426523 --- /dev/null +++ b/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl @@ -0,0 +1,248 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_listener_tls_verify_chain_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import( + emqx_test_tls_certs_helper, + [ + emqx_start_listener/4, + fail_when_ssl_error/1, + fail_when_no_ssl_alert/2, + generate_tls_certs/1 + ] +). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + generate_tls_certs(Config), + application:ensure_all_started(esockd), + [{ssl_config, ssl_config_verify_peer()} | Config]. + +end_per_suite(_Config) -> + application:stop(esockd). + +t_conn_fail_with_intermediate_ca_cert(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + + fail_when_no_ssl_alert(Socket, unknown_ca), + ok = ssl:close(Socket). + +t_conn_fail_with_other_intermediate_ca_cert(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2.pem")} + ], + 1000 + ), + + fail_when_no_ssl_alert(Socket, unknown_ca), + ok = ssl:close(Socket). + +t_conn_success_with_server_client_composed_complete_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% Server has root ca cert + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "root.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + %% Client has complete chain + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_success_with_other_signed_client_composed_complete_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% Server has root ca cert + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "root.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + %% Client has partial_chain + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_success_with_renewed_intermediate_root_bundle(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% Server has root ca cert + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1_renewed-root-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_success_with_client_complete_cert_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "root.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_fail_with_server_partial_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% imcomplete at server side + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ok = ssl:close(Socket). + +t_conn_fail_without_root_cacert(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ok = ssl:close(Socket). + +ssl_config_verify_peer() -> + [ + {verify, verify_peer}, + {fail_if_no_peer_cert, true} + ]. diff --git a/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl new file mode 100644 index 000000000..c12618566 --- /dev/null +++ b/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl @@ -0,0 +1,360 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_listener_tls_verify_keyusage_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import(emqx_test_tls_certs_helper, [ + fail_when_ssl_error/1, + fail_when_no_ssl_alert/2, + generate_tls_certs/1, + gen_host_cert/4 +]). + +all() -> + [ + {group, full_chain}, + {group, partial_chain} + ]. + +all_tc() -> + emqx_ct:all(?MODULE). + +groups() -> + [ + {partial_chain, [], all_tc()}, + {full_chain, [], all_tc()} + ]. + +init_per_suite(Config) -> + generate_tls_certs(Config), + application:ensure_all_started(esockd), + Config. + +end_per_suite(_Config) -> + application:stop(esockd). + +init_per_group(full_chain, Config) -> + [{ssl_config, ssl_config_verify_peer_full_chain(Config)} | Config]; +init_per_group(partial_chain, Config) -> + [{ssl_config, ssl_config_verify_peer_partial_chain(Config)} | Config]; +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +t_conn_success_verify_peer_ext_key_usage_unset(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% Given listener keyusage unset + Options = [{ssl_options, ?config(ssl_config, Config)}], + emqx_listeners:start_listener(ssl, Port, Options), + %% when client connect with cert without keyusage ext + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + %% Then connection success + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_success_verify_peer_ext_key_usage_undefined(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% Give listener keyusage is set to undefined + Options = [ + {ssl_options, [ + {verify_peer_ext_key_usage, undefined} + | ?config(ssl_config, Config) + ]} + ], + emqx_listeners:start_listener(ssl, Port, Options), + %% when client connect with cert without keyusages ext + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + %% Then connection success + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_success_verify_peer_ext_key_usage_matched_predefined(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% Give listener keyusage is set to clientAuth + Options = [ + {ssl_options, [ + {verify_peer_ext_key_usage, "clientAuth"} + | ?config(ssl_config, Config) + ]} + ], + + %% When client cert has clientAuth that is matched + gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth"), + emqx_listeners:start_listener(ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + ], + 1000 + ), + %% Then connection success + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_success_verify_peer_ext_key_usage_matched_raw_oid(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% Give listener keyusage is set to raw OID + + %% from OTP-PUB-KEY.hrl + Options = [ + {ssl_options, [ + {verify_peer_ext_key_usage, "OID:1.3.6.1.5.5.7.3.2"} + | ?config(ssl_config, Config) + ]} + ], + emqx_listeners:start_listener(ssl, Port, Options), + %% When client cert has keyusage and matched. + gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth"), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + ], + 1000 + ), + %% Then connection success + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_success_verify_peer_ext_key_usage_matched_ordered_list(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + + %% Give listener keyusage is clientAuth,serverAuth + Options = [ + {ssl_options, [ + {verify_peer_ext_key_usage, "clientAuth,serverAuth"} + | ?config(ssl_config, Config) + ]} + ], + emqx_listeners:start_listener(ssl, Port, Options), + %% When client cert has the same keyusage ext list + gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth,serverAuth"), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + ], + 1000 + ), + %% Then connection success + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_success_verify_peer_ext_key_usage_matched_unordered_list(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% Give listener keyusage is clientAuth,serverAuth + Options = [ + {ssl_options, [ + {verify_peer_ext_key_usage, "serverAuth,clientAuth"} + | ?config(ssl_config, Config) + ]} + ], + emqx_listeners:start_listener(ssl, Port, Options), + %% When client cert has the same keyusage ext list but different order + gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth,serverAuth"), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + ], + 1000 + ), + %% Then connection success + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_fail_verify_peer_ext_key_usage_unmatched_raw_oid(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% Give listener keyusage is using OID + Options = [ + {ssl_options, [ + {verify_peer_ext_key_usage, "OID:1.3.6.1.5.5.7.3.1"} + | ?config(ssl_config, Config) + ]} + ], + emqx_listeners:start_listener(ssl, Port, Options), + + %% When client cert has the keyusage but not matching OID + gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth"), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + ], + 1000 + ), + + %% Then connecion should fail. + fail_when_no_ssl_alert(Socket, handshake_failure), + ok = ssl:close(Socket). + +t_conn_fail_verify_peer_ext_key_usage_empty_str(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {verify_peer_ext_key_usage, ""} + | ?config(ssl_config, Config) + ]} + ], + %% Give listener keyusage is empty string + emqx_listeners:start_listener(ssl, Port, Options), + %% When client connect with cert without keyusage + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + %% Then connecion should fail. + fail_when_no_ssl_alert(Socket, handshake_failure), + ok = ssl:close(Socket). + +t_conn_fail_client_keyusage_unmatch(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + + %% Give listener keyusage is clientAuth + Options = [ + {ssl_options, [ + {verify_peer_ext_key_usage, "clientAuth"} + | ?config(ssl_config, Config) + ]} + ], + emqx_listeners:start_listener(ssl, Port, Options), + %% When client connect with mismatch cert keyusage = codeSigning + gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "codeSigning"), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + ], + 1000 + ), + %% Then connecion should fail. + fail_when_no_ssl_alert(Socket, handshake_failure), + ok = ssl:close(Socket). + +t_conn_fail_client_keyusage_incomplete(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% Give listener keyusage is codeSigning,clientAuth + Options = [ + {ssl_options, [ + {verify_peer_ext_key_usage, + "serverAuth,clientAuth,codeSigning,emailProtection,timeStamping,ocspSigning"} + | ?config(ssl_config, Config) + ]} + ], + emqx_listeners:start_listener(ssl, Port, Options), + %% When client connect with cert keyusage = clientAuth + gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "codeSigning"), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + %% Then connection should fail + fail_when_no_ssl_alert(Socket, handshake_failure), + ok = ssl:close(Socket). + +%%% +%%% Helpers +%%% +gen_client_cert_ext_keyusage(Name, CA, DataDir, Usage) when is_atom(Name) -> + gen_client_cert_ext_keyusage(atom_to_list(Name), CA, DataDir, Usage); +gen_client_cert_ext_keyusage(Name, CA, DataDir, Usage) -> + gen_host_cert(Name, CA, DataDir, #{ext => "extendedKeyUsage=" ++ Usage}). + +client_key_file(DataDir, Name) -> + filename:join(DataDir, Name) ++ ".key". + +client_pem_file(DataDir, Name) -> + filename:join(DataDir, Name) ++ ".pem". + +ssl_config_verify_peer_full_chain(Config) -> + [ + {cacertfile, filename:join(?config(data_dir, Config), "intermediate1-root-bundle.pem")} + | ssl_config_verify_peer(Config) + ]. +ssl_config_verify_peer_partial_chain(Config) -> + [ + {cacertfile, filename:join(?config(data_dir, Config), "intermediate1.pem")}, + {partial_chain, true} + | ssl_config_verify_peer(Config) + ]. + +ssl_config_verify_peer(Config) -> + DataDir = ?config(data_dir, Config), + [ + {verify, verify_peer}, + {fail_if_no_peer_cert, true}, + {keyfile, filename:join(DataDir, "server1.key")}, + {certfile, filename:join(DataDir, "server1.pem")} + %% , {log_level, debug} + ]. diff --git a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl new file mode 100644 index 000000000..5ca00bc1d --- /dev/null +++ b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl @@ -0,0 +1,668 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_listener_tls_verify_partial_chain_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import( + emqx_test_tls_certs_helper, + [ + emqx_start_listener/4, + fail_when_ssl_error/1, + fail_when_no_ssl_alert/2, + generate_tls_certs/1 + ] +). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + generate_tls_certs(Config), + application:ensure_all_started(esockd), + dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), + dbg:p(all, c), + dbg:tpl(emqx_listeners, esockd_opts, cx), + dbg:tpl(emqx_listeners, inject_root_fun, cx), + dbg:tpl(esockd, open, cx), + + [{ssl_config, ssl_config_verify_partial_chain()} | Config]. + +end_per_suite(_Config) -> + application:stop(esockd). + +t_conn_success_with_server_intermediate_cacert_and_client_cert(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ssl:close(Socket). + +t_conn_success_with_intermediate_cacert_bundle(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "server1-intermediate1-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ssl:close(Socket). + +t_conn_success_with_renewed_intermediate_cacert(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1_renewed.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ssl:close(Socket). + +t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_complete_bundle(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ssl:close(Socket). + +t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_bundle(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ssl:close(Socket). + +t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_renewed_client_cert( + Config +) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")}, + {partial_chain, two_cacerts_from_cacertfile} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2_renewed.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ssl:close(Socket). + +%% Note, this is good to have for usecase coverage +t_conn_success_with_new_intermediate_cacert_and_client_provides_renewed_client_cert_signed_by_old_intermediate( + Config +) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2_renewed.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ssl:close(Socket). + +%% @doc server should build a partial_chain with old version of ca cert. +t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_client_cert(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")}, + {partial_chain, two_cacerts_from_cacertfile} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ssl:close(Socket). + +%% @doc verify when config does not allow two versions of certs from same trusted CA. +t_conn_fail_with_renewed_and_old_intermediate_cacert_and_client_using_old_bundle(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ssl:close(Socket). + +%% @doc verify when config (two_cacerts_from_cacertfile) allows two versions of certs from same trusted CA. +t_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_old_bundle(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")}, + {partial_chain, two_cacerts_from_cacertfile} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ssl:close(Socket). + +%% @doc: verify even if listener has old/new intermediate2 certs, +%% client1 should not able to connect with old intermediate2 cert. +%% In this case, listener verify_fun returns {trusted_ca, Oldintermediate2Cert} but OTP should still fail the validation +%% since the client1 cert is not signed by Oldintermediate2Cert (trusted CA cert). +%% @end +t_fail_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_all_CAcerts(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")}, + {partial_chain, two_cacerts_from_cacertfile} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "all-CAcerts-bundle.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ssl:close(Socket). + +t_conn_fail_with_renewed_intermediate_cacert_other_client(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1_renewed.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ssl:close(Socket). + +t_conn_fail_with_intermediate_cacert_bundle_but_incorrect_order(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1-server1-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ssl:close(Socket). + +t_conn_fail_when_singed_by_other_intermediate_ca(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ok = ssl:close(Socket). + +t_conn_success_with_complete_chain_that_server_root_cacert_and_client_complete_cert_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "root.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_fail_with_other_client_complete_cert_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ok = ssl:close(Socket). + +t_conn_fail_with_server_intermediate_and_other_client_complete_cert_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1-root-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_success_with_server_intermediate_cacert_and_client_complete_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_fail_with_server_intermediate_chain_and_client_other_incomplete_cert_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ok = ssl:close(Socket). + +t_conn_fail_with_server_intermediate_and_other_client_root_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-root-bundle.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ok = ssl:close(Socket). + +t_conn_success_with_server_intermediate_and_client_root_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate2.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-root-bundle.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +%% @doc once rootCA cert present in cacertfile, sibling CA signed Client cert could connect. +t_conn_success_with_server_all_CA_bundle_and_client_root_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "all-CAcerts-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-root-bundle.pem")} + ], + 1000 + ), + fail_when_ssl_error(Socket), + ok = ssl:close(Socket). + +t_conn_fail_with_server_two_IA_bundle_and_client_root_chain(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "two-intermediates-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client2.key")}, + {certfile, filename:join(DataDir, "client2-root-bundle.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ok = ssl:close(Socket). + +t_conn_fail_with_server_partial_chain_false_intermediate_cacert_and_client_cert(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")}, + {partial_chain, false} + | ?config(ssl_config, Config) + ]} + ], + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), + {ok, Socket} = ssl:connect( + {127, 0, 0, 1}, + Port, + [ + {keyfile, filename:join(DataDir, "client1.key")}, + {certfile, filename:join(DataDir, "client1.pem")} + ], + 1000 + ), + fail_when_no_ssl_alert(Socket, unknown_ca), + ssl:close(Socket). + +t_error_handling_invalid_cacertfile(Config) -> + Port = emqx_test_tls_certs_helper:select_free_port(ssl), + DataDir = ?config(data_dir, Config), + %% trigger error + Options = [ + {ssl_options, [ + {cacertfile, filename:join(DataDir, "server2.key")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + | ?config(ssl_config, Config) + ]} + ], + ?assertException( + throw, + {error, rootfun_trusted_ca_from_cacertfile}, + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options) + ). + +ssl_config_verify_partial_chain() -> + [ + {verify, verify_peer}, + {fail_if_no_peer_cert, true}, + {partial_chain, true} + ]. diff --git a/apps/emqx/test/emqx_test_tls_certs_helper.erl b/apps/emqx/test/emqx_test_tls_certs_helper.erl new file mode 100644 index 000000000..81babac19 --- /dev/null +++ b/apps/emqx/test/emqx_test_tls_certs_helper.erl @@ -0,0 +1,311 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_test_tls_certs_helper). +-export([ + gen_ca/2, + gen_host_cert/3, + gen_host_cert/4, + + select_free_port/1, + generate_tls_certs/1, + + fail_when_ssl_error/1, + fail_when_ssl_error/2, + fail_when_no_ssl_alert/2, + fail_when_no_ssl_alert/3, + + emqx_start_listener/4 +]). + +-include_lib("common_test/include/ct.hrl"). + +%%------------------------------------------------------------------------------- +%% Start Listener +%%------------------------------------------------------------------------------- +emqx_start_listener(Name, Type, Port, Opts) when is_list(Opts) -> + emqx_start_listener(Name, Type, Port, maps:from_list(Opts)); +emqx_start_listener(Name, ssl, Port, #{ssl_options := SslOptions} = Opts0) -> + Opts = Opts0#{ + bind => {{127, 0, 0, 1}, Port}, + mountpoint => <<>>, + zone => default, + ssl_options => maps:from_list(SslOptions) + }, + ct:pal("start listsner with ~p ~p", [Name, Opts]), + emqx_listeners:start_listener(ssl, Name, Opts). + +%%------------------------------------------------------------------------------- +%% TLS certs +%%------------------------------------------------------------------------------- +gen_ca(Path, Name) -> + %% Generate ca.pem and ca.key which will be used to generate certs + %% for hosts server and clients + ECKeyFile = eckey_name(Path), + filelib:ensure_dir(ECKeyFile), + os:cmd("openssl ecparam -name secp256r1 > " ++ ECKeyFile), + Cmd = lists:flatten( + io_lib:format( + "openssl req -new -x509 -nodes " + "-newkey ec:~s " + "-keyout ~s -out ~s -days 3650 " + "-addext basicConstraints=CA:TRUE " + "-subj \"/C=SE/O=TEST CA\"", + [ + ECKeyFile, + ca_key_name(Path, Name), + ca_cert_name(Path, Name) + ] + ) + ), + os:cmd(Cmd). + +ca_cert_name(Path, Name) -> + filename(Path, "~s.pem", [Name]). +ca_key_name(Path, Name) -> + filename(Path, "~s.key", [Name]). + +eckey_name(Path) -> + filename(Path, "ec.key", []). + +gen_host_cert(H, CaName, Path) -> + gen_host_cert(H, CaName, Path, #{}). + +gen_host_cert(H, CaName, Path, Opts) -> + ECKeyFile = eckey_name(Path), + CN = str(H), + HKey = filename(Path, "~s.key", [H]), + HCSR = filename(Path, "~s.csr", [H]), + HCSR2 = filename(Path, "~s.csr", [H]), + HPEM = filename(Path, "~s.pem", [H]), + HPEM2 = filename(Path, "~s_renewed.pem", [H]), + HEXT = filename(Path, "~s.extfile", [H]), + PasswordArg = + case maps:get(password, Opts, undefined) of + undefined -> + " -nodes "; + Password -> + io_lib:format(" -passout pass:'~s' ", [Password]) + end, + + create_file( + HEXT, + "keyUsage=digitalSignature,keyAgreement,keyCertSign\n" + "basicConstraints=CA:TRUE \n" + "~s \n" + "subjectAltName=DNS:~s\n", + [maps:get(ext, Opts, ""), CN] + ), + + CSR_Cmd = csr_cmd(PasswordArg, ECKeyFile, HKey, HCSR, CN), + CSR_Cmd2 = csr_cmd(PasswordArg, ECKeyFile, HKey, HCSR2, CN), + + CERT_Cmd = cert_sign_cmd( + HEXT, HCSR, ca_cert_name(Path, CaName), ca_key_name(Path, CaName), HPEM + ), + %% 2nd cert for testing renewed cert. + CERT_Cmd2 = cert_sign_cmd( + HEXT, HCSR2, ca_cert_name(Path, CaName), ca_key_name(Path, CaName), HPEM2 + ), + ct:pal(os:cmd(CSR_Cmd)), + ct:pal(os:cmd(CSR_Cmd2)), + ct:pal(os:cmd(CERT_Cmd)), + ct:pal(os:cmd(CERT_Cmd2)), + file:delete(HEXT). + +cert_sign_cmd(ExtFile, CSRFile, CACert, CAKey, OutputCert) -> + lists:flatten( + io_lib:format( + "openssl x509 -req " + "-extfile ~s " + "-in ~s -CA ~s -CAkey ~s -CAcreateserial " + "-out ~s -days 500", + [ + ExtFile, + CSRFile, + CACert, + CAKey, + OutputCert + ] + ) + ). + +csr_cmd(PasswordArg, ECKeyFile, HKey, HCSR, CN) -> + lists:flatten( + io_lib:format( + "openssl req -new ~s -newkey ec:~s " + "-keyout ~s -out ~s " + "-addext \"subjectAltName=DNS:~s\" " + "-addext basicConstraints=CA:TRUE " + "-addext keyUsage=digitalSignature,keyAgreement,keyCertSign " + "-subj \"/C=SE/O=TEST/CN=~s\"", + [PasswordArg, ECKeyFile, HKey, HCSR, CN, CN] + ) + ). + +filename(Path, F, A) -> + filename:join(Path, str(io_lib:format(F, A))). + +str(Arg) -> + binary_to_list(iolist_to_binary(Arg)). + +create_file(Filename, Fmt, Args) -> + filelib:ensure_dir(Filename), + {ok, F} = file:open(Filename, [write]), + try + io:format(F, Fmt, Args) + after + file:close(F) + end, + ok. + +%% @doc get unused port from OS +-spec select_free_port(tcp | udp | ssl | quic) -> inets:port_number(). +select_free_port(tcp) -> + select_free_port(gen_tcp, listen); +select_free_port(udp) -> + select_free_port(gen_udp, open); +select_free_port(ssl) -> + select_free_port(tcp); +select_free_port(quic) -> + select_free_port(udp). + +select_free_port(GenModule, Fun) when + GenModule == gen_tcp orelse + GenModule == gen_udp +-> + {ok, S} = GenModule:Fun(0, [{reuseaddr, true}]), + {ok, Port} = inet:port(S), + ok = GenModule:close(S), + case os:type() of + {unix, darwin} -> + %% in MacOS, still get address_in_use after close port + timer:sleep(500); + _ -> + skip + end, + ct:pal("Select free OS port: ~p", [Port]), + Port. + +%% @doc fail the test if ssl_error recvd +%% post check for success conn establishment +fail_when_ssl_error(Socket) -> + fail_when_ssl_error(Socket, 1000). +fail_when_ssl_error(Socket, Timeout) -> + receive + {ssl_error, Socket, _} -> + ct:fail("Handshake failed!") + after Timeout -> + ok + end. + +%% @doc fail the test if no ssl_error recvd +fail_when_no_ssl_alert(Socket, Alert) -> + fail_when_no_ssl_alert(Socket, Alert, 1000). +fail_when_no_ssl_alert(Socket, Alert, Timeout) -> + receive + {ssl_error, Socket, {tls_alert, {Alert, AlertInfo}}} -> + ct:pal("alert info: ~p~n", [AlertInfo]); + {ssl_error, Socket, Other} -> + ct:fail("recv unexpected ssl_error: ~p~n", [Other]) + after Timeout -> + ct:fail("No expected alert: ~p from Socket: ~p ", [Alert, Socket]) + end. + +%% @doc Generate TLS cert chain for tests +generate_tls_certs(Config) -> + DataDir = ?config(data_dir, Config), + gen_ca(DataDir, "root"), + gen_host_cert("intermediate1", "root", DataDir), + gen_host_cert("intermediate2", "root", DataDir), + gen_host_cert("server1", "intermediate1", DataDir), + gen_host_cert("client1", "intermediate1", DataDir), + gen_host_cert("server2", "intermediate2", DataDir), + gen_host_cert("client2", "intermediate2", DataDir), + + %% Build bundles below + os:cmd( + io_lib:format("cat ~p ~p ~p > ~p", [ + filename:join(DataDir, "client2.pem"), + filename:join(DataDir, "intermediate2.pem"), + filename:join(DataDir, "root.pem"), + filename:join(DataDir, "client2-complete-bundle.pem") + ]) + ), + os:cmd( + io_lib:format("cat ~p ~p > ~p", [ + filename:join(DataDir, "client2.pem"), + filename:join(DataDir, "intermediate2.pem"), + filename:join(DataDir, "client2-intermediate2-bundle.pem") + ]) + ), + os:cmd( + io_lib:format("cat ~p ~p > ~p", [ + filename:join(DataDir, "client2.pem"), + filename:join(DataDir, "root.pem"), + filename:join(DataDir, "client2-root-bundle.pem") + ]) + ), + os:cmd( + io_lib:format("cat ~p ~p > ~p", [ + filename:join(DataDir, "server1.pem"), + filename:join(DataDir, "intermediate1.pem"), + filename:join(DataDir, "server1-intermediate1-bundle.pem") + ]) + ), + os:cmd( + io_lib:format("cat ~p ~p > ~p", [ + filename:join(DataDir, "intermediate1.pem"), + filename:join(DataDir, "server1.pem"), + filename:join(DataDir, "intermediate1-server1-bundle.pem") + ]) + ), + os:cmd( + io_lib:format("cat ~p ~p > ~p", [ + filename:join(DataDir, "intermediate1_renewed.pem"), + filename:join(DataDir, "root.pem"), + filename:join(DataDir, "intermediate1_renewed-root-bundle.pem") + ]) + ), + os:cmd( + io_lib:format("cat ~p ~p > ~p", [ + filename:join(DataDir, "intermediate2.pem"), + filename:join(DataDir, "intermediate2_renewed.pem"), + filename:join(DataDir, "intermediate2_renewed_old-bundle.pem") + ]) + ), + os:cmd( + io_lib:format("cat ~p ~p > ~p", [ + filename:join(DataDir, "intermediate1.pem"), + filename:join(DataDir, "root.pem"), + filename:join(DataDir, "intermediate1-root-bundle.pem") + ]) + ), + os:cmd( + io_lib:format("cat ~p ~p ~p > ~p", [ + filename:join(DataDir, "root.pem"), + filename:join(DataDir, "intermediate2.pem"), + filename:join(DataDir, "intermediate1.pem"), + filename:join(DataDir, "all-CAcerts-bundle.pem") + ]) + ), + os:cmd( + io_lib:format("cat ~p ~p > ~p", [ + filename:join(DataDir, "intermediate2.pem"), + filename:join(DataDir, "intermediate1.pem"), + filename:join(DataDir, "two-intermediates-bundle.pem") + ]) + ). From c5657029ab8cad0d15224f07d5df50f8612cb3e2 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 5 Oct 2023 16:07:27 +0200 Subject: [PATCH 06/39] feat(config): partial_chain --- apps/emqx/src/emqx_schema.erl | 8 ++++++++ rel/i18n/emqx_schema.hocon | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index da8b885c6..9cac98d7a 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2178,6 +2178,14 @@ common_ssl_opts_schema(Defaults, Type) -> desc => ?DESC(common_ssl_opts_schema_verify) } )}, + {"partial_chain", + sc( + hoconsc:enum([true, false, two_cacerts_from_cacertfile, cacert_from_cacertfile]), + #{ + default => Df(partial_chain, false), + desc => ?DESC(common_ssl_opts_schema_partial_chain) + } + )}, {"reuse_sessions", sc( boolean(), diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index e80f36817..7d5ac005f 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -684,6 +684,12 @@ common_ssl_opts_schema_verify.desc: common_ssl_opts_schema_verify.label: """Verify peer""" +common_ssl_opts_schema_partial_chain.desc: +"""Enable or disable peer verification with partial_chain""" + +common_ssl_opts_schema_partial_chain.label: +"""Partial chain""" + fields_listeners_ssl.desc: """SSL listeners.""" From 9e5cfea8c073359eeb214d19f54b0c64570c327d Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 5 Oct 2023 16:20:24 +0200 Subject: [PATCH 07/39] test(tls): verify peer keyusage --- ...mqx_listener_tls_verify_keyusage_SUITE.erl | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl index c12618566..54ef07be0 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl +++ b/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl @@ -18,17 +18,19 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --import(emqx_test_tls_certs_helper, [ - fail_when_ssl_error/1, - fail_when_no_ssl_alert/2, - generate_tls_certs/1, - gen_host_cert/4 -]). +-import( + emqx_test_tls_certs_helper, + [ + fail_when_ssl_error/1, + fail_when_no_ssl_alert/2, + generate_tls_certs/1, + gen_host_cert/4, + emqx_start_listener/4 + ] +). all() -> [ @@ -37,7 +39,7 @@ all() -> ]. all_tc() -> - emqx_ct:all(?MODULE). + emqx_common_test_helpers:all(?MODULE). groups() -> [ @@ -68,7 +70,7 @@ t_conn_success_verify_peer_ext_key_usage_unset(Config) -> DataDir = ?config(data_dir, Config), %% Given listener keyusage unset Options = [{ssl_options, ?config(ssl_config, Config)}], - emqx_listeners:start_listener(ssl, Port, Options), + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), %% when client connect with cert without keyusage ext {ok, Socket} = ssl:connect( {127, 0, 0, 1}, @@ -93,7 +95,7 @@ t_conn_success_verify_peer_ext_key_usage_undefined(Config) -> | ?config(ssl_config, Config) ]} ], - emqx_listeners:start_listener(ssl, Port, Options), + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), %% when client connect with cert without keyusages ext {ok, Socket} = ssl:connect( {127, 0, 0, 1}, @@ -121,7 +123,7 @@ t_conn_success_verify_peer_ext_key_usage_matched_predefined(Config) -> %% When client cert has clientAuth that is matched gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth"), - emqx_listeners:start_listener(ssl, Port, Options), + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( {127, 0, 0, 1}, Port, @@ -147,7 +149,7 @@ t_conn_success_verify_peer_ext_key_usage_matched_raw_oid(Config) -> | ?config(ssl_config, Config) ]} ], - emqx_listeners:start_listener(ssl, Port, Options), + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), %% When client cert has keyusage and matched. gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth"), {ok, Socket} = ssl:connect( @@ -174,7 +176,7 @@ t_conn_success_verify_peer_ext_key_usage_matched_ordered_list(Config) -> | ?config(ssl_config, Config) ]} ], - emqx_listeners:start_listener(ssl, Port, Options), + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), %% When client cert has the same keyusage ext list gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth,serverAuth"), {ok, Socket} = ssl:connect( @@ -200,7 +202,7 @@ t_conn_success_verify_peer_ext_key_usage_matched_unordered_list(Config) -> | ?config(ssl_config, Config) ]} ], - emqx_listeners:start_listener(ssl, Port, Options), + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), %% When client cert has the same keyusage ext list but different order gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth,serverAuth"), {ok, Socket} = ssl:connect( @@ -226,7 +228,7 @@ t_conn_fail_verify_peer_ext_key_usage_unmatched_raw_oid(Config) -> | ?config(ssl_config, Config) ]} ], - emqx_listeners:start_listener(ssl, Port, Options), + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), %% When client cert has the keyusage but not matching OID gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth"), @@ -254,7 +256,7 @@ t_conn_fail_verify_peer_ext_key_usage_empty_str(Config) -> ]} ], %% Give listener keyusage is empty string - emqx_listeners:start_listener(ssl, Port, Options), + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), %% When client connect with cert without keyusage {ok, Socket} = ssl:connect( {127, 0, 0, 1}, @@ -280,7 +282,7 @@ t_conn_fail_client_keyusage_unmatch(Config) -> | ?config(ssl_config, Config) ]} ], - emqx_listeners:start_listener(ssl, Port, Options), + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), %% When client connect with mismatch cert keyusage = codeSigning gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "codeSigning"), {ok, Socket} = ssl:connect( @@ -307,7 +309,7 @@ t_conn_fail_client_keyusage_incomplete(Config) -> | ?config(ssl_config, Config) ]} ], - emqx_listeners:start_listener(ssl, Port, Options), + emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), %% When client connect with cert keyusage = clientAuth gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "codeSigning"), {ok, Socket} = ssl:connect( From aa25e3badd6058d5cd9997e38d4dc47f1ebb16f8 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 5 Oct 2023 16:24:39 +0200 Subject: [PATCH 08/39] fix(tls): undefined keyusage --- apps/emqx/src/emqx_tls_lib.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 8ab4a7a5d..6c9916c2d 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -703,7 +703,7 @@ opt_partial_chain(SslOpts) -> %% @doc make verify_fun if set. -spec opt_verify_fun(SslOpts :: map()) -> NewSslOpts :: map(). -opt_verify_fun(#{verify_peer_ext_key_usage := V} = SslOpts) -> +opt_verify_fun(#{verify_peer_ext_key_usage := V} = SslOpts) when V =/= undefined -> SslOpts#{verify_fun => emqx_const_v2:make_tls_verify_fun(verify_cert_extKeyUsage, V)}; opt_verify_fun(SslOpts) -> SslOpts. From c5dccdf526cab2716c0f68cc67782e743b7cc817 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 5 Oct 2023 16:25:09 +0200 Subject: [PATCH 09/39] feat(tls): update schema for TLS keyusage --- apps/emqx/src/emqx_schema.erl | 8 ++++++++ rel/i18n/emqx_schema.hocon | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 9cac98d7a..ce4840eb9 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2186,6 +2186,14 @@ common_ssl_opts_schema(Defaults, Type) -> desc => ?DESC(common_ssl_opts_schema_partial_chain) } )}, + {"verify_peer_ext_key_usage", + sc( + string(), + #{ + required => false, + desc => ?DESC(common_ssl_opts_verify_peer_ext_key_usage) + } + )}, {"reuse_sessions", sc( boolean(), diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 7d5ac005f..ee3dd1095 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -690,6 +690,12 @@ common_ssl_opts_schema_partial_chain.desc: common_ssl_opts_schema_partial_chain.label: """Partial chain""" +common_ssl_opts_verify_peer_ext_key_usage.desc: +"""Verify Extended Key Usage in Peer's certificate""" + +common_ssl_opts_verify_peer_ext_key_usage.label: +"""Verify KeyUsage in cert""" + fields_listeners_ssl.desc: """SSL listeners.""" From 3fc99315e0a54e0b4451cc1116492d34c8f38f77 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 5 Oct 2023 17:02:27 +0200 Subject: [PATCH 10/39] chore: happy elvis --- apps/emqx/src/emqx_const_v2.erl | 1 + apps/emqx/src/emqx_tls_lib.erl | 1 + .../test/emqx_listener_tls_verify_partial_chain_SUITE.erl | 5 +++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_const_v2.erl b/apps/emqx/src/emqx_const_v2.erl index 9fb2b7fa7..a4c321b4c 100644 --- a/apps/emqx/src/emqx_const_v2.erl +++ b/apps/emqx/src/emqx_const_v2.erl @@ -17,6 +17,7 @@ %%-------------------------------------------------------------------- -module(emqx_const_v2). +-elvis([{elvis_style, atom_naming_convention, #{regex => "^([a-z][a-z0-9A-Z]*_?)*(_SUITE)?$"}}]). -export([ make_tls_root_fun/2, diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 6c9916c2d..09a846832 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -15,6 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_tls_lib). +-elvis([{elvis_style, atom_naming_convention, #{regex => "^([a-z][a-z0-9A-Z]*_?)*(_SUITE)?$"}}]). %% version & cipher suites -export([ diff --git a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl index 5ca00bc1d..fa270f5ce 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl +++ b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl @@ -300,8 +300,9 @@ t_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_ %% @doc: verify even if listener has old/new intermediate2 certs, %% client1 should not able to connect with old intermediate2 cert. -%% In this case, listener verify_fun returns {trusted_ca, Oldintermediate2Cert} but OTP should still fail the validation -%% since the client1 cert is not signed by Oldintermediate2Cert (trusted CA cert). +%% In this case, listener verify_fun returns {trusted_ca, Oldintermediate2Cert} but +%% OTP should still fail the validation since the client1 cert is not signed by +%% Oldintermediate2Cert (trusted CA cert). %% @end t_fail_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_all_CAcerts(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), From 788cdbc6dd3b961953b31279eb4c2c1d327a397b Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 6 Oct 2023 15:07:58 +0200 Subject: [PATCH 11/39] fix(listener): remove partial_chain in wss opts --- apps/emqx/src/emqx_listeners.erl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 4e8d6274f..122118c6d 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -637,8 +637,18 @@ ranch_opts(Type, Opts = #{bind := ListenOn}) -> MaxConnections = maps:get(max_connections, Opts, 1024), SocketOpts = case Type of - wss -> tcp_opts(Opts) ++ proplists:delete(handshake_timeout, ssl_opts(Opts)); - ws -> tcp_opts(Opts) + wss -> + tcp_opts(Opts) ++ + lists:filter( + fun + ({partial_chain, _}) -> false; + ({handshake_timeout, _}) -> false; + (_) -> true + end, + ssl_opts(Opts) + ); + ws -> + tcp_opts(Opts) end, #{ num_acceptors => NumAcceptors, From 463d1a187559c9ead9d7d7aba68c28180681d16a Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 6 Oct 2023 15:32:18 +0200 Subject: [PATCH 12/39] fix(test): tls_verify_partial_chain --- ...istener_tls_verify_partial_chain_SUITE.erl | 347 ++++++++++-------- 1 file changed, 184 insertions(+), 163 deletions(-) diff --git a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl index fa270f5ce..872bb9aaf 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl +++ b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl @@ -36,12 +36,6 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> generate_tls_certs(Config), application:ensure_all_started(esockd), - dbg:tracer(process, {fun dbg:dhandler/2, group_leader()}), - dbg:p(all, c), - dbg:tpl(emqx_listeners, esockd_opts, cx), - dbg:tpl(emqx_listeners, inject_root_fun, cx), - dbg:tpl(esockd, open, cx), - [{ssl_config, ssl_config_verify_partial_chain()} | Config]. end_per_suite(_Config) -> @@ -51,12 +45,13 @@ t_conn_success_with_server_intermediate_cacert_and_client_cert(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -75,12 +70,13 @@ t_conn_success_with_intermediate_cacert_bundle(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "server1-intermediate1-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "server1-intermediate1-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -99,12 +95,13 @@ t_conn_success_with_renewed_intermediate_cacert(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1_renewed.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate1_renewed.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -123,12 +120,13 @@ t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_complete_bundl Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -147,12 +145,13 @@ t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_bundle(Config) Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -173,13 +172,14 @@ t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_rene Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")}, - {partial_chain, two_cacerts_from_cacertfile} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")}, + {partial_chain, two_cacerts_from_cacertfile} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -201,12 +201,13 @@ t_conn_success_with_new_intermediate_cacert_and_client_provides_renewed_client_c Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -226,13 +227,14 @@ t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_clie Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")}, - {partial_chain, two_cacerts_from_cacertfile} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")}, + {partial_chain, two_cacerts_from_cacertfile} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -252,12 +254,13 @@ t_conn_fail_with_renewed_and_old_intermediate_cacert_and_client_using_old_bundle Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -273,17 +276,20 @@ t_conn_fail_with_renewed_and_old_intermediate_cacert_and_client_using_old_bundle ssl:close(Socket). %% @doc verify when config (two_cacerts_from_cacertfile) allows two versions of certs from same trusted CA. -t_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_old_bundle(Config) -> +t_001_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_old_bundle( + Config +) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")}, - {partial_chain, two_cacerts_from_cacertfile} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")}, + {partial_chain, two_cacerts_from_cacertfile} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -304,17 +310,18 @@ t_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_ %% OTP should still fail the validation since the client1 cert is not signed by %% Oldintermediate2Cert (trusted CA cert). %% @end -t_fail_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_all_CAcerts(Config) -> +t_conn_fail_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_all_CAcerts(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")}, - {partial_chain, two_cacerts_from_cacertfile} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")}, + {partial_chain, two_cacerts_from_cacertfile} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -333,12 +340,13 @@ t_conn_fail_with_renewed_intermediate_cacert_other_client(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1_renewed.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate1_renewed.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -357,12 +365,13 @@ t_conn_fail_with_intermediate_cacert_bundle_but_incorrect_order(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1-server1-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate1-server1-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -381,12 +390,13 @@ t_conn_fail_when_singed_by_other_intermediate_ca(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -405,12 +415,13 @@ t_conn_success_with_complete_chain_that_server_root_cacert_and_client_complete_c Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "root.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "root.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -429,12 +440,13 @@ t_conn_fail_with_other_client_complete_cert_chain(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -453,12 +465,13 @@ t_conn_fail_with_server_intermediate_and_other_client_complete_cert_chain(Config Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1-root-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate1-root-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -477,12 +490,13 @@ t_conn_success_with_server_intermediate_cacert_and_client_complete_chain(Config) Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate2.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -501,12 +515,13 @@ t_conn_fail_with_server_intermediate_chain_and_client_other_incomplete_cert_chai Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -525,12 +540,13 @@ t_conn_fail_with_server_intermediate_and_other_client_root_chain(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -549,12 +565,13 @@ t_conn_success_with_server_intermediate_and_client_root_chain(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate2.pem")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -574,12 +591,13 @@ t_conn_success_with_server_all_CA_bundle_and_client_root_chain(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "all-CAcerts-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "all-CAcerts-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -598,12 +616,13 @@ t_conn_fail_with_server_two_IA_bundle_and_client_root_chain(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "two-intermediates-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "two-intermediates-bundle.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -622,13 +641,14 @@ t_conn_fail_with_server_partial_chain_false_intermediate_cacert_and_client_cert( Port = emqx_test_tls_certs_helper:select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")}, - {partial_chain, false} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "intermediate1.pem")}, + {certfile, filename:join(DataDir, "server1.pem")}, + {keyfile, filename:join(DataDir, "server1.key")}, + {partial_chain, false} + ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), {ok, Socket} = ssl:connect( @@ -648,12 +668,13 @@ t_error_handling_invalid_cacertfile(Config) -> DataDir = ?config(data_dir, Config), %% trigger error Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "server2.key")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} + {ssl_options, + ?config(ssl_config, Config) ++ + [ + {cacertfile, filename:join(DataDir, "server2.key")}, + {certfile, filename:join(DataDir, "server2.pem")}, + {keyfile, filename:join(DataDir, "server2.key")} + ]} ], ?assertException( throw, From 221b748b0f15006095a794dd524db120c3083c45 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 11 Oct 2023 21:15:37 +0200 Subject: [PATCH 13/39] test(partial_chain): update tcs for OTP-25 --- .../emqx_listener_tls_verify_chain_SUITE.erl | 18 ++-- ...istener_tls_verify_partial_chain_SUITE.erl | 87 +++++++++++-------- apps/emqx/test/emqx_test_tls_certs_helper.erl | 13 ++- 3 files changed, 71 insertions(+), 47 deletions(-) diff --git a/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl index c38426523..a0d4ab9d1 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl +++ b/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl @@ -205,17 +205,18 @@ t_conn_fail_with_server_partial_chain(Config) -> ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + {certfile, filename:join(DataDir, "client2-complete-bundle.pem")}, + {versions, ['tlsv1.2']}, + {verify, verify_none} ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ok = ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_without_root_cacert(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -229,17 +230,18 @@ t_conn_fail_without_root_cacert(Config) -> ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}, + %% stick to tlsv1.2 for consistent error message + {versions, ['tlsv1.2']} ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ok = ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). ssl_config_verify_peer() -> [ diff --git a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl index 872bb9aaf..7c5f471b9 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl +++ b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl @@ -60,6 +60,7 @@ t_conn_success_with_server_intermediate_cacert_and_client_cert(Config) -> [ {keyfile, filename:join(DataDir, "client1.key")}, {certfile, filename:join(DataDir, "client1.pem")} + | client_default_tls_opts() ], 1000 ), @@ -85,6 +86,7 @@ t_conn_success_with_intermediate_cacert_bundle(Config) -> [ {keyfile, filename:join(DataDir, "client1.key")}, {certfile, filename:join(DataDir, "client1.pem")} + | client_default_tls_opts() ], 1000 ), @@ -110,6 +112,7 @@ t_conn_success_with_renewed_intermediate_cacert(Config) -> [ {keyfile, filename:join(DataDir, "client1.key")}, {certfile, filename:join(DataDir, "client1.pem")} + | client_default_tls_opts() ], 1000 ), @@ -129,17 +132,17 @@ t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_complete_bundl ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_bundle(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -154,17 +157,17 @@ t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_bundle(Config) ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_renewed_client_cert( Config @@ -188,6 +191,7 @@ t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_rene [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2_renewed.pem")} + | client_default_tls_opts() ], 1000 ), @@ -216,6 +220,7 @@ t_conn_success_with_new_intermediate_cacert_and_client_provides_renewed_client_c [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2_renewed.pem")} + | client_default_tls_opts() ], 1000 ), @@ -243,6 +248,7 @@ t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_clie [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2.pem")} + | client_default_tls_opts() ], 1000 ), @@ -263,17 +269,17 @@ t_conn_fail_with_renewed_and_old_intermediate_cacert_and_client_using_old_bundle ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). %% @doc verify when config (two_cacerts_from_cacertfile) allows two versions of certs from same trusted CA. t_001_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_old_bundle( @@ -298,6 +304,7 @@ t_001_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_us [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + | client_default_tls_opts() ], 1000 ), @@ -324,17 +331,17 @@ t_conn_fail_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_all ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client1.key")}, {certfile, filename:join(DataDir, "all-CAcerts-bundle.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_renewed_intermediate_cacert_other_client(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -349,17 +356,17 @@ t_conn_fail_with_renewed_intermediate_cacert_other_client(Config) -> ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_intermediate_cacert_bundle_but_incorrect_order(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -374,17 +381,17 @@ t_conn_fail_with_intermediate_cacert_bundle_but_incorrect_order(Config) -> ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client1.key")}, {certfile, filename:join(DataDir, "client1.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_when_singed_by_other_intermediate_ca(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -399,17 +406,17 @@ t_conn_fail_when_singed_by_other_intermediate_ca(Config) -> ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ok = ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_success_with_complete_chain_that_server_root_cacert_and_client_complete_cert_chain(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -430,6 +437,7 @@ t_conn_success_with_complete_chain_that_server_root_cacert_and_client_complete_c [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + | client_default_tls_opts() ], 1000 ), @@ -449,17 +457,17 @@ t_conn_fail_with_other_client_complete_cert_chain(Config) -> ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ok = ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_server_intermediate_and_other_client_complete_cert_chain(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -480,6 +488,7 @@ t_conn_fail_with_server_intermediate_and_other_client_complete_cert_chain(Config [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + | client_default_tls_opts() ], 1000 ), @@ -505,6 +514,7 @@ t_conn_success_with_server_intermediate_cacert_and_client_complete_chain(Config) [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + | client_default_tls_opts() ], 1000 ), @@ -524,17 +534,17 @@ t_conn_fail_with_server_intermediate_chain_and_client_other_incomplete_cert_chai ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ok = ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_server_intermediate_and_other_client_root_chain(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -549,17 +559,17 @@ t_conn_fail_with_server_intermediate_and_other_client_root_chain(Config) -> ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-root-bundle.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ok = ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_success_with_server_intermediate_and_client_root_chain(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -580,6 +590,7 @@ t_conn_success_with_server_intermediate_and_client_root_chain(Config) -> [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-root-bundle.pem")} + | client_default_tls_opts() ], 1000 ), @@ -606,6 +617,7 @@ t_conn_success_with_server_all_CA_bundle_and_client_root_chain(Config) -> [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-root-bundle.pem")} + | client_default_tls_opts() ], 1000 ), @@ -625,17 +637,17 @@ t_conn_fail_with_server_two_IA_bundle_and_client_root_chain(Config) -> ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-root-bundle.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ok = ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_server_partial_chain_false_intermediate_cacert_and_client_cert(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -651,17 +663,17 @@ t_conn_fail_with_server_partial_chain_false_intermediate_cacert_and_client_cert( ]} ], emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( + Res = ssl:connect( {127, 0, 0, 1}, Port, [ {keyfile, filename:join(DataDir, "client1.key")}, {certfile, filename:join(DataDir, "client1.pem")} + | client_default_tls_opts() ], 1000 ), - fail_when_no_ssl_alert(Socket, unknown_ca), - ssl:close(Socket). + fail_when_no_ssl_alert(Res, unknown_ca). t_error_handling_invalid_cacertfile(Config) -> Port = emqx_test_tls_certs_helper:select_free_port(ssl), @@ -688,3 +700,6 @@ ssl_config_verify_partial_chain() -> {fail_if_no_peer_cert, true}, {partial_chain, true} ]. + +client_default_tls_opts() -> + [{versions, ['tlsv1.2']}]. diff --git a/apps/emqx/test/emqx_test_tls_certs_helper.erl b/apps/emqx/test/emqx_test_tls_certs_helper.erl index 81babac19..880dc6bfd 100644 --- a/apps/emqx/test/emqx_test_tls_certs_helper.erl +++ b/apps/emqx/test/emqx_test_tls_certs_helper.erl @@ -212,9 +212,16 @@ fail_when_ssl_error(Socket, Timeout) -> ok end. -%% @doc fail the test if no ssl_error recvd -fail_when_no_ssl_alert(Socket, Alert) -> - fail_when_no_ssl_alert(Socket, Alert, 1000). +%% @doc fail the test if no ssl_error +fail_when_no_ssl_alert(Res, Alert) -> + fail_when_no_ssl_alert(Res, Alert, 1000). + +fail_when_no_ssl_alert({error, {tls_alert, {Alert, _}}}, Alert, _Timeout) -> + ok; +fail_when_no_ssl_alert({error, _} = Other, Alert, _Timeout) -> + ct:fail("returned unexpected ssl_error: ~p, expected ~n", [Other, Alert]); +fail_when_no_ssl_alert({ok, Socket}, Alert, Timeout) -> + fail_when_no_ssl_alert(Socket, Alert, Timeout); fail_when_no_ssl_alert(Socket, Alert, Timeout) -> receive {ssl_error, Socket, {tls_alert, {Alert, AlertInfo}}} -> From 9e196680de982349660c0e28b9c1cace0db4aab4 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 12 Oct 2023 10:20:02 +0200 Subject: [PATCH 14/39] chore: add changelog --- changes/ce/feat-11721.en.md | 5 +++++ changes/ce/feat-11721.zh.md | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 changes/ce/feat-11721.en.md create mode 100644 changes/ce/feat-11721.zh.md diff --git a/changes/ce/feat-11721.en.md b/changes/ce/feat-11721.en.md new file mode 100644 index 000000000..0dfa3245a --- /dev/null +++ b/changes/ce/feat-11721.en.md @@ -0,0 +1,5 @@ +Port two TLS handshake validation features from emqx 4.4 + +- partial_chain support +- Certificate KeyUsage Validation + diff --git a/changes/ce/feat-11721.zh.md b/changes/ce/feat-11721.zh.md new file mode 100644 index 000000000..e448f0953 --- /dev/null +++ b/changes/ce/feat-11721.zh.md @@ -0,0 +1,4 @@ + 移植 emqx 4.4 中的两项 TLS 握手验证功能 + +- 支持部分链 ( partial_chain ) +- 证书密钥使用验证 From abbf2ef62f771bc6f734853fe512e787149609df Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 30 Apr 2024 09:01:52 +0200 Subject: [PATCH 15/39] chore(TLS-chain-test): update for OTP 26 --- apps/emqx/src/emqx_const_v2.erl | 2 +- .../emqx_listener_tls_verify_chain_SUITE.erl | 23 ++++++++----- ...mqx_listener_tls_verify_keyusage_SUITE.erl | 32 ++++++++++++------- ...istener_tls_verify_partial_chain_SUITE.erl | 7 ++-- apps/emqx/test/emqx_test_tls_certs_helper.erl | 3 +- changes/ce/feat-11721.zh.md | 4 --- 6 files changed, 44 insertions(+), 27 deletions(-) delete mode 100644 changes/ce/feat-11721.zh.md diff --git a/apps/emqx/src/emqx_const_v2.erl b/apps/emqx/src/emqx_const_v2.erl index a4c321b4c..a3b7980ff 100644 --- a/apps/emqx/src/emqx_const_v2.erl +++ b/apps/emqx/src/emqx_const_v2.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl index a0d4ab9d1..0b445c939 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl +++ b/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -58,7 +58,8 @@ t_conn_fail_with_intermediate_ca_cert(Config) -> Port, [ {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} + {certfile, filename:join(DataDir, "client1.pem")}, + {verify, verify_none} ], 1000 ), @@ -83,7 +84,8 @@ t_conn_fail_with_other_intermediate_ca_cert(Config) -> Port, [ {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2.pem")} + {certfile, filename:join(DataDir, "client2.pem")}, + {verify, verify_none} ], 1000 ), @@ -110,7 +112,8 @@ t_conn_success_with_server_client_composed_complete_chain(Config) -> Port, [ {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}, + {verify, verify_none} ], 1000 ), @@ -136,7 +139,8 @@ t_conn_success_with_other_signed_client_composed_complete_chain(Config) -> Port, [ {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} + {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}, + {verify, verify_none} ], 1000 ), @@ -161,7 +165,8 @@ t_conn_success_with_renewed_intermediate_root_bundle(Config) -> Port, [ {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} + {certfile, filename:join(DataDir, "client1.pem")}, + {verify, verify_none} ], 1000 ), @@ -185,7 +190,8 @@ t_conn_success_with_client_complete_cert_chain(Config) -> Port, [ {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} + {certfile, filename:join(DataDir, "client2-complete-bundle.pem")}, + {verify, verify_none} ], 1000 ), @@ -237,7 +243,8 @@ t_conn_fail_without_root_cacert(Config) -> {keyfile, filename:join(DataDir, "client2.key")}, {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}, %% stick to tlsv1.2 for consistent error message - {versions, ['tlsv1.2']} + {versions, ['tlsv1.2']}, + {cacertfile, filename:join(DataDir, "intermediate2.pem")} ], 1000 ), diff --git a/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl index 54ef07be0..8265a7492 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl +++ b/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -77,7 +77,8 @@ t_conn_success_verify_peer_ext_key_usage_unset(Config) -> Port, [ {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} + {certfile, filename:join(DataDir, "client1.pem")}, + {verify, verify_none} ], 1000 ), @@ -102,7 +103,8 @@ t_conn_success_verify_peer_ext_key_usage_undefined(Config) -> Port, [ {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} + {certfile, filename:join(DataDir, "client1.pem")}, + {verify, verify_none} ], 1000 ), @@ -129,7 +131,8 @@ t_conn_success_verify_peer_ext_key_usage_matched_predefined(Config) -> Port, [ {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, + {verify, verify_none} ], 1000 ), @@ -157,7 +160,8 @@ t_conn_success_verify_peer_ext_key_usage_matched_raw_oid(Config) -> Port, [ {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, + {verify, verify_none} ], 1000 ), @@ -184,7 +188,8 @@ t_conn_success_verify_peer_ext_key_usage_matched_ordered_list(Config) -> Port, [ {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, + {verify, verify_none} ], 1000 ), @@ -210,7 +215,8 @@ t_conn_success_verify_peer_ext_key_usage_matched_unordered_list(Config) -> Port, [ {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, + {verify, verify_none} ], 1000 ), @@ -237,7 +243,8 @@ t_conn_fail_verify_peer_ext_key_usage_unmatched_raw_oid(Config) -> Port, [ {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, + {verify, verify_none} ], 1000 ), @@ -263,7 +270,8 @@ t_conn_fail_verify_peer_ext_key_usage_empty_str(Config) -> Port, [ {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} + {certfile, filename:join(DataDir, "client1.pem")}, + {verify, verify_none} ], 1000 ), @@ -290,7 +298,8 @@ t_conn_fail_client_keyusage_unmatch(Config) -> Port, [ {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)} + {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, + {verify, verify_none} ], 1000 ), @@ -317,7 +326,8 @@ t_conn_fail_client_keyusage_incomplete(Config) -> Port, [ {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} + {certfile, filename:join(DataDir, "client1.pem")}, + {verify, verify_none} ], 1000 ), diff --git a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl index 7c5f471b9..1a1963dc9 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl +++ b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -702,4 +702,7 @@ ssl_config_verify_partial_chain() -> ]. client_default_tls_opts() -> - [{versions, ['tlsv1.2']}]. + [ + {versions, ['tlsv1.2']}, + {verify, verify_none} + ]. diff --git a/apps/emqx/test/emqx_test_tls_certs_helper.erl b/apps/emqx/test/emqx_test_tls_certs_helper.erl index 880dc6bfd..759b42821 100644 --- a/apps/emqx/test/emqx_test_tls_certs_helper.erl +++ b/apps/emqx/test/emqx_test_tls_certs_helper.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -40,6 +40,7 @@ emqx_start_listener(Name, Type, Port, Opts) when is_list(Opts) -> emqx_start_listener(Name, Type, Port, maps:from_list(Opts)); emqx_start_listener(Name, ssl, Port, #{ssl_options := SslOptions} = Opts0) -> Opts = Opts0#{ + enable => true, bind => {{127, 0, 0, 1}, Port}, mountpoint => <<>>, zone => default, diff --git a/changes/ce/feat-11721.zh.md b/changes/ce/feat-11721.zh.md deleted file mode 100644 index e448f0953..000000000 --- a/changes/ce/feat-11721.zh.md +++ /dev/null @@ -1,4 +0,0 @@ - 移植 emqx 4.4 中的两项 TLS 握手验证功能 - -- 支持部分链 ( partial_chain ) -- 证书密钥使用验证 From 1739bc0c24439ec299ccbac24052fb2eef39460b Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 30 Apr 2024 16:41:26 +0200 Subject: [PATCH 16/39] feat(partial_chain): gateway support --- apps/emqx_gateway/src/emqx_gateway_utils.erl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 8fd9a1519..3150ec675 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -559,6 +559,8 @@ ssl_opts(Name, Opts) -> [ fun ssl_opts_crl_config/2, fun ssl_opts_drop_unsupported/2, + fun ssl_partial_chain/2, + fun ssl_verify_fun/2, fun ssl_server_opts/2 ], SSLOpts, @@ -586,6 +588,12 @@ ssl_server_opts(SSLOpts, ssl_options) -> ssl_server_opts(SSLOpts, dtls_options) -> emqx_tls_lib:to_server_opts(dtls, SSLOpts). +ssl_partial_chain(SSLOpts, _Options) -> + emqx_tls_lib:opt_partial_chain(SSLOpts). + +ssl_verify_fun(SSLOpts, _Options) -> + emqx_tls_lib:opt_verify_fun(SSLOpts). + ranch_opts(Type, ListenOn, Opts) -> NumAcceptors = maps:get(acceptors, Opts, 4), MaxConnections = maps:get(max_connections, Opts, 1024), From e9b813d8ef1f5be2203a6f5a71bd2485ef03a0a2 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 30 Apr 2024 16:41:46 +0200 Subject: [PATCH 17/39] chore: fix test --- apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index a2d4d21af..496192e39 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -421,6 +421,7 @@ t_create_webhook_v1_bridges_api(Config) -> <<"enable">> => true, <<"hibernate_after">> => <<"5s">>, <<"log_level">> => <<"notice">>, + <<"partial_chain">> => false, <<"reuse_sessions">> => true, <<"secure_renegotiate">> => true, <<"user_lookup_fun">> => From e60380d205fbdb8a6f0dc9c749149bb3b3508d13 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 2 May 2024 10:13:57 +0200 Subject: [PATCH 18/39] chore: fix nit --- apps/emqx/src/emqx_const_v2.erl | 16 +++++++--------- apps/emqx/test/emqx_test_tls_certs_helper.erl | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/emqx/src/emqx_const_v2.erl b/apps/emqx/src/emqx_const_v2.erl index a3b7980ff..0d95cf43c 100644 --- a/apps/emqx/src/emqx_const_v2.erl +++ b/apps/emqx/src/emqx_const_v2.erl @@ -49,8 +49,8 @@ make_tls_root_fun(cacert_from_cacertfile, [TrustedOne, TrustedTwo]) -> end. make_tls_verify_fun(verify_cert_extKeyUsage, KeyUsages) -> - AllowedKeyUsages = ext_key_opts(KeyUsages), - {fun verify_fun_peer_extKeyUsage/3, AllowedKeyUsages}. + RequiredKeyUsages = ext_key_opts(KeyUsages), + {fun verify_fun_peer_extKeyUsage/3, RequiredKeyUsages}. verify_fun_peer_extKeyUsage(_, {bad_cert, invalid_ext_key_usage}, UserState) -> %% !! Override OTP verify peer default @@ -69,17 +69,17 @@ verify_fun_peer_extKeyUsage( #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{extensions = ExtL}}, %% valid peer cert valid_peer, - AllowedKeyUsages + RequiredKeyUsages ) -> %% override OTP verify_peer default %% must have id-ce-extKeyUsage case lists:keyfind(?'id-ce-extKeyUsage', 2, ExtL) of #'Extension'{extnID = ?'id-ce-extKeyUsage', extnValue = VL} -> - case do_verify_ext_key_usage(VL, AllowedKeyUsages) of + case do_verify_ext_key_usage(VL, RequiredKeyUsages) of true -> %% pass the check, %% fallback to OTP verify_peer default - {valid, AllowedKeyUsages}; + {valid, RequiredKeyUsages}; false -> {fail, extKeyUsage_unmatched} end; @@ -100,9 +100,7 @@ do_verify_ext_key_usage(CertExtL, [Usage | T] = _Required) -> end. %% @doc Helper tls cert extension --spec ext_key_opts - (string()) -> [OidString :: string() | public_key:oid()]; - (undefined) -> undefined. +-spec ext_key_opts(string()) -> [OidString :: string() | public_key:oid()]. ext_key_opts(Str) -> Usages = string:tokens(Str, ","), lists:map( @@ -119,7 +117,7 @@ ext_key_opts(Str) -> ?'id-kp-timeStamping'; ("ocspSigning") -> ?'id-kp-OCSPSigning'; - ([$O, $I, $D, $: | OidStr]) -> + ("OID:" ++ OidStr) -> OidList = string:tokens(OidStr, "."), list_to_tuple(lists:map(fun list_to_integer/1, OidList)) end, diff --git a/apps/emqx/test/emqx_test_tls_certs_helper.erl b/apps/emqx/test/emqx_test_tls_certs_helper.erl index 759b42821..78d51c5e0 100644 --- a/apps/emqx/test/emqx_test_tls_certs_helper.erl +++ b/apps/emqx/test/emqx_test_tls_certs_helper.erl @@ -46,7 +46,7 @@ emqx_start_listener(Name, ssl, Port, #{ssl_options := SslOptions} = Opts0) -> zone => default, ssl_options => maps:from_list(SslOptions) }, - ct:pal("start listsner with ~p ~p", [Name, Opts]), + ct:pal("start listener with ~p ~p", [Name, Opts]), emqx_listeners:start_listener(ssl, Name, Opts). %%------------------------------------------------------------------------------- From 38115f923394b5bf541ccc060371794257ca3d91 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 6 May 2024 11:17:45 +0200 Subject: [PATCH 19/39] chore: update doc for `partial_chain` and `verify_peer_ext_key_usage` --- changes/ce/feat-11721.en.md | 19 +++++++++++++- rel/i18n/emqx_schema.hocon | 43 +++++++++++++++++++++++++++++-- scripts/spellcheck/dicts/emqx.txt | 7 +++++ 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/changes/ce/feat-11721.en.md b/changes/ce/feat-11721.en.md index 0dfa3245a..42f1f3a2f 100644 --- a/changes/ce/feat-11721.en.md +++ b/changes/ce/feat-11721.en.md @@ -1,5 +1,22 @@ -Port two TLS handshake validation features from emqx 4.4 +Enhance TLS listener to support more flexible TLS verifications. - partial_chain support + + If the option `partial_chain` is set to `true`, allow connections with incomplete certificate chains. + + Check the description in emqx schema for more. + - Certificate KeyUsage Validation + Added support for required Extended Key Usage defined in + [rfc5280](https://www.rfc-editor.org/rfc/rfc5280#section-4.2.1.12). + + Introduced a new option (`verify_peer_ext_key_usage`) to require specific key usages (like "serverAuth") + in peer certificates during the TLS handshake. + This strengthens security by ensuring certificates are used for their intended purposes. + + example: + "serverAuth,OID:1.3.6.1.5.5.7.3.2" + + Check the description in emqx schema for more. + diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index ee3dd1095..5b0b07c62 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -685,13 +685,52 @@ common_ssl_opts_schema_verify.label: """Verify peer""" common_ssl_opts_schema_partial_chain.desc: -"""Enable or disable peer verification with partial_chain""" +"""Enable or disable peer verification with partial_chain: +- `false` +- `true` +- `cacert_from_cacertfile` +- `two_cacerts_from_cacertfile` + +When local verifies a peer certificate during the x509 path validation +process, it constructs a certificate chain that starts with the peer +certificate and ends with a trust anchor. + +By default, if the setting is set to `false`, the trust anchor is the +rootCA, and the certificate chain must be complete. + +If the setting is set to `true` or `cacert_from_cacertfile`, +the last certificate in the cacertfile will be used as the trust anchor +certificate (such as an intermediate CA). This creates a partial chain +in the path validation. + +Alternatively, if the setting is set to `two_cacerts_from_cacertfile`, +one of the last two certificates in the cacertfile will be used as the +trust anchor certificate, forming a partial chain. This option is +particularly useful for CA certificate rotation. +However, please note that it incurs some additional overhead, so it +should only be used for certificate rotation purposes.""" common_ssl_opts_schema_partial_chain.label: """Partial chain""" common_ssl_opts_verify_peer_ext_key_usage.desc: -"""Verify Extended Key Usage in Peer's certificate""" +"""Verify Extended Key Usage in Peer's certificate +For additional peer certificate validation, the value defined here must present in the +'Extended Key Usage' of peer certificate defined in +[rfc5280](https://www.rfc-editor.org/rfc/rfc5280#section-4.2.1.12). + +Allowed values are +- "clientAuth" +- "serverAuth" +- "codeSigning" +- "emailProtection" +- "timeStamping" +- "ocspSigning" +- raw OID, for example: "OID:1.3.6.1.5.5.7.3.2" + +Comma-separated string is also supported for validating the subset of key usages. + +For example, `"serverAuth,OID:1.3.6.1.5.5.7.3.2"`""" common_ssl_opts_verify_peer_ext_key_usage.label: """Verify KeyUsage in cert""" diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 7c888af49..201227c34 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -310,3 +310,10 @@ ElasticSearch doc_as_upsert upsert aliyun +rootCA +clientAuth +serverAuth +codeSigning +emailProtection +ocspSigning +OID From 1040c752dbf93ebeee50d62cb84a8dc37dfb17ef Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 6 May 2024 17:02:33 +0200 Subject: [PATCH 20/39] docs: Apply suggestions from code review Co-authored-by: Zaiming (Stone) Shi --- changes/ce/feat-11721.en.md | 4 ++-- rel/i18n/emqx_schema.hocon | 32 ++++++++++++-------------------- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/changes/ce/feat-11721.en.md b/changes/ce/feat-11721.en.md index 42f1f3a2f..37eac8a5f 100644 --- a/changes/ce/feat-11721.en.md +++ b/changes/ce/feat-11721.en.md @@ -4,7 +4,7 @@ Enhance TLS listener to support more flexible TLS verifications. If the option `partial_chain` is set to `true`, allow connections with incomplete certificate chains. - Check the description in emqx schema for more. + Check the configuration manual document for more details. - Certificate KeyUsage Validation @@ -18,5 +18,5 @@ Enhance TLS listener to support more flexible TLS verifications. example: "serverAuth,OID:1.3.6.1.5.5.7.3.2" - Check the description in emqx schema for more. + Check the configuration manual document for more details. diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 5b0b07c62..f46b9268f 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -685,28 +685,20 @@ common_ssl_opts_schema_verify.label: """Verify peer""" common_ssl_opts_schema_partial_chain.desc: -"""Enable or disable peer verification with partial_chain: -- `false` -- `true` -- `cacert_from_cacertfile` -- `two_cacerts_from_cacertfile` - +"""Enable or disable peer verification with partial_chain. When local verifies a peer certificate during the x509 path validation process, it constructs a certificate chain that starts with the peer certificate and ends with a trust anchor. - -By default, if the setting is set to `false`, the trust anchor is the -rootCA, and the certificate chain must be complete. - -If the setting is set to `true` or `cacert_from_cacertfile`, -the last certificate in the cacertfile will be used as the trust anchor -certificate (such as an intermediate CA). This creates a partial chain +By default, if it is set to `false`, the trust anchor is the +Root CA, and the certificate chain must be complete. +However, if the setting is set to `true` or `cacert_from_cacertfile`, +the last certificate in `cacertfile` will be used as the trust anchor +certificate (intermediate CA). This creates a partial chain in the path validation. - -Alternatively, if the setting is set to `two_cacerts_from_cacertfile`, -one of the last two certificates in the cacertfile will be used as the +Alternatively, if it is configured with `two_cacerts_from_cacertfile`, +one of the last two certificates in `cacertfile` will be used as the trust anchor certificate, forming a partial chain. This option is -particularly useful for CA certificate rotation. +particularly useful for intermediate CA certificate rotation. However, please note that it incurs some additional overhead, so it should only be used for certificate rotation purposes.""" @@ -714,7 +706,7 @@ common_ssl_opts_schema_partial_chain.label: """Partial chain""" common_ssl_opts_verify_peer_ext_key_usage.desc: -"""Verify Extended Key Usage in Peer's certificate +"""Verify extended key usage in peer's certificate For additional peer certificate validation, the value defined here must present in the 'Extended Key Usage' of peer certificate defined in [rfc5280](https://www.rfc-editor.org/rfc/rfc5280#section-4.2.1.12). @@ -726,9 +718,9 @@ Allowed values are - "emailProtection" - "timeStamping" - "ocspSigning" -- raw OID, for example: "OID:1.3.6.1.5.5.7.3.2" +- raw OID, for example: "OID:1.3.6.1.5.5.7.3.2" means `id-pk 2` which is equivalent to `clientAuth` -Comma-separated string is also supported for validating the subset of key usages. +Comma-separated string is also supported for validating more than one key usages. For example, `"serverAuth,OID:1.3.6.1.5.5.7.3.2"`""" From 2b50610a60aa3f46e5a47b7f0b14b24359d2e701 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 6 May 2024 21:02:19 +0200 Subject: [PATCH 21/39] chore: fix nit for spellcheck --- rel/i18n/emqx_schema.hocon | 12 ++++++------ scripts/spellcheck/dicts/emqx.txt | 6 ------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index f46b9268f..c6ec68d63 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -712,12 +712,12 @@ For additional peer certificate validation, the value defined here must present [rfc5280](https://www.rfc-editor.org/rfc/rfc5280#section-4.2.1.12). Allowed values are -- "clientAuth" -- "serverAuth" -- "codeSigning" -- "emailProtection" -- "timeStamping" -- "ocspSigning" +- `clientAuth` +- `serverAuth` +- `codeSigning` +- `emailProtection` +- `timeStamping` +- `ocspSigning` - raw OID, for example: "OID:1.3.6.1.5.5.7.3.2" means `id-pk 2` which is equivalent to `clientAuth` Comma-separated string is also supported for validating more than one key usages. diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 201227c34..ce08d0f6b 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -310,10 +310,4 @@ ElasticSearch doc_as_upsert upsert aliyun -rootCA -clientAuth -serverAuth -codeSigning -emailProtection -ocspSigning OID From 7c37bf99656b17a2e7631e7f916cd027e76dbe5a Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 27 May 2024 12:44:30 +0200 Subject: [PATCH 22/39] feat(tls): ee only: TLS partial_chain and Keyusage --- apps/emqx/src/emqx_listeners.erl | 14 ++- apps/emqx/src/emqx_schema.erl | 20 +--- apps/emqx/src/emqx_tls_lib.erl | 54 ----------- apps/emqx_auth_ext/.gitignore | 20 ++++ apps/emqx_auth_ext/BSL.txt | 94 +++++++++++++++++++ apps/emqx_auth_ext/README.md | 7 ++ apps/emqx_auth_ext/rebar.config | 2 + apps/emqx_auth_ext/src/emqx_auth_ext.app.src | 21 +++++ apps/emqx_auth_ext/src/emqx_auth_ext.erl | 6 ++ .../src/emqx_auth_ext_schema.erl | 42 +++++++++ .../src/emqx_auth_ext_tls_const_v1.erl} | 16 +--- .../src/emqx_auth_ext_tls_lib.erl | 66 +++++++++++++ ...h_ext_listener_tls_verify_chain_SUITE.erl} | 36 +++---- ...xt_listener_tls_verify_keyusage_SUITE.erl} | 40 +++----- ...stener_tls_verify_partial_chain_SUITE.erl} | 57 +++++------ .../emqx_auth_ext_test_tls_certs_helper.erl} | 14 +-- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf_schema.erl | 10 ++ apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_gateway/src/emqx_gateway_utils.erl | 14 ++- apps/emqx_machine/priv/reboot_lists.eterm | 3 +- apps/emqx_machine/src/emqx_machine.app.src | 2 +- .../feat-11721.en.md => ee/feat-13128.en.md} | 0 mix.exs | 3 +- rebar.config.erl | 1 + 25 files changed, 362 insertions(+), 184 deletions(-) create mode 100644 apps/emqx_auth_ext/.gitignore create mode 100644 apps/emqx_auth_ext/BSL.txt create mode 100644 apps/emqx_auth_ext/README.md create mode 100644 apps/emqx_auth_ext/rebar.config create mode 100644 apps/emqx_auth_ext/src/emqx_auth_ext.app.src create mode 100644 apps/emqx_auth_ext/src/emqx_auth_ext.erl create mode 100644 apps/emqx_auth_ext/src/emqx_auth_ext_schema.erl rename apps/{emqx/src/emqx_const_v2.erl => emqx_auth_ext/src/emqx_auth_ext_tls_const_v1.erl} (86%) create mode 100644 apps/emqx_auth_ext/src/emqx_auth_ext_tls_lib.erl rename apps/{emqx/test/emqx_listener_tls_verify_chain_SUITE.erl => emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_chain_SUITE.erl} (86%) rename apps/{emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl => emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_keyusage_SUITE.erl} (89%) rename apps/{emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl => emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_partial_chain_SUITE.erl} (93%) rename apps/{emqx/test/emqx_test_tls_certs_helper.erl => emqx_auth_ext/test/emqx_auth_ext_test_tls_certs_helper.erl} (94%) rename changes/{ce/feat-11721.en.md => ee/feat-13128.en.md} (100%) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 122118c6d..aa4fe1516 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -75,6 +75,10 @@ -define(TYPES_STRING, ["tcp", "ssl", "ws", "wss", "quic"]). -define(MARK_DEL, ?TOMBSTONE_CONFIG_CHANGE_REQ). +-ifndef(EMQX_RELEASE_EDITION). +-define(EMQX_RELEASE_EDITION, ce). +-endif. + -spec id_example() -> atom(). id_example() -> 'tcp:default'. @@ -974,15 +978,21 @@ quic_listener_optional_settings() -> stateless_operation_expiration_ms ]. +-if(?EMQX_RELEASE_EDITION == ee). inject_root_fun(#{ssl_options := SslOpts} = Opts) -> - Opts#{ssl_options := emqx_tls_lib:opt_partial_chain(SslOpts)}; + Opts#{ssl_options := emqx_auth_ext_tls_lib:opt_partial_chain(SslOpts)}. +-else. inject_root_fun(Opts) -> Opts. +-endif. +-if(?EMQX_RELEASE_EDITION == ee). inject_verify_fun(#{ssl_options := SslOpts} = Opts) -> - Opts#{ssl_options := emqx_tls_lib:opt_verify_fun(SslOpts)}; + Opts#{ssl_options := emqx_auth_ext_tls_lib:opt_verify_fun(SslOpts)}. +-else. inject_verify_fun(Opts) -> Opts. +-endif. inject_sni_fun(ListenerId, Conf = #{ssl_options := #{ocsp := #{enable_ocsp_stapling := true}}}) -> emqx_ocsp_cache:inject_sni_fun(ListenerId, Conf); diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ce4840eb9..bcb353477 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -191,6 +191,8 @@ -define(DEFAULT_MULTIPLIER, 1.5). -define(DEFAULT_BACKOFF, 0.75). +-define(INJECTING_CONFIGS, [?AUTH_EXT_SCHEMA_MODS]). + namespace() -> emqx. tags() -> @@ -2178,22 +2180,6 @@ common_ssl_opts_schema(Defaults, Type) -> desc => ?DESC(common_ssl_opts_schema_verify) } )}, - {"partial_chain", - sc( - hoconsc:enum([true, false, two_cacerts_from_cacertfile, cacert_from_cacertfile]), - #{ - default => Df(partial_chain, false), - desc => ?DESC(common_ssl_opts_schema_partial_chain) - } - )}, - {"verify_peer_ext_key_usage", - sc( - string(), - #{ - required => false, - desc => ?DESC(common_ssl_opts_verify_peer_ext_key_usage) - } - )}, {"reuse_sessions", sc( boolean(), @@ -2263,7 +2249,7 @@ common_ssl_opts_schema(Defaults, Type) -> desc => ?DESC(common_ssl_opts_schema_hibernate_after) } )} - ]. + ] ++ emqx_schema_hooks:injection_point('common_ssl_opts_schema'). %% @doc Make schema for SSL listener options. -spec server_ssl_opts_schema(map(), boolean()) -> hocon_schema:field_schema(). diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 09a846832..57d26220d 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -24,8 +24,6 @@ default_ciphers/0, selected_ciphers/1, integral_ciphers/2, - opt_partial_chain/1, - opt_verify_fun/1, all_ciphers_set_cached/0 ]). @@ -688,55 +686,3 @@ ensure_ssl_file_key(SSL, RequiredKeyPaths) -> [] -> ok; Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} end. - -%% @doc enable TLS partial_chain validation if set. --spec opt_partial_chain(SslOpts :: map()) -> NewSslOpts :: map(). -opt_partial_chain(#{partial_chain := false} = SslOpts) -> - maps:remove(partial_chain, SslOpts); -opt_partial_chain(#{partial_chain := true} = SslOpts) -> - SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(1, SslOpts)}; -opt_partial_chain(#{partial_chain := cacert_from_cacertfile} = SslOpts) -> - SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(1, SslOpts)}; -opt_partial_chain(#{partial_chain := two_cacerts_from_cacertfile} = SslOpts) -> - SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(2, SslOpts)}; -opt_partial_chain(SslOpts) -> - SslOpts. - -%% @doc make verify_fun if set. --spec opt_verify_fun(SslOpts :: map()) -> NewSslOpts :: map(). -opt_verify_fun(#{verify_peer_ext_key_usage := V} = SslOpts) when V =/= undefined -> - SslOpts#{verify_fun => emqx_const_v2:make_tls_verify_fun(verify_cert_extKeyUsage, V)}; -opt_verify_fun(SslOpts) -> - SslOpts. - -%% @doc Helper, make TLS root_fun -rootfun_trusted_ca_from_cacertfile(NumOfCerts, #{cacertfile := Cacertfile}) -> - case file:read_file(Cacertfile) of - {ok, PemBin} -> - try - do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, PemBin) - catch - _Error:_Info:ST -> - %% The cacertfile will be checked by OTP SSL as well and OTP choice to be silent on this. - %% We are touching security sutffs, don't leak extra info.. - ?SLOG(error, #{ - msg => "trusted_cacert_not_found_in_cacertfile", stacktrace => ST - }), - throw({error, ?FUNCTION_NAME}) - end; - {error, Reason} -> - throw({error, {read_cacertfile_error, Cacertfile, Reason}}) - end; -rootfun_trusted_ca_from_cacertfile(_NumOfCerts, _SslOpts) -> - throw({error, cacertfile_unset}). - -do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, PemBin) -> - %% The last one or two should be the top parent in the chain if it is a chain - Certs = public_key:pem_decode(PemBin), - Pos = length(Certs) - NumOfCerts + 1, - Trusted = [ - CADer - || {'Certificate', CADer, _} <- - lists:sublist(public_key:pem_decode(PemBin), Pos, NumOfCerts) - ], - emqx_const_v2:make_tls_root_fun(cacert_from_cacertfile, Trusted). diff --git a/apps/emqx_auth_ext/.gitignore b/apps/emqx_auth_ext/.gitignore new file mode 100644 index 000000000..df53f7d92 --- /dev/null +++ b/apps/emqx_auth_ext/.gitignore @@ -0,0 +1,20 @@ +.rebar3 +_build +_checkouts +_vendor +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_auth_ext/BSL.txt b/apps/emqx_auth_ext/BSL.txt new file mode 100644 index 000000000..f0cd31c6f --- /dev/null +++ b/apps/emqx_auth_ext/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2028-01-26 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_auth_ext/README.md b/apps/emqx_auth_ext/README.md new file mode 100644 index 000000000..f378988f7 --- /dev/null +++ b/apps/emqx_auth_ext/README.md @@ -0,0 +1,7 @@ +# EMQX Extended Auth Library + +Library that extends EMQX authentication capbility for enterprise. + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_auth_ext/rebar.config b/apps/emqx_auth_ext/rebar.config new file mode 100644 index 000000000..df40dd330 --- /dev/null +++ b/apps/emqx_auth_ext/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, [{emqx, {path, "../emqx"}}]}. diff --git a/apps/emqx_auth_ext/src/emqx_auth_ext.app.src b/apps/emqx_auth_ext/src/emqx_auth_ext.app.src new file mode 100644 index 000000000..9b5034571 --- /dev/null +++ b/apps/emqx_auth_ext/src/emqx_auth_ext.app.src @@ -0,0 +1,21 @@ +{application, emqx_auth_ext, [ + {description, "EMQX Extended Auth Library"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib, + ssl, + emqx + ]}, + {env, []}, + {modules, [ + emqx_auth_ext, + emqx_auth_ext_schema, + emqx_auth_ext_tls_lib, + emqx_auth_ext_tls_const_v1 + ]}, + + {licenses, ["Apache-2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_auth_ext/src/emqx_auth_ext.erl b/apps/emqx_auth_ext/src/emqx_auth_ext.erl new file mode 100644 index 000000000..c6385dd63 --- /dev/null +++ b/apps/emqx_auth_ext/src/emqx_auth_ext.erl @@ -0,0 +1,6 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_auth_ext). + +-export([]). diff --git a/apps/emqx_auth_ext/src/emqx_auth_ext_schema.erl b/apps/emqx_auth_ext/src/emqx_auth_ext_schema.erl new file mode 100644 index 000000000..d3cde3273 --- /dev/null +++ b/apps/emqx_auth_ext/src/emqx_auth_ext_schema.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_auth_ext_schema). +-behaviour(emqx_schema_hooks). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +%%------------------------------------------------------------------------------ +%% emqx_schema_hooks callbacks +%%------------------------------------------------------------------------------ +-export([injected_fields/0]). + +-spec injected_fields() -> #{emqx_schema_hooks:hookpoint() => [hocon_schema:field()]}. +injected_fields() -> + #{ + 'common_ssl_opts_schema' => fields(auth_ext) + }. + +fields(auth_ext) -> + [ + {"partial_chain", + sc( + hoconsc:enum([true, false, two_cacerts_from_cacertfile, cacert_from_cacertfile]), + #{ + default => false, + desc => ?DESC(common_ssl_opts_schema_partial_chain) + } + )}, + {"verify_peer_ext_key_usage", + sc( + string(), + #{ + required => false, + desc => ?DESC(common_ssl_opts_verify_peer_ext_key_usage) + } + )} + ]. + +sc(Type, Meta) -> hoconsc:mk(Type, Meta). diff --git a/apps/emqx/src/emqx_const_v2.erl b/apps/emqx_auth_ext/src/emqx_auth_ext_tls_const_v1.erl similarity index 86% rename from apps/emqx/src/emqx_const_v2.erl rename to apps/emqx_auth_ext/src/emqx_auth_ext_tls_const_v1.erl index 0d95cf43c..ed95b8270 100644 --- a/apps/emqx/src/emqx_const_v2.erl +++ b/apps/emqx_auth_ext/src/emqx_auth_ext_tls_const_v1.erl @@ -1,22 +1,8 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% @doc Never update this module, create a v3 instead. %%-------------------------------------------------------------------- --module(emqx_const_v2). +-module(emqx_auth_ext_tls_const_v1). -elvis([{elvis_style, atom_naming_convention, #{regex => "^([a-z][a-z0-9A-Z]*_?)*(_SUITE)?$"}}]). -export([ diff --git a/apps/emqx_auth_ext/src/emqx_auth_ext_tls_lib.erl b/apps/emqx_auth_ext/src/emqx_auth_ext_tls_lib.erl new file mode 100644 index 000000000..e858920e7 --- /dev/null +++ b/apps/emqx_auth_ext/src/emqx_auth_ext_tls_lib.erl @@ -0,0 +1,66 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_auth_ext_tls_lib). +-elvis([{elvis_style, atom_naming_convention, #{regex => "^([a-z][a-z0-9A-Z]*_?)*(_SUITE)?$"}}]). + +-export([ + opt_partial_chain/1, + opt_verify_fun/1 +]). + +-include_lib("emqx/include/logger.hrl"). + +-define(CONST_MOD_V1, emqx_auth_ext_tls_const_v1). +%% @doc enable TLS partial_chain validation if set. +-spec opt_partial_chain(SslOpts :: map()) -> NewSslOpts :: map(). +opt_partial_chain(#{partial_chain := false} = SslOpts) -> + maps:remove(partial_chain, SslOpts); +opt_partial_chain(#{partial_chain := true} = SslOpts) -> + SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(1, SslOpts)}; +opt_partial_chain(#{partial_chain := cacert_from_cacertfile} = SslOpts) -> + SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(1, SslOpts)}; +opt_partial_chain(#{partial_chain := two_cacerts_from_cacertfile} = SslOpts) -> + SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(2, SslOpts)}; +opt_partial_chain(SslOpts) -> + SslOpts. + +%% @doc make verify_fun if set. +-spec opt_verify_fun(SslOpts :: map()) -> NewSslOpts :: map(). +opt_verify_fun(#{verify_peer_ext_key_usage := V} = SslOpts) when V =/= undefined -> + SslOpts#{verify_fun => ?CONST_MOD_V1:make_tls_verify_fun(verify_cert_extKeyUsage, V)}; +opt_verify_fun(SslOpts) -> + SslOpts. + +%% @doc Helper, make TLS root_fun +rootfun_trusted_ca_from_cacertfile(NumOfCerts, #{cacertfile := Cacertfile}) -> + case file:read_file(emqx_schema:naive_env_interpolation(Cacertfile)) of + {ok, PemBin} -> + try + do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, PemBin) + catch + _Error:_Info:ST -> + %% The cacertfile will be checked by OTP SSL as well and OTP choice to be silent on this. + %% We are touching security sutffs, don't leak extra info.. + ?SLOG(error, #{ + msg => "trusted_cacert_not_found_in_cacertfile", stacktrace => ST + }), + throw({error, ?FUNCTION_NAME}) + end; + {error, Reason} -> + throw({error, {read_cacertfile_error, Cacertfile, Reason}}) + end; +rootfun_trusted_ca_from_cacertfile(_NumOfCerts, _SslOpts) -> + throw({error, cacertfile_unset}). + +do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, PemBin) -> + %% The last one or two should be the top parent in the chain if it is a chain + Certs = public_key:pem_decode(PemBin), + Pos = length(Certs) - NumOfCerts + 1, + Trusted = [ + CADer + || {'Certificate', CADer, _} <- + lists:sublist(public_key:pem_decode(PemBin), Pos, NumOfCerts) + ], + ?CONST_MOD_V1:make_tls_root_fun(cacert_from_cacertfile, Trusted). diff --git a/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_chain_SUITE.erl similarity index 86% rename from apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl rename to apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_chain_SUITE.erl index 0b445c939..3587deb60 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl +++ b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_chain_SUITE.erl @@ -1,19 +1,8 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_listener_tls_verify_chain_SUITE). + +-module(emqx_auth_ext_listener_tls_verify_chain_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -22,12 +11,13 @@ -include_lib("common_test/include/ct.hrl"). -import( - emqx_test_tls_certs_helper, + emqx_auth_ext_test_tls_certs_helper, [ emqx_start_listener/4, fail_when_ssl_error/1, fail_when_no_ssl_alert/2, - generate_tls_certs/1 + generate_tls_certs/1, + select_free_port/1 ] ). @@ -42,7 +32,7 @@ end_per_suite(_Config) -> application:stop(esockd). t_conn_fail_with_intermediate_ca_cert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, [ @@ -68,7 +58,7 @@ t_conn_fail_with_intermediate_ca_cert(Config) -> ok = ssl:close(Socket). t_conn_fail_with_other_intermediate_ca_cert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, [ @@ -94,7 +84,7 @@ t_conn_fail_with_other_intermediate_ca_cert(Config) -> ok = ssl:close(Socket). t_conn_success_with_server_client_composed_complete_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Server has root ca cert Options = [ @@ -121,7 +111,7 @@ t_conn_success_with_server_client_composed_complete_chain(Config) -> ok = ssl:close(Socket). t_conn_success_with_other_signed_client_composed_complete_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Server has root ca cert Options = [ @@ -148,7 +138,7 @@ t_conn_success_with_other_signed_client_composed_complete_chain(Config) -> ok = ssl:close(Socket). t_conn_success_with_renewed_intermediate_root_bundle(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Server has root ca cert Options = [ @@ -174,7 +164,7 @@ t_conn_success_with_renewed_intermediate_root_bundle(Config) -> ok = ssl:close(Socket). t_conn_success_with_client_complete_cert_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, [ @@ -199,7 +189,7 @@ t_conn_success_with_client_complete_cert_chain(Config) -> ok = ssl:close(Socket). t_conn_fail_with_server_partial_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% imcomplete at server side Options = [ @@ -225,7 +215,7 @@ t_conn_fail_with_server_partial_chain(Config) -> fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_without_root_cacert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, [ diff --git a/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_keyusage_SUITE.erl similarity index 89% rename from apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl rename to apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_keyusage_SUITE.erl index 8265a7492..4744f6f9c 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl +++ b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_keyusage_SUITE.erl @@ -1,19 +1,8 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_listener_tls_verify_keyusage_SUITE). + +-module(emqx_auth_ext_listener_tls_verify_keyusage_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -22,13 +11,14 @@ -include_lib("common_test/include/ct.hrl"). -import( - emqx_test_tls_certs_helper, + emqx_auth_ext_test_tls_certs_helper, [ fail_when_ssl_error/1, fail_when_no_ssl_alert/2, generate_tls_certs/1, gen_host_cert/4, - emqx_start_listener/4 + emqx_start_listener/4, + select_free_port/1 ] ). @@ -66,7 +56,7 @@ end_per_group(_, Config) -> Config. t_conn_success_verify_peer_ext_key_usage_unset(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Given listener keyusage unset Options = [{ssl_options, ?config(ssl_config, Config)}], @@ -87,7 +77,7 @@ t_conn_success_verify_peer_ext_key_usage_unset(Config) -> ok = ssl:close(Socket). t_conn_success_verify_peer_ext_key_usage_undefined(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Give listener keyusage is set to undefined Options = [ @@ -113,7 +103,7 @@ t_conn_success_verify_peer_ext_key_usage_undefined(Config) -> ok = ssl:close(Socket). t_conn_success_verify_peer_ext_key_usage_matched_predefined(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Give listener keyusage is set to clientAuth Options = [ @@ -141,7 +131,7 @@ t_conn_success_verify_peer_ext_key_usage_matched_predefined(Config) -> ok = ssl:close(Socket). t_conn_success_verify_peer_ext_key_usage_matched_raw_oid(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Give listener keyusage is set to raw OID @@ -170,7 +160,7 @@ t_conn_success_verify_peer_ext_key_usage_matched_raw_oid(Config) -> ok = ssl:close(Socket). t_conn_success_verify_peer_ext_key_usage_matched_ordered_list(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Give listener keyusage is clientAuth,serverAuth @@ -198,7 +188,7 @@ t_conn_success_verify_peer_ext_key_usage_matched_ordered_list(Config) -> ok = ssl:close(Socket). t_conn_success_verify_peer_ext_key_usage_matched_unordered_list(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Give listener keyusage is clientAuth,serverAuth Options = [ @@ -225,7 +215,7 @@ t_conn_success_verify_peer_ext_key_usage_matched_unordered_list(Config) -> ok = ssl:close(Socket). t_conn_fail_verify_peer_ext_key_usage_unmatched_raw_oid(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Give listener keyusage is using OID Options = [ @@ -254,7 +244,7 @@ t_conn_fail_verify_peer_ext_key_usage_unmatched_raw_oid(Config) -> ok = ssl:close(Socket). t_conn_fail_verify_peer_ext_key_usage_empty_str(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, [ @@ -280,7 +270,7 @@ t_conn_fail_verify_peer_ext_key_usage_empty_str(Config) -> ok = ssl:close(Socket). t_conn_fail_client_keyusage_unmatch(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Give listener keyusage is clientAuth @@ -308,7 +298,7 @@ t_conn_fail_client_keyusage_unmatch(Config) -> ok = ssl:close(Socket). t_conn_fail_client_keyusage_incomplete(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% Give listener keyusage is codeSigning,clientAuth Options = [ diff --git a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_partial_chain_SUITE.erl similarity index 93% rename from apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl rename to apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_partial_chain_SUITE.erl index 1a1963dc9..7563ff86e 100644 --- a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl +++ b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_partial_chain_SUITE.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_listener_tls_verify_partial_chain_SUITE). +-module(emqx_auth_ext_listener_tls_verify_partial_chain_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -22,12 +22,13 @@ -include_lib("common_test/include/ct.hrl"). -import( - emqx_test_tls_certs_helper, + emqx_auth_ext_test_tls_certs_helper, [ emqx_start_listener/4, fail_when_ssl_error/1, fail_when_no_ssl_alert/2, - generate_tls_certs/1 + generate_tls_certs/1, + select_free_port/1 ] ). @@ -42,7 +43,7 @@ end_per_suite(_Config) -> application:stop(esockd). t_conn_success_with_server_intermediate_cacert_and_client_cert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -68,7 +69,7 @@ t_conn_success_with_server_intermediate_cacert_and_client_cert(Config) -> ssl:close(Socket). t_conn_success_with_intermediate_cacert_bundle(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -94,7 +95,7 @@ t_conn_success_with_intermediate_cacert_bundle(Config) -> ssl:close(Socket). t_conn_success_with_renewed_intermediate_cacert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -120,7 +121,7 @@ t_conn_success_with_renewed_intermediate_cacert(Config) -> ssl:close(Socket). t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_complete_bundle(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -145,7 +146,7 @@ t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_complete_bundl fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_bundle(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -172,7 +173,7 @@ t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_bundle(Config) t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_renewed_client_cert( Config ) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -202,7 +203,7 @@ t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_rene t_conn_success_with_new_intermediate_cacert_and_client_provides_renewed_client_cert_signed_by_old_intermediate( Config ) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -229,7 +230,7 @@ t_conn_success_with_new_intermediate_cacert_and_client_provides_renewed_client_c %% @doc server should build a partial_chain with old version of ca cert. t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_client_cert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -257,7 +258,7 @@ t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_clie %% @doc verify when config does not allow two versions of certs from same trusted CA. t_conn_fail_with_renewed_and_old_intermediate_cacert_and_client_using_old_bundle(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -285,7 +286,7 @@ t_conn_fail_with_renewed_and_old_intermediate_cacert_and_client_using_old_bundle t_001_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_old_bundle( Config ) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -318,7 +319,7 @@ t_001_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_us %% Oldintermediate2Cert (trusted CA cert). %% @end t_conn_fail_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_all_CAcerts(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -344,7 +345,7 @@ t_conn_fail_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_all fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_renewed_intermediate_cacert_other_client(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -369,7 +370,7 @@ t_conn_fail_with_renewed_intermediate_cacert_other_client(Config) -> fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_intermediate_cacert_bundle_but_incorrect_order(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -394,7 +395,7 @@ t_conn_fail_with_intermediate_cacert_bundle_but_incorrect_order(Config) -> fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_when_singed_by_other_intermediate_ca(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -419,7 +420,7 @@ t_conn_fail_when_singed_by_other_intermediate_ca(Config) -> fail_when_no_ssl_alert(Res, unknown_ca). t_conn_success_with_complete_chain_that_server_root_cacert_and_client_complete_cert_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -445,7 +446,7 @@ t_conn_success_with_complete_chain_that_server_root_cacert_and_client_complete_c ok = ssl:close(Socket). t_conn_fail_with_other_client_complete_cert_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -470,7 +471,7 @@ t_conn_fail_with_other_client_complete_cert_chain(Config) -> fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_server_intermediate_and_other_client_complete_cert_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -496,7 +497,7 @@ t_conn_fail_with_server_intermediate_and_other_client_complete_cert_chain(Config ok = ssl:close(Socket). t_conn_success_with_server_intermediate_cacert_and_client_complete_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -522,7 +523,7 @@ t_conn_success_with_server_intermediate_cacert_and_client_complete_chain(Config) ok = ssl:close(Socket). t_conn_fail_with_server_intermediate_chain_and_client_other_incomplete_cert_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -547,7 +548,7 @@ t_conn_fail_with_server_intermediate_chain_and_client_other_incomplete_cert_chai fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_server_intermediate_and_other_client_root_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -572,7 +573,7 @@ t_conn_fail_with_server_intermediate_and_other_client_root_chain(Config) -> fail_when_no_ssl_alert(Res, unknown_ca). t_conn_success_with_server_intermediate_and_client_root_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -599,7 +600,7 @@ t_conn_success_with_server_intermediate_and_client_root_chain(Config) -> %% @doc once rootCA cert present in cacertfile, sibling CA signed Client cert could connect. t_conn_success_with_server_all_CA_bundle_and_client_root_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -625,7 +626,7 @@ t_conn_success_with_server_all_CA_bundle_and_client_root_chain(Config) -> ok = ssl:close(Socket). t_conn_fail_with_server_two_IA_bundle_and_client_root_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -650,7 +651,7 @@ t_conn_fail_with_server_two_IA_bundle_and_client_root_chain(Config) -> fail_when_no_ssl_alert(Res, unknown_ca). t_conn_fail_with_server_partial_chain_false_intermediate_cacert_and_client_cert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), Options = [ {ssl_options, @@ -676,7 +677,7 @@ t_conn_fail_with_server_partial_chain_false_intermediate_cacert_and_client_cert( fail_when_no_ssl_alert(Res, unknown_ca). t_error_handling_invalid_cacertfile(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), + Port = select_free_port(ssl), DataDir = ?config(data_dir, Config), %% trigger error Options = [ diff --git a/apps/emqx/test/emqx_test_tls_certs_helper.erl b/apps/emqx_auth_ext/test/emqx_auth_ext_test_tls_certs_helper.erl similarity index 94% rename from apps/emqx/test/emqx_test_tls_certs_helper.erl rename to apps/emqx_auth_ext/test/emqx_auth_ext_test_tls_certs_helper.erl index 78d51c5e0..eaaec3695 100644 --- a/apps/emqx/test/emqx_test_tls_certs_helper.erl +++ b/apps/emqx_auth_ext/test/emqx_auth_ext_test_tls_certs_helper.erl @@ -1,20 +1,8 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_test_tls_certs_helper). +-module(emqx_auth_ext_test_tls_certs_helper). -export([ gen_ca/2, gen_host_cert/3, diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index d09090a74..1c2fbc77a 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 262c517e8..272eda0b4 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -70,9 +70,19 @@ emqx_otel_schema, emqx_mgmt_api_key_schema ]). + +-define(AUTH_EXT_SCHEMA_MODS, [emqx_auth_ext_schema]). + +-if(defined(EMQX_RELEASE_EDITION) andalso ?EMQX_RELEASE_EDITION == ee). +-define(OTHER_INJECTING_CONFIGS, ?AUTH_EXT_SCHEMA_MODS). +-else. +-define(OTHER_INJECTING_CONFIGS, []). +-endif. + -define(INJECTING_CONFIGS, [ {emqx_authn_schema, ?AUTHN_PROVIDER_SCHEMA_MODS}, {emqx_authz_schema, ?AUTHZ_SOURCE_SCHEMA_MODS} + | ?OTHER_INJECTING_CONFIGS ]). %% 1 million default ports counter diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index 3c6634edc..fa8a774ed 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.32"}, + {vsn, "0.1.33"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_auth, emqx_ctl]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 3150ec675..28208683d 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -588,11 +588,21 @@ ssl_server_opts(SSLOpts, ssl_options) -> ssl_server_opts(SSLOpts, dtls_options) -> emqx_tls_lib:to_server_opts(dtls, SSLOpts). +-if(defined(EMQX_RELEASE_EDITION) andalso ?EMQX_RELEASE_EDITION == ee). ssl_partial_chain(SSLOpts, _Options) -> - emqx_tls_lib:opt_partial_chain(SSLOpts). + emqx_auth_ext_tls_lib:opt_partial_chain(SSLOpts). +-else. +ssl_partial_chain(SSLOpts, _) -> + SSLOpts. +-endif. +-if(defined(EMQX_RELEASE_EDITION) andalso ?EMQX_RELEASE_EDITION == ee). ssl_verify_fun(SSLOpts, _Options) -> - emqx_tls_lib:opt_verify_fun(SSLOpts). + emqx_auth_ext_tls_lib:opt_verify_fun(SSLOpts). +-else. +ssl_verify_fun(SSLOpts, _) -> + SSLOpts. +-endif. ranch_opts(Type, ListenOn, Opts) -> NumAcceptors = maps:get(acceptors, Opts, 4), diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 8d5f83698..7477bf8a4 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -129,7 +129,8 @@ emqx_gateway_ocpp, emqx_gateway_jt808, emqx_bridge_syskeeper, - emqx_bridge_confluent + emqx_bridge_confluent, + emqx_auth_ext ], %% must always be of type `load' ce_business_apps => diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 2a74027d9..228d69463 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.3.0"}, + {vsn, "0.3.1"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/changes/ce/feat-11721.en.md b/changes/ee/feat-13128.en.md similarity index 100% rename from changes/ce/feat-11721.en.md rename to changes/ee/feat-13128.en.md diff --git a/mix.exs b/mix.exs index 5432b64ae..63a74ae16 100644 --- a/mix.exs +++ b/mix.exs @@ -200,7 +200,8 @@ defmodule EMQXUmbrella.MixProject do :emqx_gateway_gbt32960, :emqx_gateway_ocpp, :emqx_gateway_jt808, - :emqx_bridge_syskeeper + :emqx_bridge_syskeeper, + :emqx_auth_ext ]) end diff --git a/rebar.config.erl b/rebar.config.erl index 8320cc62a..257293bb2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -119,6 +119,7 @@ is_community_umbrella_app("apps/emqx_bridge_syskeeper") -> false; is_community_umbrella_app("apps/emqx_schema_validation") -> false; is_community_umbrella_app("apps/emqx_eviction_agent") -> false; is_community_umbrella_app("apps/emqx_node_rebalance") -> false; +is_community_umbrella_app("apps/emqx_auth_ext") -> false; is_community_umbrella_app(_) -> true. %% BUILD_WITHOUT_JQ From 76cfc309a984546b147189659c0da93b1d994187 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 27 May 2024 15:12:23 +0200 Subject: [PATCH 23/39] feat(tls): partial_chain not required --- apps/emqx_auth_ext/src/emqx_auth_ext_schema.erl | 2 +- apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_auth_ext/src/emqx_auth_ext_schema.erl b/apps/emqx_auth_ext/src/emqx_auth_ext_schema.erl index d3cde3273..c98524c1e 100644 --- a/apps/emqx_auth_ext/src/emqx_auth_ext_schema.erl +++ b/apps/emqx_auth_ext/src/emqx_auth_ext_schema.erl @@ -25,7 +25,7 @@ fields(auth_ext) -> sc( hoconsc:enum([true, false, two_cacerts_from_cacertfile, cacert_from_cacertfile]), #{ - default => false, + required => false, desc => ?DESC(common_ssl_opts_schema_partial_chain) } )}, diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 496192e39..a2d4d21af 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -421,7 +421,6 @@ t_create_webhook_v1_bridges_api(Config) -> <<"enable">> => true, <<"hibernate_after">> => <<"5s">>, <<"log_level">> => <<"notice">>, - <<"partial_chain">> => false, <<"reuse_sessions">> => true, <<"secure_renegotiate">> => true, <<"user_lookup_fun">> => From 95e515d58529950574291c351d1cdb8ba6604737 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 27 May 2024 17:19:05 +0200 Subject: [PATCH 24/39] docs: add emqx_auth_ext_schema.hocon --- rel/i18n/emqx_auth_ext_schema.hocon | 46 +++++++++++++++++++++++++++++ rel/i18n/emqx_schema.hocon | 42 -------------------------- 2 files changed, 46 insertions(+), 42 deletions(-) create mode 100644 rel/i18n/emqx_auth_ext_schema.hocon diff --git a/rel/i18n/emqx_auth_ext_schema.hocon b/rel/i18n/emqx_auth_ext_schema.hocon new file mode 100644 index 000000000..3589a3436 --- /dev/null +++ b/rel/i18n/emqx_auth_ext_schema.hocon @@ -0,0 +1,46 @@ +emqx_auth_ext_schema { + +common_ssl_opts_schema_partial_chain.desc: +"""Enable or disable peer verification with partial_chain. +When local verifies a peer certificate during the x509 path validation +process, it constructs a certificate chain that starts with the peer +certificate and ends with a trust anchor. +By default, if it is set to `false`, the trust anchor is the +Root CA, and the certificate chain must be complete. +However, if the setting is set to `true` or `cacert_from_cacertfile`, +the last certificate in `cacertfile` will be used as the trust anchor +certificate (intermediate CA). This creates a partial chain +in the path validation. +Alternatively, if it is configured with `two_cacerts_from_cacertfile`, +one of the last two certificates in `cacertfile` will be used as the +trust anchor certificate, forming a partial chain. This option is +particularly useful for intermediate CA certificate rotation. +However, please note that it incurs some additional overhead, so it +should only be used for certificate rotation purposes.""" + +common_ssl_opts_schema_partial_chain.label: +"""Partial chain""" + +common_ssl_opts_verify_peer_ext_key_usage.desc: +"""Verify extended key usage in peer's certificate +For additional peer certificate validation, the value defined here must present in the +'Extended Key Usage' of peer certificate defined in +[rfc5280](https://www.rfc-editor.org/rfc/rfc5280#section-4.2.1.12). + +Allowed values are +- `clientAuth` +- `serverAuth` +- `codeSigning` +- `emailProtection` +- `timeStamping` +- `ocspSigning` +- raw OID, for example: "OID:1.3.6.1.5.5.7.3.2" means `id-pk 2` which is equivalent to `clientAuth` + +Comma-separated string is also supported for validating more than one key usages. + +For example, `"serverAuth,OID:1.3.6.1.5.5.7.3.2"`""" + +common_ssl_opts_verify_peer_ext_key_usage.label: +"""Verify KeyUsage in cert""" + +} diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index c6ec68d63..156d9dce9 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -684,48 +684,6 @@ common_ssl_opts_schema_verify.desc: common_ssl_opts_schema_verify.label: """Verify peer""" -common_ssl_opts_schema_partial_chain.desc: -"""Enable or disable peer verification with partial_chain. -When local verifies a peer certificate during the x509 path validation -process, it constructs a certificate chain that starts with the peer -certificate and ends with a trust anchor. -By default, if it is set to `false`, the trust anchor is the -Root CA, and the certificate chain must be complete. -However, if the setting is set to `true` or `cacert_from_cacertfile`, -the last certificate in `cacertfile` will be used as the trust anchor -certificate (intermediate CA). This creates a partial chain -in the path validation. -Alternatively, if it is configured with `two_cacerts_from_cacertfile`, -one of the last two certificates in `cacertfile` will be used as the -trust anchor certificate, forming a partial chain. This option is -particularly useful for intermediate CA certificate rotation. -However, please note that it incurs some additional overhead, so it -should only be used for certificate rotation purposes.""" - -common_ssl_opts_schema_partial_chain.label: -"""Partial chain""" - -common_ssl_opts_verify_peer_ext_key_usage.desc: -"""Verify extended key usage in peer's certificate -For additional peer certificate validation, the value defined here must present in the -'Extended Key Usage' of peer certificate defined in -[rfc5280](https://www.rfc-editor.org/rfc/rfc5280#section-4.2.1.12). - -Allowed values are -- `clientAuth` -- `serverAuth` -- `codeSigning` -- `emailProtection` -- `timeStamping` -- `ocspSigning` -- raw OID, for example: "OID:1.3.6.1.5.5.7.3.2" means `id-pk 2` which is equivalent to `clientAuth` - -Comma-separated string is also supported for validating more than one key usages. - -For example, `"serverAuth,OID:1.3.6.1.5.5.7.3.2"`""" - -common_ssl_opts_verify_peer_ext_key_usage.label: -"""Verify KeyUsage in cert""" fields_listeners_ssl.desc: """SSL listeners.""" From a1aa9a43750e9f7bba8510015a1c6f9e6888bfd9 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 7 Jun 2024 23:27:20 +0200 Subject: [PATCH 25/39] fix(ee): emqx no longer deps on emqx_auth_ext --- apps/emqx/include/emqx_schema.hrl | 5 ++ apps/emqx/src/emqx_listeners.erl | 23 ++----- apps/emqx/src/emqx_tls_lib.erl | 14 ++++ apps/emqx_auth_ext/src/emqx_auth_ext.erl | 22 +++++++ .../test/emqx_auth_ext_schema_SUITE.erl | 66 +++++++++++++++++++ apps/emqx_gateway/src/emqx_gateway_utils.erl | 14 +--- 6 files changed, 113 insertions(+), 31 deletions(-) create mode 100644 apps/emqx_auth_ext/test/emqx_auth_ext_schema_SUITE.erl diff --git a/apps/emqx/include/emqx_schema.hrl b/apps/emqx/include/emqx_schema.hrl index b0d465e9c..9f9b09b9d 100644 --- a/apps/emqx/include/emqx_schema.hrl +++ b/apps/emqx/include/emqx_schema.hrl @@ -21,4 +21,9 @@ -define(TOMBSTONE_CONFIG_CHANGE_REQ, mark_it_for_deletion). -define(CONFIG_NOT_FOUND_MAGIC, '$0tFound'). +%%-------------------------------------------------------------------- +%% EE injections +%%-------------------------------------------------------------------- +-define(EMQX_SSL_FUN_MFA(Name), {emqx_ssl_fun_mfa, Name}). + -endif. diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index aa4fe1516..e325263c5 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -75,10 +75,6 @@ -define(TYPES_STRING, ["tcp", "ssl", "ws", "wss", "quic"]). -define(MARK_DEL, ?TOMBSTONE_CONFIG_CHANGE_REQ). --ifndef(EMQX_RELEASE_EDITION). --define(EMQX_RELEASE_EDITION, ce). --endif. - -spec id_example() -> atom(). id_example() -> 'tcp:default'. @@ -978,21 +974,10 @@ quic_listener_optional_settings() -> stateless_operation_expiration_ms ]. --if(?EMQX_RELEASE_EDITION == ee). -inject_root_fun(#{ssl_options := SslOpts} = Opts) -> - Opts#{ssl_options := emqx_auth_ext_tls_lib:opt_partial_chain(SslOpts)}. --else. -inject_root_fun(Opts) -> - Opts. --endif. - --if(?EMQX_RELEASE_EDITION == ee). -inject_verify_fun(#{ssl_options := SslOpts} = Opts) -> - Opts#{ssl_options := emqx_auth_ext_tls_lib:opt_verify_fun(SslOpts)}. --else. -inject_verify_fun(Opts) -> - Opts. --endif. +inject_root_fun(#{ssl_options := SSLOpts} = Opts) -> + Opts#{ssl_options := emqx_tls_lib:maybe_inject_ssl_fun(root_fun, SSLOpts)}. +inject_verify_fun(#{ssl_options := SSLOpts} = Opts) -> + Opts#{ssl_options := emqx_tls_lib:maybe_inject_ssl_fun(verify_fun, SSLOpts)}. inject_sni_fun(ListenerId, Conf = #{ssl_options := #{ocsp := #{enable_ocsp_stapling := true}}}) -> emqx_ocsp_cache:inject_sni_fun(ListenerId, Conf); diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 57d26220d..e1de50385 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -45,10 +45,13 @@ to_client_opts/2 ]). +-export([maybe_inject_ssl_fun/2]). + %% ssl:tls_version/0 is not exported. -type tls_version() :: tlsv1 | 'tlsv1.1' | 'tlsv1.2' | 'tlsv1.3'. -include("logger.hrl"). +-include("emqx_schema.hrl"). -define(IS_TRUE(Val), ((Val =:= true) orelse (Val =:= <<"true">>))). -define(IS_FALSE(Val), ((Val =:= false) orelse (Val =:= <<"false">>))). @@ -686,3 +689,14 @@ ensure_ssl_file_key(SSL, RequiredKeyPaths) -> [] -> ok; Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} end. + +-spec maybe_inject_ssl_fun(root_fun | verify_fun, map()) -> map(). +maybe_inject_ssl_fun(FunName, SslOpts) -> + case persistent_term:get(?EMQX_SSL_FUN_MFA(FunName), undefined) of + undefined -> + SslOpts; + {M, F, A} -> + %% We should have one entry not a list of {M,F,A}, + %% as ordering matters in validations + erlang:apply(M, F, [SslOpts | A]) + end. diff --git a/apps/emqx_auth_ext/src/emqx_auth_ext.erl b/apps/emqx_auth_ext/src/emqx_auth_ext.erl index c6385dd63..3558be4a5 100644 --- a/apps/emqx_auth_ext/src/emqx_auth_ext.erl +++ b/apps/emqx_auth_ext/src/emqx_auth_ext.erl @@ -3,4 +3,26 @@ %%-------------------------------------------------------------------- -module(emqx_auth_ext). +-include_lib("emqx/include/emqx_schema.hrl"). + +-on_load(on_load/0). + -export([]). + +-spec on_load() -> ok. +on_load() -> + init_ssl_fun_cb(). + +init_ssl_fun_cb() -> + lists:foreach( + fun({FunName, {_, _, _} = MFA}) -> + persistent_term:put( + ?EMQX_SSL_FUN_MFA(FunName), + MFA + ) + end, + [ + {root_fun, {emqx_auth_ext_tls_lib, opt_partial_chain, []}}, + {verify_fun, {emqx_auth_ext_tls_lib, opt_verify_fun, []}} + ] + ). diff --git a/apps/emqx_auth_ext/test/emqx_auth_ext_schema_SUITE.erl b/apps/emqx_auth_ext/test/emqx_auth_ext_schema_SUITE.erl new file mode 100644 index 000000000..b47f5fa39 --- /dev/null +++ b/apps/emqx_auth_ext/test/emqx_auth_ext_schema_SUITE.erl @@ -0,0 +1,66 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_auth_ext_schema_SUITE). +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(BASE_CONF, + "\n" + " listeners.ssl.auth_ext.bind = 28883\n" + " listeners.ssl.auth_ext.enable = true\n" + " listeners.ssl.auth_ext.ssl_options.partial_chain = true\n" + " listeners.ssl.auth_ext.ssl_options.verify = verify_peer\n" + " listeners.ssl.auth_ext.ssl_options.verify_peer_ext_key_usage = \"clientAuth\"\n" + " " +). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + %% injection happens when module is loaded. + code:load_file(emqx_auth_ext), + Apps = emqx_cth_suite:start( + [ + emqx, + {emqx_conf, ?BASE_CONF} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + emqx_listeners:restart(), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + Apps = ?config(apps, Config), + ok = emqx_cth_suite:stop(Apps), + code:delete(emqx_auth_ext), + code:purge(emqx_auth_ext), + ok. + +t_conf_check_default(_Config) -> + Opts = esockd:get_options({'ssl:default', {{0, 0, 0, 0}, 8883}}), + SSLOpts = proplists:get_value(ssl_options, Opts), + ?assertEqual(none, proplists:lookup(partial_chain, SSLOpts)), + ?assertEqual(none, proplists:lookup(verify_fun, SSLOpts)). + +t_conf_check_auth_ext(_Config) -> + Opts = esockd:get_options({'ssl:auth_ext', 28883}), + SSLOpts = proplists:get_value(ssl_options, Opts), + ?assertMatch(Fun when is_function(Fun), proplists:get_value(partial_chain, SSLOpts)), + ?assertMatch({Fun, _} when is_function(Fun), proplists:get_value(verify_fun, SSLOpts)). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 28208683d..e6a5be8ab 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -588,21 +588,11 @@ ssl_server_opts(SSLOpts, ssl_options) -> ssl_server_opts(SSLOpts, dtls_options) -> emqx_tls_lib:to_server_opts(dtls, SSLOpts). --if(defined(EMQX_RELEASE_EDITION) andalso ?EMQX_RELEASE_EDITION == ee). ssl_partial_chain(SSLOpts, _Options) -> - emqx_auth_ext_tls_lib:opt_partial_chain(SSLOpts). --else. -ssl_partial_chain(SSLOpts, _) -> - SSLOpts. --endif. + emqx_tls_lib:maybe_inject_ssl_fun(root_fun, SSLOpts). --if(defined(EMQX_RELEASE_EDITION) andalso ?EMQX_RELEASE_EDITION == ee). ssl_verify_fun(SSLOpts, _Options) -> - emqx_auth_ext_tls_lib:opt_verify_fun(SSLOpts). --else. -ssl_verify_fun(SSLOpts, _) -> - SSLOpts. --endif. + emqx_tls_lib:maybe_inject_ssl_fun(verify_fun, SSLOpts). ranch_opts(Type, ListenOn, Opts) -> NumAcceptors = maps:get(acceptors, Opts, 4), From 2ca7e9c55e4bda3456633a3d8e0c532c1ebc5da7 Mon Sep 17 00:00:00 2001 From: William Yang Date: Fri, 7 Jun 2024 23:55:12 +0200 Subject: [PATCH 26/39] chore: rename changelog --- changes/ee/{feat-13128.en.md => feat-11721.en.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/ee/{feat-13128.en.md => feat-11721.en.md} (100%) diff --git a/changes/ee/feat-13128.en.md b/changes/ee/feat-11721.en.md similarity index 100% rename from changes/ee/feat-13128.en.md rename to changes/ee/feat-11721.en.md From 64d7d2484f89837475e87be3d716bf0d01088356 Mon Sep 17 00:00:00 2001 From: William Yang Date: Mon, 10 Jun 2024 10:36:13 +0200 Subject: [PATCH 27/39] chore(tls): move changelog again --- changes/ee/{feat-11721.en.md => feat-13211.en.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/ee/{feat-11721.en.md => feat-13211.en.md} (100%) diff --git a/changes/ee/feat-11721.en.md b/changes/ee/feat-13211.en.md similarity index 100% rename from changes/ee/feat-11721.en.md rename to changes/ee/feat-13211.en.md From 44258204bd1543e1010a48cddf9cd6a9dfc1c563 Mon Sep 17 00:00:00 2001 From: William Yang Date: Tue, 11 Jun 2024 10:06:18 +0200 Subject: [PATCH 28/39] chore: move tls_certs test helper to apps/emqx --- .../test/emqx_test_tls_certs_helper.erl} | 2 +- .../test/emqx_auth_ext_listener_tls_verify_chain_SUITE.erl | 2 +- .../test/emqx_auth_ext_listener_tls_verify_keyusage_SUITE.erl | 2 +- .../emqx_auth_ext_listener_tls_verify_partial_chain_SUITE.erl | 2 +- rel/i18n/emqx_schema.hocon | 1 - 5 files changed, 4 insertions(+), 5 deletions(-) rename apps/{emqx_auth_ext/test/emqx_auth_ext_test_tls_certs_helper.erl => emqx/test/emqx_test_tls_certs_helper.erl} (99%) diff --git a/apps/emqx_auth_ext/test/emqx_auth_ext_test_tls_certs_helper.erl b/apps/emqx/test/emqx_test_tls_certs_helper.erl similarity index 99% rename from apps/emqx_auth_ext/test/emqx_auth_ext_test_tls_certs_helper.erl rename to apps/emqx/test/emqx_test_tls_certs_helper.erl index eaaec3695..3cb4923d1 100644 --- a/apps/emqx_auth_ext/test/emqx_auth_ext_test_tls_certs_helper.erl +++ b/apps/emqx/test/emqx_test_tls_certs_helper.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_auth_ext_test_tls_certs_helper). +-module(emqx_test_tls_certs_helper). -export([ gen_ca/2, gen_host_cert/3, diff --git a/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_chain_SUITE.erl b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_chain_SUITE.erl index 3587deb60..5594d825d 100644 --- a/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_chain_SUITE.erl +++ b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_chain_SUITE.erl @@ -11,7 +11,7 @@ -include_lib("common_test/include/ct.hrl"). -import( - emqx_auth_ext_test_tls_certs_helper, + emqx_test_tls_certs_helper, [ emqx_start_listener/4, fail_when_ssl_error/1, diff --git a/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_keyusage_SUITE.erl b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_keyusage_SUITE.erl index 4744f6f9c..6d81277c2 100644 --- a/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_keyusage_SUITE.erl +++ b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_keyusage_SUITE.erl @@ -11,7 +11,7 @@ -include_lib("common_test/include/ct.hrl"). -import( - emqx_auth_ext_test_tls_certs_helper, + emqx_test_tls_certs_helper, [ fail_when_ssl_error/1, fail_when_no_ssl_alert/2, diff --git a/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_partial_chain_SUITE.erl b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_partial_chain_SUITE.erl index 7563ff86e..3f6ec63a1 100644 --- a/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_partial_chain_SUITE.erl +++ b/apps/emqx_auth_ext/test/emqx_auth_ext_listener_tls_verify_partial_chain_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("common_test/include/ct.hrl"). -import( - emqx_auth_ext_test_tls_certs_helper, + emqx_test_tls_certs_helper, [ emqx_start_listener/4, fail_when_ssl_error/1, diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index 156d9dce9..e80f36817 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -684,7 +684,6 @@ common_ssl_opts_schema_verify.desc: common_ssl_opts_schema_verify.label: """Verify peer""" - fields_listeners_ssl.desc: """SSL listeners.""" From 751f7a24e9c6368e82a349ae4f1a449b73233d2a Mon Sep 17 00:00:00 2001 From: zmstone Date: Tue, 11 Jun 2024 19:31:11 +0200 Subject: [PATCH 29/39] feat(authn): support ${cert_pem} placeholder --- apps/emqx/include/emqx_placeholder.hrl | 2 ++ apps/emqx/src/emqx_channel.erl | 22 +++++++++++++++---- .../src/emqx_authn/emqx_authn_utils.erl | 3 +++ .../test/emqx_authn_http_SUITE.erl | 12 +++++++--- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/apps/emqx/include/emqx_placeholder.hrl b/apps/emqx/include/emqx_placeholder.hrl index 31ce6a070..7c538f4cb 100644 --- a/apps/emqx/include/emqx_placeholder.hrl +++ b/apps/emqx/include/emqx_placeholder.hrl @@ -28,8 +28,10 @@ %% cert -define(VAR_CERT_SUBJECT, "cert_subject"). -define(VAR_CERT_CN_NAME, "cert_common_name"). +-define(VAR_CERT_PEM, "cert_pem"). -define(PH_CERT_SUBJECT, ?PH(?VAR_CERT_SUBJECT)). -define(PH_CERT_CN_NAME, ?PH(?VAR_CERT_CN_NAME)). +-define(PH_CERT_PEM, ?PH(?VAR_CERT_PEM)). %% MQTT/Gateway -define(VAR_PASSWORD, "password"). diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index eb54f6ba1..acc4538bd 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1722,6 +1722,16 @@ count_flapping_event(_ConnPkt, #channel{clientinfo = ClientInfo}) -> %%-------------------------------------------------------------------- %% Authenticate +%% If peercert exists, add it as `cert_pem` credential field. +maybe_add_cert(Map, #channel{conninfo = ConnInfo}) -> + maybe_add_cert(Map, ConnInfo); +maybe_add_cert(Map, #{peercert := PeerCert}) when is_binary(PeerCert) -> + %% NOTE: it's raw binary at this point, + %% encoding to PEM (base64) is done lazy in emqx_authn_utils:render_var + Map#{cert_pem => PeerCert}; +maybe_add_cert(Map, _) -> + Map. + authenticate( ?CONNECT_PACKET( #mqtt_packet_connect{ @@ -1734,20 +1744,23 @@ authenticate( auth_cache = AuthCache } = Channel ) -> + %% Auth with CONNECT packet for MQTT v5 AuthData = emqx_mqtt_props:get('Authentication-Data', Properties, undefined), - do_authenticate( + Credential0 = ClientInfo#{ auth_method => AuthMethod, auth_data => AuthData, auth_cache => AuthCache }, - Channel - ); + Credential = maybe_add_cert(Credential0, Channel), + do_authenticate(Credential, Channel); authenticate( ?CONNECT_PACKET(#mqtt_packet_connect{password = Password}), #channel{clientinfo = ClientInfo} = Channel ) -> - do_authenticate(ClientInfo#{password => Password}, Channel); + %% Auth with CONNECT packet for MQTT v3 + Credential = maybe_add_cert(ClientInfo#{password => Password}, Channel), + do_authenticate(Credential, Channel); authenticate( ?AUTH_PACKET(_, #{'Authentication-Method' := AuthMethod} = Properties), #channel{ @@ -1756,6 +1769,7 @@ authenticate( auth_cache = AuthCache } = Channel ) -> + %% Enhanced auth case emqx_mqtt_props:get('Authentication-Method', ConnProps, undefined) of AuthMethod -> AuthData = emqx_mqtt_props:get('Authentication-Data', Properties, undefined), diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index a2050fbf0..8d4d245da 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -55,6 +55,7 @@ ?VAR_PEERHOST, ?VAR_CERT_SUBJECT, ?VAR_CERT_CN_NAME, + ?VAR_CERT_PEM, ?VAR_NS_CLIENT_ATTRS ]). @@ -357,6 +358,8 @@ render_var(_, undefined) -> % Any allowed but undefined binding will be replaced with empty string, even when % rendering SQL values. <<>>; +render_var(?VAR_CERT_PEM, Value) -> + base64:encode(Value); render_var(?VAR_PEERHOST, Value) -> inet:ntoa(Value); render_var(_Name, Value) -> diff --git a/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl index 54e1f4b11..864aa6e0e 100644 --- a/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authn_http_SUITE.erl @@ -37,6 +37,7 @@ protocol => mqtt, cert_subject => <<"cert_subject_data">>, cert_common_name => <<"cert_common_name_data">>, + cert_pem => <<"fake_raw_cert_to_be_base64_encoded">>, client_attrs => #{<<"group">> => <<"g1">>} }). @@ -222,7 +223,8 @@ t_no_value_for_placeholder(_Config) -> {ok, RawBody, Req1} = cowboy_req:read_body(Req0), #{ <<"cert_subject">> := <<"">>, - <<"cert_common_name">> := <<"">> + <<"cert_common_name">> := <<"">>, + <<"cert_pem">> := <<"">> } = emqx_utils_json:decode(RawBody, [return_maps]), Req = cowboy_req:reply( 200, @@ -238,7 +240,8 @@ t_no_value_for_placeholder(_Config) -> <<"headers">> => #{<<"content-type">> => <<"application/json">>}, <<"body">> => #{ <<"cert_subject">> => ?PH_CERT_SUBJECT, - <<"cert_common_name">> => ?PH_CERT_CN_NAME + <<"cert_common_name">> => ?PH_CERT_CN_NAME, + <<"cert_pem">> => ?PH_CERT_PEM } }, @@ -251,7 +254,7 @@ t_no_value_for_placeholder(_Config) -> ok = emqx_authn_http_test_server:set_handler(Handler), - Credentials = maps:without([cert_subject, cert_common_name], ?CREDENTIALS), + Credentials = maps:without([cert_subject, cert_common_name, cert_pem], ?CREDENTIALS), ?assertMatch({ok, _}, emqx_access_control:authenticate(Credentials)), @@ -656,8 +659,10 @@ samples() -> <<"peerhost">> := <<"127.0.0.1">>, <<"cert_subject">> := <<"cert_subject_data">>, <<"cert_common_name">> := <<"cert_common_name_data">>, + <<"cert_pem">> := CertPem, <<"the_group">> := <<"g1">> } = emqx_utils_json:decode(RawBody, [return_maps]), + <<"fake_raw_cert_to_be_base64_encoded">> = base64:decode(CertPem), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, @@ -676,6 +681,7 @@ samples() -> <<"peerhost">> => ?PH_PEERHOST, <<"cert_subject">> => ?PH_CERT_SUBJECT, <<"cert_common_name">> => ?PH_CERT_CN_NAME, + <<"cert_pem">> => ?PH_CERT_PEM, <<"the_group">> => <<"${client_attrs.group}">> } }, From cbac4019b3e9ec4db4db505e6bd8386285f22537 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Jun 2024 18:09:50 +0800 Subject: [PATCH 30/39] fix: rm havn't used counter idx --- apps/emqx/src/emqx_metrics.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 06d0046ec..ee175b986 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -684,11 +684,11 @@ reserved_idx('messages.dropped') -> 109; reserved_idx('messages.dropped.await_pubrel_timeout') -> 110; reserved_idx('messages.dropped.no_subscribers') -> 111; reserved_idx('messages.forward') -> 112; -%%reserved_idx('messages.retained') -> 113; %% keep the index, new metrics can use this +%% reserved_idx('messages.retained') -> 113; %% keep the index, new metrics can use this reserved_idx('messages.delayed') -> 114; reserved_idx('messages.delivered') -> 115; reserved_idx('messages.acked') -> 116; -reserved_idx('delivery.expired') -> 117; +%% reserved_idx('delivery.expired') -> 117; %% have never used reserved_idx('delivery.dropped') -> 118; reserved_idx('delivery.dropped.no_local') -> 119; reserved_idx('delivery.dropped.too_large') -> 120; @@ -699,7 +699,7 @@ reserved_idx('client.connect') -> 200; reserved_idx('client.connack') -> 201; reserved_idx('client.connected') -> 202; reserved_idx('client.authenticate') -> 203; -reserved_idx('client.enhanced_authenticate') -> 204; +%% reserved_idx('client.enhanced_authenticate') -> 204; %% have never used reserved_idx('client.auth.anonymous') -> 205; reserved_idx('client.authorize') -> 206; reserved_idx('client.subscribe') -> 207; From 9a78a6f6404e80b2e7a883c55c54e09d27ac0583 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Jun 2024 18:20:00 +0800 Subject: [PATCH 31/39] refactor: mv metrics macros in hrl file --- apps/emqx/include/emqx_metrics.hrl | 224 +++++++++++++++++++++++++++++ apps/emqx/src/emqx_metrics.erl | 203 +------------------------- 2 files changed, 225 insertions(+), 202 deletions(-) create mode 100644 apps/emqx/include/emqx_metrics.hrl diff --git a/apps/emqx/include/emqx_metrics.hrl b/apps/emqx/include/emqx_metrics.hrl new file mode 100644 index 000000000..8856eec35 --- /dev/null +++ b/apps/emqx/include/emqx_metrics.hrl @@ -0,0 +1,224 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_METRICS_HRL). +-define(EMQX_METRICS_HRL, true). + +%% Bytes sent and received +-define(BYTES_METRICS, [ + %% Total bytes received + {counter, 'bytes.received'}, + %% Total bytes sent + {counter, 'bytes.sent'} +]). + +%% Packets sent and received +-define(PACKET_METRICS, [ + %% All Packets received + {counter, 'packets.received'}, + %% All Packets sent + {counter, 'packets.sent'}, + %% CONNECT Packets received + {counter, 'packets.connect.received'}, + %% CONNACK Packets sent + {counter, 'packets.connack.sent'}, + %% CONNACK error sent + {counter, 'packets.connack.error'}, + %% CONNACK auth_error sent + {counter, 'packets.connack.auth_error'}, + %% PUBLISH packets received + {counter, 'packets.publish.received'}, + %% PUBLISH packets sent + {counter, 'packets.publish.sent'}, + %% PUBLISH packet_id inuse + {counter, 'packets.publish.inuse'}, + %% PUBLISH failed for error + {counter, 'packets.publish.error'}, + %% PUBLISH failed for auth error + {counter, 'packets.publish.auth_error'}, + %% PUBLISH(QoS2) packets dropped + {counter, 'packets.publish.dropped'}, + %% PUBACK packets received + {counter, 'packets.puback.received'}, + %% PUBACK packets sent + {counter, 'packets.puback.sent'}, + %% PUBACK packet_id inuse + {counter, 'packets.puback.inuse'}, + %% PUBACK packets missed + {counter, 'packets.puback.missed'}, + %% PUBREC packets received + {counter, 'packets.pubrec.received'}, + %% PUBREC packets sent + {counter, 'packets.pubrec.sent'}, + %% PUBREC packet_id inuse + {counter, 'packets.pubrec.inuse'}, + %% PUBREC packets missed + {counter, 'packets.pubrec.missed'}, + %% PUBREL packets received + {counter, 'packets.pubrel.received'}, + %% PUBREL packets sent + {counter, 'packets.pubrel.sent'}, + %% PUBREL packets missed + {counter, 'packets.pubrel.missed'}, + %% PUBCOMP packets received + {counter, 'packets.pubcomp.received'}, + %% PUBCOMP packets sent + {counter, 'packets.pubcomp.sent'}, + %% PUBCOMP packet_id inuse + {counter, 'packets.pubcomp.inuse'}, + %% PUBCOMP packets missed + {counter, 'packets.pubcomp.missed'}, + %% SUBSCRIBE Packets received + {counter, 'packets.subscribe.received'}, + %% SUBSCRIBE error + {counter, 'packets.subscribe.error'}, + %% SUBSCRIBE failed for not auth + {counter, 'packets.subscribe.auth_error'}, + %% SUBACK packets sent + {counter, 'packets.suback.sent'}, + %% UNSUBSCRIBE Packets received + {counter, 'packets.unsubscribe.received'}, + %% UNSUBSCRIBE error + {counter, 'packets.unsubscribe.error'}, + %% UNSUBACK Packets sent + {counter, 'packets.unsuback.sent'}, + %% PINGREQ packets received + {counter, 'packets.pingreq.received'}, + %% PINGRESP Packets sent + {counter, 'packets.pingresp.sent'}, + %% DISCONNECT Packets received + {counter, 'packets.disconnect.received'}, + %% DISCONNECT Packets sent + {counter, 'packets.disconnect.sent'}, + %% Auth Packets received + {counter, 'packets.auth.received'}, + %% Auth Packets sent + {counter, 'packets.auth.sent'} +]). + +%% Messages sent/received and pubsub +-define(MESSAGE_METRICS, [ + %% All Messages received + {counter, 'messages.received'}, + %% All Messages sent + {counter, 'messages.sent'}, + %% QoS0 Messages received + {counter, 'messages.qos0.received'}, + %% QoS0 Messages sent + {counter, 'messages.qos0.sent'}, + %% QoS1 Messages received + {counter, 'messages.qos1.received'}, + %% QoS1 Messages sent + {counter, 'messages.qos1.sent'}, + %% QoS2 Messages received + {counter, 'messages.qos2.received'}, + %% QoS2 Messages sent + {counter, 'messages.qos2.sent'}, + %% PubSub Metrics + + %% Messages Publish + {counter, 'messages.publish'}, + %% Messages dropped due to no subscribers + {counter, 'messages.dropped'}, + %% Messages that failed validations + {counter, 'messages.validation_failed'}, + %% Messages that passed validations + {counter, 'messages.validation_succeeded'}, + %% % Messages that failed transformations + {counter, 'messages.transformation_failed'}, + %% % Messages that passed transformations + {counter, 'messages.transformation_succeeded'}, + %% QoS2 Messages expired + {counter, 'messages.dropped.await_pubrel_timeout'}, + %% Messages dropped + {counter, 'messages.dropped.no_subscribers'}, + %% Messages forward + {counter, 'messages.forward'}, + %% Messages delayed + {counter, 'messages.delayed'}, + %% Messages delivered + {counter, 'messages.delivered'}, + %% Messages acked + {counter, 'messages.acked'}, + %% Messages persistently stored + {counter, 'messages.persisted'} +]). + +%% Delivery metrics +-define(DELIVERY_METRICS, [ + %% All Dropped during delivery + {counter, 'delivery.dropped'}, + %% Dropped due to no_local + {counter, 'delivery.dropped.no_local'}, + %% Dropped due to message too large + {counter, 'delivery.dropped.too_large'}, + %% Dropped qos0 message + {counter, 'delivery.dropped.qos0_msg'}, + %% Dropped due to queue full + {counter, 'delivery.dropped.queue_full'}, + %% Dropped due to expired + {counter, 'delivery.dropped.expired'} +]). + +%% Client Lifecircle metrics +-define(CLIENT_METRICS, [ + {counter, 'client.connect'}, + {counter, 'client.connack'}, + {counter, 'client.connected'}, + {counter, 'client.authenticate'}, + {counter, 'client.auth.anonymous'}, + {counter, 'client.authorize'}, + {counter, 'client.subscribe'}, + {counter, 'client.unsubscribe'}, + {counter, 'client.disconnected'} +]). + +%% Session Lifecircle metrics +-define(SESSION_METRICS, [ + {counter, 'session.created'}, + {counter, 'session.resumed'}, + %% Session taken over by another client (Connect with clean_session|clean_start=false) + {counter, 'session.takenover'}, + %% Session taken over by another client (Connect with clean_session|clean_start=true) + {counter, 'session.discarded'}, + {counter, 'session.terminated'} +]). + +%% Statistic metrics for ACL checking +-define(STASTS_ACL_METRICS, [ + {counter, 'authorization.allow'}, + {counter, 'authorization.deny'}, + {counter, 'authorization.cache_hit'}, + {counter, 'authorization.cache_miss'} +]). + +%% Statistic metrics for auth checking +-define(STASTS_AUTHN_METRICS, [ + {counter, 'authentication.success'}, + {counter, 'authentication.success.anonymous'}, + {counter, 'authentication.failure'} +]). + +%% Overload protection counters +-define(OLP_METRICS, [ + {counter, 'overload_protection.delay.ok'}, + {counter, 'overload_protection.delay.timeout'}, + {counter, 'overload_protection.hibernation'}, + {counter, 'overload_protection.gc'}, + {counter, 'overload_protection.new_conn'} +]). + +-endif. diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index ee175b986..9567eb404 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -22,6 +22,7 @@ -include("logger.hrl"). -include("types.hrl"). -include("emqx_mqtt.hrl"). +-include("emqx_metrics.hrl"). -export([ start_link/0, @@ -86,208 +87,6 @@ -define(TAB, ?MODULE). -define(SERVER, ?MODULE). -%% Bytes sent and received --define(BYTES_METRICS, - % Total bytes received - [ - {counter, 'bytes.received'}, - % Total bytes sent - {counter, 'bytes.sent'} - ] -). - -%% Packets sent and received --define(PACKET_METRICS, - % All Packets received - [ - {counter, 'packets.received'}, - % All Packets sent - {counter, 'packets.sent'}, - % CONNECT Packets received - {counter, 'packets.connect.received'}, - % CONNACK Packets sent - {counter, 'packets.connack.sent'}, - % CONNACK error sent - {counter, 'packets.connack.error'}, - % CONNACK auth_error sent - {counter, 'packets.connack.auth_error'}, - % PUBLISH packets received - {counter, 'packets.publish.received'}, - % PUBLISH packets sent - {counter, 'packets.publish.sent'}, - % PUBLISH packet_id inuse - {counter, 'packets.publish.inuse'}, - % PUBLISH failed for error - {counter, 'packets.publish.error'}, - % PUBLISH failed for auth error - {counter, 'packets.publish.auth_error'}, - % PUBLISH(QoS2) packets dropped - {counter, 'packets.publish.dropped'}, - % PUBACK packets received - {counter, 'packets.puback.received'}, - % PUBACK packets sent - {counter, 'packets.puback.sent'}, - % PUBACK packet_id inuse - {counter, 'packets.puback.inuse'}, - % PUBACK packets missed - {counter, 'packets.puback.missed'}, - % PUBREC packets received - {counter, 'packets.pubrec.received'}, - % PUBREC packets sent - {counter, 'packets.pubrec.sent'}, - % PUBREC packet_id inuse - {counter, 'packets.pubrec.inuse'}, - % PUBREC packets missed - {counter, 'packets.pubrec.missed'}, - % PUBREL packets received - {counter, 'packets.pubrel.received'}, - % PUBREL packets sent - {counter, 'packets.pubrel.sent'}, - % PUBREL packets missed - {counter, 'packets.pubrel.missed'}, - % PUBCOMP packets received - {counter, 'packets.pubcomp.received'}, - % PUBCOMP packets sent - {counter, 'packets.pubcomp.sent'}, - % PUBCOMP packet_id inuse - {counter, 'packets.pubcomp.inuse'}, - % PUBCOMP packets missed - {counter, 'packets.pubcomp.missed'}, - % SUBSCRIBE Packets received - {counter, 'packets.subscribe.received'}, - % SUBSCRIBE error - {counter, 'packets.subscribe.error'}, - % SUBSCRIBE failed for not auth - {counter, 'packets.subscribe.auth_error'}, - % SUBACK packets sent - {counter, 'packets.suback.sent'}, - % UNSUBSCRIBE Packets received - {counter, 'packets.unsubscribe.received'}, - % UNSUBSCRIBE error - {counter, 'packets.unsubscribe.error'}, - % UNSUBACK Packets sent - {counter, 'packets.unsuback.sent'}, - % PINGREQ packets received - {counter, 'packets.pingreq.received'}, - % PINGRESP Packets sent - {counter, 'packets.pingresp.sent'}, - % DISCONNECT Packets received - {counter, 'packets.disconnect.received'}, - % DISCONNECT Packets sent - {counter, 'packets.disconnect.sent'}, - % Auth Packets received - {counter, 'packets.auth.received'}, - % Auth Packets sent - {counter, 'packets.auth.sent'} - ] -). - -%% Messages sent/received and pubsub --define(MESSAGE_METRICS, - % All Messages received - [ - {counter, 'messages.received'}, - % All Messages sent - {counter, 'messages.sent'}, - % QoS0 Messages received - {counter, 'messages.qos0.received'}, - % QoS0 Messages sent - {counter, 'messages.qos0.sent'}, - % QoS1 Messages received - {counter, 'messages.qos1.received'}, - % QoS1 Messages sent - {counter, 'messages.qos1.sent'}, - % QoS2 Messages received - {counter, 'messages.qos2.received'}, - % QoS2 Messages sent - {counter, 'messages.qos2.sent'}, - %% PubSub Metrics - - % Messages Publish - {counter, 'messages.publish'}, - % Messages dropped due to no subscribers - {counter, 'messages.dropped'}, - %% % Messages that failed validations - {counter, 'messages.validation_failed'}, - %% % Messages that passed validations - {counter, 'messages.validation_succeeded'}, - %% % Messages that failed transformations - {counter, 'messages.transformation_failed'}, - %% % Messages that passed transformations - {counter, 'messages.transformation_succeeded'}, - % QoS2 Messages expired - {counter, 'messages.dropped.await_pubrel_timeout'}, - % Messages dropped - {counter, 'messages.dropped.no_subscribers'}, - % Messages forward - {counter, 'messages.forward'}, - % Messages delayed - {counter, 'messages.delayed'}, - % Messages delivered - {counter, 'messages.delivered'}, - % Messages acked - {counter, 'messages.acked'}, - % Messages persistently stored - {counter, 'messages.persisted'} - ] -). - -%% Delivery metrics --define(DELIVERY_METRICS, [ - {counter, 'delivery.dropped'}, - {counter, 'delivery.dropped.no_local'}, - {counter, 'delivery.dropped.too_large'}, - {counter, 'delivery.dropped.qos0_msg'}, - {counter, 'delivery.dropped.queue_full'}, - {counter, 'delivery.dropped.expired'} -]). - -%% Client Lifecircle metrics --define(CLIENT_METRICS, [ - {counter, 'client.connect'}, - {counter, 'client.connack'}, - {counter, 'client.connected'}, - {counter, 'client.authenticate'}, - {counter, 'client.auth.anonymous'}, - {counter, 'client.authorize'}, - {counter, 'client.subscribe'}, - {counter, 'client.unsubscribe'}, - {counter, 'client.disconnected'} -]). - -%% Session Lifecircle metrics --define(SESSION_METRICS, [ - {counter, 'session.created'}, - {counter, 'session.resumed'}, - {counter, 'session.takenover'}, - {counter, 'session.discarded'}, - {counter, 'session.terminated'} -]). - -%% Statistic metrics for ACL checking --define(STASTS_ACL_METRICS, [ - {counter, 'authorization.allow'}, - {counter, 'authorization.deny'}, - {counter, 'authorization.cache_hit'}, - {counter, 'authorization.cache_miss'} -]). - -%% Statistic metrics for auth checking --define(STASTS_AUTHN_METRICS, [ - {counter, 'authentication.success'}, - {counter, 'authentication.success.anonymous'}, - {counter, 'authentication.failure'} -]). - -%% Overload protection counters --define(OLP_METRICS, [ - {counter, 'overload_protection.delay.ok'}, - {counter, 'overload_protection.delay.timeout'}, - {counter, 'overload_protection.hibernation'}, - {counter, 'overload_protection.gc'}, - {counter, 'overload_protection.new_conn'} -]). - olp_metrics() -> lists:map(fun({_, Metric}) -> Metric end, ?OLP_METRICS). From 263d2dbae2d501b75c75877a7711458489772879 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 11 Jun 2024 18:45:36 +0800 Subject: [PATCH 32/39] fix: inc metric `'client.auth.anonymous'` when client anonymous --- apps/emqx/src/emqx_access_control.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index e3c730cd5..2d75feea3 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -238,6 +238,7 @@ inc_authn_metrics(error) -> inc_authn_metrics(ok) -> emqx_metrics:inc('authentication.success'); inc_authn_metrics(anonymous) -> + emqx_metrics:inc('client.auth.anonymous'), emqx_metrics:inc('authentication.success.anonymous'), emqx_metrics:inc('authentication.success'). From 756797b25818833a34af723e2d04a409a523d377 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 12 Jun 2024 18:10:45 +0800 Subject: [PATCH 33/39] refactor: gen metric schema with desc from macros --- .../src/emqx_mgmt_api_metrics.erl | 551 +++++++----------- 1 file changed, 204 insertions(+), 347 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index f2e302569..f9834c81d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -112,354 +112,211 @@ fields(node_metrics) -> [{node, mk(binary(), #{desc => <<"Node name">>})}] ++ properties(). properties() -> - [ - m( - 'actions.failure', - <<"Number of failure executions of the rule engine action">> - ), - m( - 'actions.success', - <<"Number of successful executions of the rule engine action">> - ), - m( - 'bytes.received', - <<"Number of bytes received ">> - ), - m( - 'bytes.sent', - <<"Number of bytes sent on this connection">> - ), - m( - 'client.auth.anonymous', - <<"Number of clients who log in anonymously">> - ), - m( - 'client.authenticate', - <<"Number of client authentications">> - ), - m( - 'client.check_authz', - <<"Number of Authorization rule checks">> - ), - m( - 'client.connack', - <<"Number of CONNACK packet sent">> - ), - m( - 'client.connect', - <<"Number of client connections">> - ), - m( - 'client.connected', - <<"Number of successful client connections">> - ), - m( - 'client.disconnected', - <<"Number of client disconnects">> - ), - m( - 'client.subscribe', - <<"Number of client subscriptions">> - ), - m( - 'client.unsubscribe', - <<"Number of client unsubscriptions">> - ), - m( - 'delivery.dropped', - <<"Total number of discarded messages when sending">> - ), - m( - 'delivery.dropped.expired', - <<"Number of messages dropped due to message expiration on sending">> - ), - m( - 'delivery.dropped.no_local', - << - "Number of messages that were dropped due to the No Local subscription " - "option when sending" - >> - ), - m( - 'delivery.dropped.qos0_msg', - << - "Number of messages with QoS 0 that were dropped because the message " - "queue was full when sending" - >> - ), - m( - 'delivery.dropped.queue_full', - << - "Number of messages with a non-zero QoS that were dropped because the " - "message queue was full when sending" - >> - ), - m( - 'delivery.dropped.too_large', - << - "The number of messages that were dropped because the length exceeded " - "the limit when sending" - >> - ), - m( - 'messages.acked', - <<"Number of received PUBACK and PUBREC packet">> - ), - m( - 'messages.delayed', - <<"Number of delay-published messages">> - ), - m( - 'messages.delivered', - <<"Number of messages forwarded to the subscription process internally">> - ), - m( - 'messages.dropped', - <<"Total number of messages dropped before forwarding to the subscription process">> - ), - m( - 'messages.dropped.await_pubrel_timeout', - <<"Number of messages dropped due to waiting PUBREL timeout">> - ), - m( - 'messages.dropped.no_subscribers', - <<"Number of messages dropped due to no subscribers">> - ), - m( - 'messages.forward', - <<"Number of messages forwarded to other nodes">> - ), - m( - 'messages.publish', - <<"Number of messages published in addition to system messages">> - ), - m( - 'messages.qos0.received', - <<"Number of QoS 0 messages received from clients">> - ), - m( - 'messages.qos0.sent', - <<"Number of QoS 0 messages sent to clients">> - ), - m( - 'messages.qos1.received', - <<"Number of QoS 1 messages received from clients">> - ), - m( - 'messages.qos1.sent', - <<"Number of QoS 1 messages sent to clients">> - ), - m( - 'messages.qos2.received', - <<"Number of QoS 2 messages received from clients">> - ), - m( - 'messages.qos2.sent', - <<"Number of QoS 2 messages sent to clients">> - ), - m( - 'messages.received', - << - "Number of messages received from the client, equal to the sum of " - "messages.qos0.received\fmessages.qos1.received and messages.qos2.received" - >> - ), - %% m( - %% 'messages.retained', - %% <<"Number of retained messages">> - %% ), - m( - 'messages.sent', - << - "Number of messages sent to the client, equal to the sum of " - "messages.qos0.sent\fmessages.qos1.sent and messages.qos2.sent" - >> - ), - m( - 'packets.auth.received', - <<"Number of received AUTH packet">> - ), - m( - 'packets.auth.sent', - <<"Number of sent AUTH packet">> - ), - m( - 'packets.connack.auth_error', - <<"Number of received CONNECT packet with failed authentication">> - ), - m( - 'packets.connack.error', - <<"Number of received CONNECT packet with unsuccessful connections">> - ), - m( - 'packets.connack.sent', - <<"Number of sent CONNACK packet">> - ), - m( - 'packets.connect.received', - <<"Number of received CONNECT packet">> - ), - m( - 'packets.disconnect.received', - <<"Number of received DISCONNECT packet">> - ), - m( - 'packets.disconnect.sent', - <<"Number of sent DISCONNECT packet">> - ), - m( - 'packets.pingreq.received', - <<"Number of received PINGREQ packet">> - ), - m( - 'packets.pingresp.sent', - <<"Number of sent PUBRESP packet">> - ), - m( - 'packets.puback.inuse', - <<"Number of received PUBACK packet with occupied identifiers">> - ), - m( - 'packets.puback.missed', - <<"Number of received packet with identifiers.">> - ), - m( - 'packets.puback.received', - <<"Number of received PUBACK packet">> - ), - m( - 'packets.puback.sent', - <<"Number of sent PUBACK packet">> - ), - m( - 'packets.pubcomp.inuse', - <<"Number of received PUBCOMP packet with occupied identifiers">> - ), - m( - 'packets.pubcomp.missed', - <<"Number of missed PUBCOMP packet">> - ), - m( - 'packets.pubcomp.received', - <<"Number of received PUBCOMP packet">> - ), - m( - 'packets.pubcomp.sent', - <<"Number of sent PUBCOMP packet">> - ), - m( - 'packets.publish.auth_error', - <<"Number of received PUBLISH packets with failed the Authorization check">> - ), - m( - 'packets.publish.dropped', - <<"Number of messages discarded due to the receiving limit">> - ), - m( - 'packets.publish.error', - <<"Number of received PUBLISH packet that cannot be published">> - ), - m( - 'packets.publish.inuse', - <<"Number of received PUBLISH packet with occupied identifiers">> - ), - m( - 'packets.publish.received', - <<"Number of received PUBLISH packet">> - ), - m( - 'packets.publish.sent', - <<"Number of sent PUBLISH packet">> - ), - m( - 'packets.pubrec.inuse', - <<"Number of received PUBREC packet with occupied identifiers">> - ), - m( - 'packets.pubrec.missed', - <<"Number of received PUBREC packet with unknown identifiers">> - ), - m( - 'packets.pubrec.received', - <<"Number of received PUBREC packet">> - ), - m( - 'packets.pubrec.sent', - <<"Number of sent PUBREC packet">> - ), - m( - 'packets.pubrel.missed', - <<"Number of received PUBREC packet with unknown identifiers">> - ), - m( - 'packets.pubrel.received', - <<"Number of received PUBREL packet">> - ), - m( - 'packets.pubrel.sent', - <<"Number of sent PUBREL packet">> - ), - m( - 'packets.received', - <<"Number of received packet">> - ), - m( - 'packets.sent', - <<"Number of sent packet">> - ), - m( - 'packets.suback.sent', - <<"Number of sent SUBACK packet">> - ), - m( - 'packets.subscribe.auth_error', - <<"Number of received SUBACK packet with failed Authorization check">> - ), - m( - 'packets.subscribe.error', - <<"Number of received SUBSCRIBE packet with failed subscriptions">> - ), - m( - 'packets.subscribe.received', - <<"Number of received SUBSCRIBE packet">> - ), - m( - 'packets.unsuback.sent', - <<"Number of sent UNSUBACK packet">> - ), - m( - 'packets.unsubscribe.error', - <<"Number of received UNSUBSCRIBE packet with failed unsubscriptions">> - ), - m( - 'packets.unsubscribe.received', - <<"Number of received UNSUBSCRIBE packet">> - ), - m( - 'rules.matched', - <<"Number of rule matched">> - ), - m( - 'session.created', - <<"Number of sessions created">> - ), - m( - 'session.discarded', - <<"Number of sessions dropped because Clean Session or Clean Start is true">> - ), - m( - 'session.resumed', - <<"Number of sessions resumed because Clean Session or Clean Start is false">> - ), - m( - 'session.takenover', - <<"Number of sessions takenover because Clean Session or Clean Start is false">> - ), - m( - 'session.terminated', - <<"Number of terminated sessions">> + Metrics = lists:append([ + ?BYTES_METRICS, + ?PACKET_METRICS, + ?MESSAGE_METRICS, + ?DELIVERY_METRICS, + ?CLIENT_METRICS, + ?SESSION_METRICS, + ?STASTS_ACL_METRICS, + ?STASTS_AUTHN_METRICS, + ?OLP_METRICS + ]), + lists:reverse( + lists:foldl( + fun({_Type, MetricName}, Acc) -> + [m(MetricName) | Acc] + end, + [], + Metrics ) - ]. + ). + +m('actions.failure' = K) -> + m(K, <<"Number of failure executions of the rule engine action">>); +m('actions.success' = K) -> + m(K, <<"Number of successful executions of the rule engine action">>); +m('bytes.received' = K) -> + m(K, <<"Number of bytes received ">>); +m('bytes.sent' = K) -> + m(K, <<"Number of bytes sent on this connection">>); +m('client.auth.anonymous' = K) -> + m(K, <<"Number of clients who log in anonymously">>); +m('client.authenticate' = K) -> + m(K, <<"Number of client authentications">>); +m('client.check_authz' = K) -> + m(K, <<"Number of Authorization rule checks">>); +m('client.connack' = K) -> + m(K, <<"Number of CONNACK packet sent">>); +m('client.connect' = K) -> + m(K, <<"Number of client connections">>); +m('client.connected' = K) -> + m(K, <<"Number of successful client connections">>); +m('client.disconnected' = K) -> + m(K, <<"Number of client disconnects">>); +m('client.subscribe' = K) -> + m(K, <<"Number of client subscriptions">>); +m('client.unsubscribe' = K) -> + m(K, <<"Number of client unsubscriptions">>); +m('delivery.dropped' = K) -> + m(K, <<"Total number of discarded messages when sending">>); +m('delivery.dropped.expired' = K) -> + m(K, <<"Number of messages dropped due to message expiration on sending">>); +m('delivery.dropped.no_local' = K) -> + m(K, << + "Number of messages that were dropped due to the No Local subscription " + "option when sending" + >>); +m('delivery.dropped.qos0_msg' = K) -> + m(K, << + "Number of messages with QoS 0 that were dropped because the message " + "queue was full when sending" + >>); +m('delivery.dropped.queue_full' = K) -> + m(K, << + "Number of messages with a non-zero QoS that were dropped because the " + "message queue was full when sending" + >>); +m('delivery.dropped.too_large' = K) -> + m(K, << + "The number of messages that were dropped because the length exceeded " + "the limit when sending" + >>); +m('messages.acked' = K) -> + m(K, <<"Number of received PUBACK and PUBREC packet">>); +m('messages.delayed' = K) -> + m(K, <<"Number of delay-published messages">>); +m('messages.delivered' = K) -> + m(K, <<"Number of messages forwarded to the subscription process internally">>); +m('messages.dropped' = K) -> + m(K, <<"Total number of messages dropped before forwarding to the subscription process">>); +m('messages.dropped.await_pubrel_timeout' = K) -> + m(K, <<"Number of messages dropped due to waiting PUBREL timeout">>); +m('messages.dropped.no_subscribers' = K) -> + m(K, <<"Number of messages dropped due to no subscribers">>); +m('messages.forward' = K) -> + m(K, <<"Number of messages forwarded to other nodes">>); +m('messages.publish' = K) -> + m(K, <<"Number of messages published in addition to system messages">>); +m('messages.qos0.received' = K) -> + m(K, <<"Number of QoS 0 messages received from clients">>); +m('messages.qos0.sent' = K) -> + m(K, <<"Number of QoS 0 messages sent to clients">>); +m('messages.qos1.received' = K) -> + m(K, <<"Number of QoS 1 messages received from clients">>); +m('messages.qos1.sent' = K) -> + m(K, <<"Number of QoS 1 messages sent to clients">>); +m('messages.qos2.received' = K) -> + m(K, <<"Number of QoS 2 messages received from clients">>); +m('messages.qos2.sent' = K) -> + m(K, <<"Number of QoS 2 messages sent to clients">>); +m('messages.received' = K) -> + m(K, << + "Number of messages received from the client, equal to the sum of " + "messages.qos0.received\fmessages.qos1.received and messages.qos2.received" + >>); +%% m( +%% 'messages.retained', +%% <<"Number of retained messages">> +%% ), +m('messages.sent' = K) -> + m(K, << + "Number of messages sent to the client, equal to the sum of " + "messages.qos0.sent\fmessages.qos1.sent and messages.qos2.sent" + >>); +m('packets.auth.received' = K) -> + m(K, <<"Number of received AUTH packet">>); +m('packets.auth.sent' = K) -> + m(K, <<"Number of sent AUTH packet">>); +m('packets.connack.auth_error' = K) -> + m(K, <<"Number of received CONNECT packet with failed authentication">>); +m('packets.connack.error' = K) -> + m(K, <<"Number of received CONNECT packet with unsuccessful connections">>); +m('packets.connack.sent' = K) -> + m(K, <<"Number of sent CONNACK packet">>); +m('packets.connect.received' = K) -> + m(K, <<"Number of received CONNECT packet">>); +m('packets.disconnect.received' = K) -> + m(K, <<"Number of received DISCONNECT packet">>); +m('packets.disconnect.sent' = K) -> + m(K, <<"Number of sent DISCONNECT packet">>); +m('packets.pingreq.received' = K) -> + m(K, <<"Number of received PINGREQ packet">>); +m('packets.pingresp.sent' = K) -> + m(K, <<"Number of sent PUBRESP packet">>); +m('packets.puback.inuse' = K) -> + m(K, <<"Number of received PUBACK packet with occupied identifiers">>); +m('packets.puback.missed' = K) -> + m(K, <<"Number of received packet with identifiers.">>); +m('packets.puback.received' = K) -> + m(K, <<"Number of received PUBACK packet">>); +m('packets.puback.sent' = K) -> + m(K, <<"Number of sent PUBACK packet">>); +m('packets.pubcomp.inuse' = K) -> + m(K, <<"Number of received PUBCOMP packet with occupied identifiers">>); +m('packets.pubcomp.missed' = K) -> + m(K, <<"Number of missed PUBCOMP packet">>); +m('packets.pubcomp.received' = K) -> + m(K, <<"Number of received PUBCOMP packet">>); +m('packets.pubcomp.sent' = K) -> + m(K, <<"Number of sent PUBCOMP packet">>); +m('packets.publish.auth_error' = K) -> + m(K, <<"Number of received PUBLISH packets with failed the Authorization check">>); +m('packets.publish.dropped' = K) -> + m(K, <<"Number of messages discarded due to the receiving limit">>); +m('packets.publish.error' = K) -> + m(K, <<"Number of received PUBLISH packet that cannot be published">>); +m('packets.publish.inuse' = K) -> + m(K, <<"Number of received PUBLISH packet with occupied identifiers">>); +m('packets.publish.received' = K) -> + m(K, <<"Number of received PUBLISH packet">>); +m('packets.publish.sent' = K) -> + m(K, <<"Number of sent PUBLISH packet">>); +m('packets.pubrec.inuse' = K) -> + m(K, <<"Number of received PUBREC packet with occupied identifiers">>); +m('packets.pubrec.missed' = K) -> + m(K, <<"Number of received PUBREC packet with unknown identifiers">>); +m('packets.pubrec.received' = K) -> + m(K, <<"Number of received PUBREC packet">>); +m('packets.pubrec.sent' = K) -> + m(K, <<"Number of sent PUBREC packet">>); +m('packets.pubrel.missed' = K) -> + m(K, <<"Number of received PUBREC packet with unknown identifiers">>); +m('packets.pubrel.received' = K) -> + m(K, <<"Number of received PUBREL packet">>); +m('packets.pubrel.sent' = K) -> + m(K, <<"Number of sent PUBREL packet">>); +m('packets.received' = K) -> + m(K, <<"Number of received packet">>); +m('packets.sent' = K) -> + m(K, <<"Number of sent packet">>); +m('packets.suback.sent' = K) -> + m(K, <<"Number of sent SUBACK packet">>); +m('packets.subscribe.auth_error' = K) -> + m(K, <<"Number of received SUBACK packet with failed Authorization check">>); +m('packets.subscribe.error' = K) -> + m(K, <<"Number of received SUBSCRIBE packet with failed subscriptions">>); +m('packets.subscribe.received' = K) -> + m(K, <<"Number of received SUBSCRIBE packet">>); +m('packets.unsuback.sent' = K) -> + m(K, <<"Number of sent UNSUBACK packet">>); +m('packets.unsubscribe.error' = K) -> + m(K, <<"Number of received UNSUBSCRIBE packet with failed unsubscriptions">>); +m('packets.unsubscribe.received' = K) -> + m(K, <<"Number of received UNSUBSCRIBE packet">>); +m('rules.matched' = K) -> + m(K, <<"Number of rule matched">>); +m('session.created' = K) -> + m(K, <<"Number of sessions created">>); +m('session.discarded' = K) -> + m(K, <<"Number of sessions dropped because Clean Session or Clean Start is true">>); +m('session.resumed' = K) -> + m(K, <<"Number of sessions resumed because Clean Session or Clean Start is false">>); +m('session.takenover' = K) -> + m(K, <<"Number of sessions takenover because Clean Session or Clean Start is false">>); +m('session.terminated' = K) -> + m(K, <<"Number of terminated sessions">>). m(K, Desc) -> {K, mk(non_neg_integer(), #{desc => Desc})}. From cde4cb1358001c91f4780bc2ea41335fd0e98df3 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 12 Jun 2024 18:55:10 +0800 Subject: [PATCH 34/39] fix: metrics macro with description --- apps/emqx/include/emqx_metrics.hrl | 246 ++++++++++-------- apps/emqx/src/emqx_metrics.erl | 4 +- .../src/emqx_mgmt_api_metrics.erl | 190 +------------- 3 files changed, 145 insertions(+), 295 deletions(-) diff --git a/apps/emqx/include/emqx_metrics.hrl b/apps/emqx/include/emqx_metrics.hrl index 8856eec35..b8d6da018 100644 --- a/apps/emqx/include/emqx_metrics.hrl +++ b/apps/emqx/include/emqx_metrics.hrl @@ -19,206 +19,240 @@ %% Bytes sent and received -define(BYTES_METRICS, [ - %% Total bytes received - {counter, 'bytes.received'}, - %% Total bytes sent - {counter, 'bytes.sent'} + {counter, 'bytes.received', <<"Number of bytes received ">>}, + {counter, 'bytes.sent', <<"Number of bytes sent on this connection">>} ]). %% Packets sent and received -define(PACKET_METRICS, [ - %% All Packets received - {counter, 'packets.received'}, - %% All Packets sent - {counter, 'packets.sent'}, - %% CONNECT Packets received - {counter, 'packets.connect.received'}, - %% CONNACK Packets sent - {counter, 'packets.connack.sent'}, - %% CONNACK error sent - {counter, 'packets.connack.error'}, - %% CONNACK auth_error sent - {counter, 'packets.connack.auth_error'}, - %% PUBLISH packets received - {counter, 'packets.publish.received'}, + {counter, 'packets.received', <<"Number of received packet">>}, + {counter, 'packets.sent', <<"Number of sent packet">>}, + {counter, 'packets.connect.received', <<"Number of received CONNECT packet">>}, + {counter, 'packets.connack.sent', <<"Number of sent CONNACK packet">>}, + {counter, 'packets.connack.error', + <<"Number of received CONNECT packet with unsuccessful connections">>}, + {counter, 'packets.connack.auth_error', + <<"Number of received CONNECT packet with failed authentication">>}, + {counter, 'packets.publish.received', <<"Number of received PUBLISH packet">>}, %% PUBLISH packets sent - {counter, 'packets.publish.sent'}, + {counter, 'packets.publish.sent', <<"Number of sent PUBLISH packet">>}, %% PUBLISH packet_id inuse - {counter, 'packets.publish.inuse'}, + {counter, 'packets.publish.inuse', + <<"Number of received PUBLISH packet with occupied identifiers">>}, %% PUBLISH failed for error - {counter, 'packets.publish.error'}, + {counter, 'packets.publish.error', + <<"Number of received PUBLISH packet that cannot be published">>}, %% PUBLISH failed for auth error - {counter, 'packets.publish.auth_error'}, + {counter, 'packets.publish.auth_error', + <<"Number of received PUBLISH packets with failed the Authorization check">>}, %% PUBLISH(QoS2) packets dropped - {counter, 'packets.publish.dropped'}, + {counter, 'packets.publish.dropped', + <<"Number of messages discarded due to the receiving limit">>}, %% PUBACK packets received - {counter, 'packets.puback.received'}, + {counter, 'packets.puback.received', <<"Number of received PUBACK packet">>}, %% PUBACK packets sent - {counter, 'packets.puback.sent'}, + {counter, 'packets.puback.sent', <<"Number of sent PUBACK packet">>}, %% PUBACK packet_id inuse - {counter, 'packets.puback.inuse'}, + {counter, 'packets.puback.inuse', + <<"Number of received PUBACK packet with occupied identifiers">>}, %% PUBACK packets missed - {counter, 'packets.puback.missed'}, + {counter, 'packets.puback.missed', <<"Number of received packet with identifiers.">>}, %% PUBREC packets received - {counter, 'packets.pubrec.received'}, + {counter, 'packets.pubrec.received', <<"Number of received PUBREC packet">>}, %% PUBREC packets sent - {counter, 'packets.pubrec.sent'}, + {counter, 'packets.pubrec.sent', <<"Number of sent PUBREC packet">>}, %% PUBREC packet_id inuse - {counter, 'packets.pubrec.inuse'}, + {counter, 'packets.pubrec.inuse', + <<"Number of received PUBREC packet with occupied identifiers">>}, %% PUBREC packets missed - {counter, 'packets.pubrec.missed'}, + {counter, 'packets.pubrec.missed', + <<"Number of received PUBREC packet with unknown identifiers">>}, %% PUBREL packets received - {counter, 'packets.pubrel.received'}, + {counter, 'packets.pubrel.received', <<"Number of received PUBREL packet">>}, %% PUBREL packets sent - {counter, 'packets.pubrel.sent'}, + {counter, 'packets.pubrel.sent', <<"Number of sent PUBREL packet">>}, %% PUBREL packets missed - {counter, 'packets.pubrel.missed'}, + {counter, 'packets.pubrel.missed', + <<"Number of received PUBREC packet with unknown identifiers">>}, %% PUBCOMP packets received - {counter, 'packets.pubcomp.received'}, + {counter, 'packets.pubcomp.received', <<"Number of received PUBCOMP packet">>}, %% PUBCOMP packets sent - {counter, 'packets.pubcomp.sent'}, + {counter, 'packets.pubcomp.sent', <<"Number of sent PUBCOMP packet">>}, %% PUBCOMP packet_id inuse - {counter, 'packets.pubcomp.inuse'}, + {counter, 'packets.pubcomp.inuse', + <<"Number of received PUBCOMP packet with occupied identifiers">>}, %% PUBCOMP packets missed - {counter, 'packets.pubcomp.missed'}, + {counter, 'packets.pubcomp.missed', <<"Number of missed PUBCOMP packet">>}, %% SUBSCRIBE Packets received - {counter, 'packets.subscribe.received'}, + {counter, 'packets.subscribe.received', <<"Number of received SUBSCRIBE packet">>}, %% SUBSCRIBE error - {counter, 'packets.subscribe.error'}, + {counter, 'packets.subscribe.error', + <<"Number of received SUBSCRIBE packet with failed subscriptions">>}, %% SUBSCRIBE failed for not auth - {counter, 'packets.subscribe.auth_error'}, + {counter, 'packets.subscribe.auth_error', + <<"Number of received SUBACK packet with failed Authorization check">>}, %% SUBACK packets sent - {counter, 'packets.suback.sent'}, + {counter, 'packets.suback.sent', <<"Number of sent SUBACK packet">>}, %% UNSUBSCRIBE Packets received - {counter, 'packets.unsubscribe.received'}, + {counter, 'packets.unsubscribe.received', <<"Number of received UNSUBSCRIBE packet">>}, %% UNSUBSCRIBE error - {counter, 'packets.unsubscribe.error'}, + {counter, 'packets.unsubscribe.error', + <<"Number of received UNSUBSCRIBE packet with failed unsubscriptions">>}, %% UNSUBACK Packets sent - {counter, 'packets.unsuback.sent'}, + {counter, 'packets.unsuback.sent', <<"Number of sent UNSUBACK packet">>}, %% PINGREQ packets received - {counter, 'packets.pingreq.received'}, + {counter, 'packets.pingreq.received', <<"Number of received PINGREQ packet">>}, %% PINGRESP Packets sent - {counter, 'packets.pingresp.sent'}, + {counter, 'packets.pingresp.sent', <<"Number of sent PUBRESP packet">>}, %% DISCONNECT Packets received - {counter, 'packets.disconnect.received'}, + {counter, 'packets.disconnect.received', <<"Number of received DISCONNECT packet">>}, %% DISCONNECT Packets sent - {counter, 'packets.disconnect.sent'}, + {counter, 'packets.disconnect.sent', <<"Number of sent DISCONNECT packet">>}, %% Auth Packets received - {counter, 'packets.auth.received'}, + {counter, 'packets.auth.received', <<"Number of received AUTH packet">>}, %% Auth Packets sent - {counter, 'packets.auth.sent'} + {counter, 'packets.auth.sent', <<"Number of sent AUTH packet">>} ]). %% Messages sent/received and pubsub -define(MESSAGE_METRICS, [ %% All Messages received - {counter, 'messages.received'}, + {counter, 'messages.received', << + "Number of messages received from the client, equal to the sum of " + "messages.qos0.received\fmessages.qos1.received and messages.qos2.received" + >>}, %% All Messages sent - {counter, 'messages.sent'}, + {counter, 'messages.sent', << + "Number of messages sent to the client, equal to the sum of " + "messages.qos0.sent\fmessages.qos1.sent and messages.qos2.sent" + >>}, %% QoS0 Messages received - {counter, 'messages.qos0.received'}, + {counter, 'messages.qos0.received', <<"Number of QoS 0 messages received from clients">>}, %% QoS0 Messages sent - {counter, 'messages.qos0.sent'}, + {counter, 'messages.qos0.sent', <<"Number of QoS 0 messages sent to clients">>}, %% QoS1 Messages received - {counter, 'messages.qos1.received'}, + {counter, 'messages.qos1.received', <<"Number of QoS 1 messages received from clients">>}, %% QoS1 Messages sent - {counter, 'messages.qos1.sent'}, + {counter, 'messages.qos1.sent', <<"Number of QoS 1 messages sent to clients">>}, %% QoS2 Messages received - {counter, 'messages.qos2.received'}, + {counter, 'messages.qos2.received', <<"Number of QoS 2 messages received from clients">>}, %% QoS2 Messages sent - {counter, 'messages.qos2.sent'}, + {counter, 'messages.qos2.sent', <<"Number of QoS 2 messages sent to clients">>}, %% PubSub Metrics %% Messages Publish - {counter, 'messages.publish'}, + {counter, 'messages.publish', + <<"Number of messages published in addition to system messages">>}, %% Messages dropped due to no subscribers - {counter, 'messages.dropped'}, + {counter, 'messages.dropped', + <<"Number of messages dropped before forwarding to the subscription process">>}, %% Messages that failed validations - {counter, 'messages.validation_failed'}, + {counter, 'messages.validation_failed', <<"Number of message validation failed">>}, %% Messages that passed validations - {counter, 'messages.validation_succeeded'}, + {counter, 'messages.validation_succeeded', <<"Number of message validation successful">>}, %% % Messages that failed transformations - {counter, 'messages.transformation_failed'}, + {counter, 'messages.transformation_failed', <<"Number fo message transformation failed">>}, %% % Messages that passed transformations - {counter, 'messages.transformation_succeeded'}, + {counter, 'messages.transformation_succeeded', + <<"Number fo message transformation succeeded">>}, %% QoS2 Messages expired - {counter, 'messages.dropped.await_pubrel_timeout'}, + {counter, 'messages.dropped.await_pubrel_timeout', + <<"Number of messages dropped due to waiting PUBREL timeout">>}, %% Messages dropped - {counter, 'messages.dropped.no_subscribers'}, + {counter, 'messages.dropped.no_subscribers', + <<"Number of messages dropped due to no subscribers">>}, %% Messages forward - {counter, 'messages.forward'}, + {counter, 'messages.forward', <<"Number of messages forwarded to other nodes">>}, %% Messages delayed - {counter, 'messages.delayed'}, + {counter, 'messages.delayed', <<"Number of delay-published messages">>}, %% Messages delivered - {counter, 'messages.delivered'}, + {counter, 'messages.delivered', + <<"Number of messages forwarded to the subscription process internally">>}, %% Messages acked - {counter, 'messages.acked'}, + {counter, 'messages.acked', <<"Number of received PUBACK and PUBREC packet">>}, %% Messages persistently stored - {counter, 'messages.persisted'} + {counter, 'messages.persisted', <<"Number of message persisted">>} ]). %% Delivery metrics -define(DELIVERY_METRICS, [ %% All Dropped during delivery - {counter, 'delivery.dropped'}, + {counter, 'delivery.dropped', <<"Total number of discarded messages when sending">>}, %% Dropped due to no_local - {counter, 'delivery.dropped.no_local'}, + {counter, 'delivery.dropped.no_local', << + "Number of messages that were dropped due to the No Local subscription " + "option when sending" + >>}, %% Dropped due to message too large - {counter, 'delivery.dropped.too_large'}, + {counter, 'delivery.dropped.too_large', << + "The number of messages that were dropped because the length exceeded " + "the limit when sending" + >>}, %% Dropped qos0 message - {counter, 'delivery.dropped.qos0_msg'}, + {counter, 'delivery.dropped.qos0_msg', << + "Number of messages with QoS 0 that were dropped because the message " + "queue was full when sending" + >>}, %% Dropped due to queue full - {counter, 'delivery.dropped.queue_full'}, + {counter, 'delivery.dropped.queue_full', << + "Number of messages with a non-zero QoS that were dropped because the " + "message queue was full when sending" + >>}, %% Dropped due to expired - {counter, 'delivery.dropped.expired'} + {counter, 'delivery.dropped.expired', + <<"Number of messages dropped due to message expiration on sending">>} ]). %% Client Lifecircle metrics -define(CLIENT_METRICS, [ - {counter, 'client.connect'}, - {counter, 'client.connack'}, - {counter, 'client.connected'}, - {counter, 'client.authenticate'}, - {counter, 'client.auth.anonymous'}, - {counter, 'client.authorize'}, - {counter, 'client.subscribe'}, - {counter, 'client.unsubscribe'}, - {counter, 'client.disconnected'} + {counter, 'client.connect', <<"Number of client connections">>}, + {counter, 'client.connack', <<"Number of CONNACK packet sent">>}, + {counter, 'client.connected', <<"Number of successful client connections">>}, + {counter, 'client.authenticate', <<"Number of client authentications">>}, + {counter, 'client.auth.anonymous', <<"Number of clients who log in anonymously">>}, + {counter, 'client.authorize', <<"Number of Authorization rule checks">>}, + {counter, 'client.subscribe', <<"Number of client subscriptions">>}, + {counter, 'client.unsubscribe', <<"Number of client unsubscriptions">>}, + {counter, 'client.disconnected', <<"Number of client disconnects">>} ]). %% Session Lifecircle metrics -define(SESSION_METRICS, [ - {counter, 'session.created'}, - {counter, 'session.resumed'}, - %% Session taken over by another client (Connect with clean_session|clean_start=false) - {counter, 'session.takenover'}, - %% Session taken over by another client (Connect with clean_session|clean_start=true) - {counter, 'session.discarded'}, - {counter, 'session.terminated'} + {counter, 'session.created', <<"Number of sessions created">>}, + {counter, 'session.resumed', + <<"Number of sessions resumed because Clean Session or Clean Start is false">>}, + {counter, 'session.takenover', + <<"Number of sessions takenover because Clean Session or Clean Start is false">>}, + {counter, 'session.discarded', + <<"Number of sessions dropped because Clean Session or Clean Start is true">>}, + {counter, 'session.terminated', <<"Number of terminated sessions">>} ]). %% Statistic metrics for ACL checking -define(STASTS_ACL_METRICS, [ - {counter, 'authorization.allow'}, - {counter, 'authorization.deny'}, - {counter, 'authorization.cache_hit'}, - {counter, 'authorization.cache_miss'} + {counter, 'authorization.allow', <<"Number of Authorization allow">>}, + {counter, 'authorization.deny', <<"Number of Authorization deny">>}, + {counter, 'authorization.cache_hit', <<"Number of Authorization hits the cache">>}, + {counter, 'authorization.cache_miss', <<"Number of Authorization cache missing">>} ]). %% Statistic metrics for auth checking -define(STASTS_AUTHN_METRICS, [ - {counter, 'authentication.success'}, - {counter, 'authentication.success.anonymous'}, - {counter, 'authentication.failure'} + {counter, 'authentication.success', <<"Number of successful client Authentication">>}, + {counter, 'authentication.success.anonymous', + <<"Number of successful client Authentication due to anonymous">>}, + {counter, 'authentication.failure', <<"Number of failed client Authentication">>} ]). %% Overload protection counters -define(OLP_METRICS, [ - {counter, 'overload_protection.delay.ok'}, - {counter, 'overload_protection.delay.timeout'}, - {counter, 'overload_protection.hibernation'}, - {counter, 'overload_protection.gc'}, - {counter, 'overload_protection.new_conn'} + {counter, 'overload_protection.delay.ok', <<"Number of overload protection delayed">>}, + {counter, 'overload_protection.delay.timeout', + <<"Number of overload protection delay timeout">>}, + {counter, 'overload_protection.hibernation', <<"Number of overload protection hibernation">>}, + {counter, 'overload_protection.gc', <<"Number of overload protection garbage collection">>}, + {counter, 'overload_protection.new_conn', + <<"Number of overload protection close new incoming connection">>} ]). -endif. diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 9567eb404..9f949633d 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -88,7 +88,7 @@ -define(SERVER, ?MODULE). olp_metrics() -> - lists:map(fun({_, Metric}) -> Metric end, ?OLP_METRICS). + lists:map(fun({_, Metric, _}) -> Metric end, ?OLP_METRICS). -record(state, {next_idx = 1}). @@ -369,7 +369,7 @@ init([]) -> ]), % Store reserved indices ok = lists:foreach( - fun({Type, Name}) -> + fun({Type, Name, _Desc}) -> Idx = reserved_idx(Name), Metric = #metric{name = Name, type = Type, idx = Idx}, true = ets:insert(?TAB, Metric), diff --git a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl index f9834c81d..8d9539978 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_metrics.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_metrics.erl @@ -20,6 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hocon_types.hrl"). +-include_lib("emqx/include/emqx_metrics.hrl"). -import(hoconsc, [mk/2, ref/2]). @@ -125,198 +126,13 @@ properties() -> ]), lists:reverse( lists:foldl( - fun({_Type, MetricName}, Acc) -> - [m(MetricName) | Acc] + fun({_Type, MetricName, Desc}, Acc) -> + [m(MetricName, Desc) | Acc] end, [], Metrics ) ). -m('actions.failure' = K) -> - m(K, <<"Number of failure executions of the rule engine action">>); -m('actions.success' = K) -> - m(K, <<"Number of successful executions of the rule engine action">>); -m('bytes.received' = K) -> - m(K, <<"Number of bytes received ">>); -m('bytes.sent' = K) -> - m(K, <<"Number of bytes sent on this connection">>); -m('client.auth.anonymous' = K) -> - m(K, <<"Number of clients who log in anonymously">>); -m('client.authenticate' = K) -> - m(K, <<"Number of client authentications">>); -m('client.check_authz' = K) -> - m(K, <<"Number of Authorization rule checks">>); -m('client.connack' = K) -> - m(K, <<"Number of CONNACK packet sent">>); -m('client.connect' = K) -> - m(K, <<"Number of client connections">>); -m('client.connected' = K) -> - m(K, <<"Number of successful client connections">>); -m('client.disconnected' = K) -> - m(K, <<"Number of client disconnects">>); -m('client.subscribe' = K) -> - m(K, <<"Number of client subscriptions">>); -m('client.unsubscribe' = K) -> - m(K, <<"Number of client unsubscriptions">>); -m('delivery.dropped' = K) -> - m(K, <<"Total number of discarded messages when sending">>); -m('delivery.dropped.expired' = K) -> - m(K, <<"Number of messages dropped due to message expiration on sending">>); -m('delivery.dropped.no_local' = K) -> - m(K, << - "Number of messages that were dropped due to the No Local subscription " - "option when sending" - >>); -m('delivery.dropped.qos0_msg' = K) -> - m(K, << - "Number of messages with QoS 0 that were dropped because the message " - "queue was full when sending" - >>); -m('delivery.dropped.queue_full' = K) -> - m(K, << - "Number of messages with a non-zero QoS that were dropped because the " - "message queue was full when sending" - >>); -m('delivery.dropped.too_large' = K) -> - m(K, << - "The number of messages that were dropped because the length exceeded " - "the limit when sending" - >>); -m('messages.acked' = K) -> - m(K, <<"Number of received PUBACK and PUBREC packet">>); -m('messages.delayed' = K) -> - m(K, <<"Number of delay-published messages">>); -m('messages.delivered' = K) -> - m(K, <<"Number of messages forwarded to the subscription process internally">>); -m('messages.dropped' = K) -> - m(K, <<"Total number of messages dropped before forwarding to the subscription process">>); -m('messages.dropped.await_pubrel_timeout' = K) -> - m(K, <<"Number of messages dropped due to waiting PUBREL timeout">>); -m('messages.dropped.no_subscribers' = K) -> - m(K, <<"Number of messages dropped due to no subscribers">>); -m('messages.forward' = K) -> - m(K, <<"Number of messages forwarded to other nodes">>); -m('messages.publish' = K) -> - m(K, <<"Number of messages published in addition to system messages">>); -m('messages.qos0.received' = K) -> - m(K, <<"Number of QoS 0 messages received from clients">>); -m('messages.qos0.sent' = K) -> - m(K, <<"Number of QoS 0 messages sent to clients">>); -m('messages.qos1.received' = K) -> - m(K, <<"Number of QoS 1 messages received from clients">>); -m('messages.qos1.sent' = K) -> - m(K, <<"Number of QoS 1 messages sent to clients">>); -m('messages.qos2.received' = K) -> - m(K, <<"Number of QoS 2 messages received from clients">>); -m('messages.qos2.sent' = K) -> - m(K, <<"Number of QoS 2 messages sent to clients">>); -m('messages.received' = K) -> - m(K, << - "Number of messages received from the client, equal to the sum of " - "messages.qos0.received\fmessages.qos1.received and messages.qos2.received" - >>); -%% m( -%% 'messages.retained', -%% <<"Number of retained messages">> -%% ), -m('messages.sent' = K) -> - m(K, << - "Number of messages sent to the client, equal to the sum of " - "messages.qos0.sent\fmessages.qos1.sent and messages.qos2.sent" - >>); -m('packets.auth.received' = K) -> - m(K, <<"Number of received AUTH packet">>); -m('packets.auth.sent' = K) -> - m(K, <<"Number of sent AUTH packet">>); -m('packets.connack.auth_error' = K) -> - m(K, <<"Number of received CONNECT packet with failed authentication">>); -m('packets.connack.error' = K) -> - m(K, <<"Number of received CONNECT packet with unsuccessful connections">>); -m('packets.connack.sent' = K) -> - m(K, <<"Number of sent CONNACK packet">>); -m('packets.connect.received' = K) -> - m(K, <<"Number of received CONNECT packet">>); -m('packets.disconnect.received' = K) -> - m(K, <<"Number of received DISCONNECT packet">>); -m('packets.disconnect.sent' = K) -> - m(K, <<"Number of sent DISCONNECT packet">>); -m('packets.pingreq.received' = K) -> - m(K, <<"Number of received PINGREQ packet">>); -m('packets.pingresp.sent' = K) -> - m(K, <<"Number of sent PUBRESP packet">>); -m('packets.puback.inuse' = K) -> - m(K, <<"Number of received PUBACK packet with occupied identifiers">>); -m('packets.puback.missed' = K) -> - m(K, <<"Number of received packet with identifiers.">>); -m('packets.puback.received' = K) -> - m(K, <<"Number of received PUBACK packet">>); -m('packets.puback.sent' = K) -> - m(K, <<"Number of sent PUBACK packet">>); -m('packets.pubcomp.inuse' = K) -> - m(K, <<"Number of received PUBCOMP packet with occupied identifiers">>); -m('packets.pubcomp.missed' = K) -> - m(K, <<"Number of missed PUBCOMP packet">>); -m('packets.pubcomp.received' = K) -> - m(K, <<"Number of received PUBCOMP packet">>); -m('packets.pubcomp.sent' = K) -> - m(K, <<"Number of sent PUBCOMP packet">>); -m('packets.publish.auth_error' = K) -> - m(K, <<"Number of received PUBLISH packets with failed the Authorization check">>); -m('packets.publish.dropped' = K) -> - m(K, <<"Number of messages discarded due to the receiving limit">>); -m('packets.publish.error' = K) -> - m(K, <<"Number of received PUBLISH packet that cannot be published">>); -m('packets.publish.inuse' = K) -> - m(K, <<"Number of received PUBLISH packet with occupied identifiers">>); -m('packets.publish.received' = K) -> - m(K, <<"Number of received PUBLISH packet">>); -m('packets.publish.sent' = K) -> - m(K, <<"Number of sent PUBLISH packet">>); -m('packets.pubrec.inuse' = K) -> - m(K, <<"Number of received PUBREC packet with occupied identifiers">>); -m('packets.pubrec.missed' = K) -> - m(K, <<"Number of received PUBREC packet with unknown identifiers">>); -m('packets.pubrec.received' = K) -> - m(K, <<"Number of received PUBREC packet">>); -m('packets.pubrec.sent' = K) -> - m(K, <<"Number of sent PUBREC packet">>); -m('packets.pubrel.missed' = K) -> - m(K, <<"Number of received PUBREC packet with unknown identifiers">>); -m('packets.pubrel.received' = K) -> - m(K, <<"Number of received PUBREL packet">>); -m('packets.pubrel.sent' = K) -> - m(K, <<"Number of sent PUBREL packet">>); -m('packets.received' = K) -> - m(K, <<"Number of received packet">>); -m('packets.sent' = K) -> - m(K, <<"Number of sent packet">>); -m('packets.suback.sent' = K) -> - m(K, <<"Number of sent SUBACK packet">>); -m('packets.subscribe.auth_error' = K) -> - m(K, <<"Number of received SUBACK packet with failed Authorization check">>); -m('packets.subscribe.error' = K) -> - m(K, <<"Number of received SUBSCRIBE packet with failed subscriptions">>); -m('packets.subscribe.received' = K) -> - m(K, <<"Number of received SUBSCRIBE packet">>); -m('packets.unsuback.sent' = K) -> - m(K, <<"Number of sent UNSUBACK packet">>); -m('packets.unsubscribe.error' = K) -> - m(K, <<"Number of received UNSUBSCRIBE packet with failed unsubscriptions">>); -m('packets.unsubscribe.received' = K) -> - m(K, <<"Number of received UNSUBSCRIBE packet">>); -m('rules.matched' = K) -> - m(K, <<"Number of rule matched">>); -m('session.created' = K) -> - m(K, <<"Number of sessions created">>); -m('session.discarded' = K) -> - m(K, <<"Number of sessions dropped because Clean Session or Clean Start is true">>); -m('session.resumed' = K) -> - m(K, <<"Number of sessions resumed because Clean Session or Clean Start is false">>); -m('session.takenover' = K) -> - m(K, <<"Number of sessions takenover because Clean Session or Clean Start is false">>); -m('session.terminated' = K) -> - m(K, <<"Number of terminated sessions">>). - m(K, Desc) -> {K, mk(non_neg_integer(), #{desc => Desc})}. From daeb10151b4af242a4b08de099533db081552e5c Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 12 Jun 2024 18:59:41 +0800 Subject: [PATCH 35/39] chore: fix typos --- apps/emqx/include/emqx_metrics.hrl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_metrics.hrl b/apps/emqx/include/emqx_metrics.hrl index b8d6da018..ddb537e6c 100644 --- a/apps/emqx/include/emqx_metrics.hrl +++ b/apps/emqx/include/emqx_metrics.hrl @@ -32,7 +32,7 @@ {counter, 'packets.connack.error', <<"Number of received CONNECT packet with unsuccessful connections">>}, {counter, 'packets.connack.auth_error', - <<"Number of received CONNECT packet with failed authentication">>}, + <<"Number of received CONNECT packet with failed Authentication">>}, {counter, 'packets.publish.received', <<"Number of received PUBLISH packet">>}, %% PUBLISH packets sent {counter, 'packets.publish.sent', <<"Number of sent PUBLISH packet">>}, @@ -207,8 +207,8 @@ -define(CLIENT_METRICS, [ {counter, 'client.connect', <<"Number of client connections">>}, {counter, 'client.connack', <<"Number of CONNACK packet sent">>}, - {counter, 'client.connected', <<"Number of successful client connections">>}, - {counter, 'client.authenticate', <<"Number of client authentications">>}, + {counter, 'client.connected', <<"Number of successful client connected">>}, + {counter, 'client.authenticate', <<"Number of client Authentication">>}, {counter, 'client.auth.anonymous', <<"Number of clients who log in anonymously">>}, {counter, 'client.authorize', <<"Number of Authorization rule checks">>}, {counter, 'client.subscribe', <<"Number of client subscriptions">>}, From 0f53b33417d4221de86b485c151074b6be72302c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 13 Jun 2024 11:57:36 +0200 Subject: [PATCH 36/39] chore(s3): make schema fields with defaults optional --- apps/emqx_s3/src/emqx_s3.app.src | 2 +- apps/emqx_s3/src/emqx_s3_schema.erl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src index c307f2c9c..f9374bfd9 100644 --- a/apps/emqx_s3/src/emqx_s3.app.src +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -1,6 +1,6 @@ {application, emqx_s3, [ {description, "EMQX S3"}, - {vsn, "5.1.0"}, + {vsn, "5.1.1"}, {modules, []}, {registered, [emqx_s3_sup]}, {applications, [ diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index efef32aa0..a8d6c8750 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -121,7 +121,7 @@ fields(s3_uploader) -> #{ default => <<"5mb">>, desc => ?DESC("min_part_size"), - required => true, + required => false, validator => fun part_size_validator/1 } )}, @@ -131,7 +131,7 @@ fields(s3_uploader) -> #{ default => <<"5gb">>, desc => ?DESC("max_part_size"), - required => true, + required => false, validator => fun part_size_validator/1 } )} From 1664ea4ad4a0ad309fad3e7f5d730329b5a91536 Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 13 Jun 2024 13:31:58 +0200 Subject: [PATCH 37/39] Revert: TLS partial-chain and keyUsage #12955 #12977 This reverts commit 28b17a25624409553c50cf5c31e79985f7cae376. This reverts commit 01467246fc253f3c64b078d19059ca92c78cea1a. This reverts commit c3f8ba57623cdb9c4439f7894d1c2fac4c3dcedb. This reverts commit 1a4a4bb3a53c72c7d52688c5ed59086f5d7e6b8f. This reverts commit fb30207ef3ca1e2c86c813ffacc489f35a5dd958. This reverts commit 337c230e79a6f53eba7dbec503940c12a478fc00. This reverts commit 3a674f44f12a5e2e3e6aa7c41609bc3e41e1815b. This reverts commit 70ffd77f995e05c0eb6821f709f4769c392e614b. This reverts commit 03b093556472fe763531c655f35db990999c413d. This reverts commit 650cf4b27ea50bd93590bfdaa629c5a207bf8990. This reverts commit 43ad665dcfdd354bc9a717db00712a52890e7d6d. This reverts commit a29a43e5fc9e1d455893ed05468f13908c21af74. This reverts commit 4e9c1ec0c9ec22559866ae19a2a0e254addb2059. This reverts commit 8eb463c58d6f8548e3c4088b9c05cf187bb27d49. This reverts commit 90430fa66dca7d828858aa06ca03ce44c0cd2629. This reverts commit eb1ab9adfe86dc917260055d1e6080a4f29021fa. This reverts commit 8bc3a86f63315e090384b0137ddc49be1df74735. This reverts commit fa4357ce89e141fbbf41ad525ab34ea472fd138e. This reverts commit 0b95a08d32db66903ba8155d7e405c901f044427. --- apps/emqx/src/emqx_const_v2.erl | 125 ---- apps/emqx/src/emqx_listeners.erl | 28 +- apps/emqx/src/emqx_schema.erl | 16 - apps/emqx/src/emqx_tls_lib.erl | 55 -- .../emqx_listener_tls_verify_chain_SUITE.erl | 257 ------- ...mqx_listener_tls_verify_keyusage_SUITE.erl | 372 --------- ...istener_tls_verify_partial_chain_SUITE.erl | 708 ------------------ apps/emqx/test/emqx_test_tls_certs_helper.erl | 319 -------- apps/emqx_gateway/src/emqx_gateway_utils.erl | 8 - .../test/emqx_mgmt_api_configs_SUITE.erl | 1 - changes/ce/feat-11721.en.md | 22 - mix.exs | 3 +- rebar.config | 3 +- rel/i18n/emqx_schema.hocon | 43 -- scripts/spellcheck/dicts/emqx.txt | 1 - 15 files changed, 5 insertions(+), 1956 deletions(-) delete mode 100644 apps/emqx/src/emqx_const_v2.erl delete mode 100644 apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl delete mode 100644 apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl delete mode 100644 apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl delete mode 100644 apps/emqx/test/emqx_test_tls_certs_helper.erl delete mode 100644 changes/ce/feat-11721.en.md diff --git a/apps/emqx/src/emqx_const_v2.erl b/apps/emqx/src/emqx_const_v2.erl deleted file mode 100644 index 0d95cf43c..000000000 --- a/apps/emqx/src/emqx_const_v2.erl +++ /dev/null @@ -1,125 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%% -%% @doc Never update this module, create a v3 instead. -%%-------------------------------------------------------------------- - --module(emqx_const_v2). --elvis([{elvis_style, atom_naming_convention, #{regex => "^([a-z][a-z0-9A-Z]*_?)*(_SUITE)?$"}}]). - --export([ - make_tls_root_fun/2, - make_tls_verify_fun/2 -]). - --include_lib("public_key/include/public_key.hrl"). -%% @doc Build a root fun for verify TLS partial_chain. -%% The `InputChain' is composed by OTP SSL with local cert store -%% AND the cert (chain if any) from the client. -%% @end -make_tls_root_fun(cacert_from_cacertfile, [Trusted]) -> - %% Allow only one trusted ca cert, and just return the defined trusted CA cert, - fun(_InputChain) -> - %% Note, returing `trusted_ca` doesn't really mean it accepts the connection - %% OTP SSL app will do the path validation, signature validation subsequently. - {trusted_ca, Trusted} - end; -make_tls_root_fun(cacert_from_cacertfile, [TrustedOne, TrustedTwo]) -> - %% Allow two trusted CA certs in case of CA cert renewal - %% This is a little expensive call as it compares the binaries. - fun(InputChain) -> - case lists:member(TrustedOne, InputChain) of - true -> - {trusted_ca, TrustedOne}; - false -> - {trusted_ca, TrustedTwo} - end - end. - -make_tls_verify_fun(verify_cert_extKeyUsage, KeyUsages) -> - RequiredKeyUsages = ext_key_opts(KeyUsages), - {fun verify_fun_peer_extKeyUsage/3, RequiredKeyUsages}. - -verify_fun_peer_extKeyUsage(_, {bad_cert, invalid_ext_key_usage}, UserState) -> - %% !! Override OTP verify peer default - %% OTP SSL is unhappy with the ext_key_usage but we will check on our own. - {unknown, UserState}; -verify_fun_peer_extKeyUsage(_, {bad_cert, _} = Reason, _UserState) -> - %% OTP verify_peer default - {fail, Reason}; -verify_fun_peer_extKeyUsage(_, {extension, _}, UserState) -> - %% OTP verify_peer default - {unknown, UserState}; -verify_fun_peer_extKeyUsage(_, valid, UserState) -> - %% OTP verify_peer default - {valid, UserState}; -verify_fun_peer_extKeyUsage( - #'OTPCertificate'{tbsCertificate = #'OTPTBSCertificate'{extensions = ExtL}}, - %% valid peer cert - valid_peer, - RequiredKeyUsages -) -> - %% override OTP verify_peer default - %% must have id-ce-extKeyUsage - case lists:keyfind(?'id-ce-extKeyUsage', 2, ExtL) of - #'Extension'{extnID = ?'id-ce-extKeyUsage', extnValue = VL} -> - case do_verify_ext_key_usage(VL, RequiredKeyUsages) of - true -> - %% pass the check, - %% fallback to OTP verify_peer default - {valid, RequiredKeyUsages}; - false -> - {fail, extKeyUsage_unmatched} - end; - _ -> - {fail, extKeyUsage_not_set} - end. - -%% @doc check required extkeyUsages are presented in the cert -do_verify_ext_key_usage(_, []) -> - %% Verify finished - true; -do_verify_ext_key_usage(CertExtL, [Usage | T] = _Required) -> - case lists:member(Usage, CertExtL) of - true -> - do_verify_ext_key_usage(CertExtL, T); - false -> - false - end. - -%% @doc Helper tls cert extension --spec ext_key_opts(string()) -> [OidString :: string() | public_key:oid()]. -ext_key_opts(Str) -> - Usages = string:tokens(Str, ","), - lists:map( - fun - ("clientAuth") -> - ?'id-kp-clientAuth'; - ("serverAuth") -> - ?'id-kp-serverAuth'; - ("codeSigning") -> - ?'id-kp-codeSigning'; - ("emailProtection") -> - ?'id-kp-emailProtection'; - ("timeStamping") -> - ?'id-kp-timeStamping'; - ("ocspSigning") -> - ?'id-kp-OCSPSigning'; - ("OID:" ++ OidStr) -> - OidList = string:tokens(OidStr, "."), - list_to_tuple(lists:map(fun list_to_integer/1, OidList)) - end, - Usages - ). diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 122118c6d..dd9024fef 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -611,9 +611,7 @@ esockd_opts(ListenerId, Type, Name, Opts0) -> ssl -> OptsWithCRL = inject_crl_config(Opts0), OptsWithSNI = inject_sni_fun(ListenerId, OptsWithCRL), - OptsWithRootFun = inject_root_fun(OptsWithSNI), - OptsWithVerifyFun = inject_verify_fun(OptsWithRootFun), - SSLOpts = ssl_opts(OptsWithVerifyFun), + SSLOpts = ssl_opts(OptsWithSNI), Opts3#{ssl_options => SSLOpts, tcp_options => tcp_opts(Opts0)} end ). @@ -637,18 +635,8 @@ ranch_opts(Type, Opts = #{bind := ListenOn}) -> MaxConnections = maps:get(max_connections, Opts, 1024), SocketOpts = case Type of - wss -> - tcp_opts(Opts) ++ - lists:filter( - fun - ({partial_chain, _}) -> false; - ({handshake_timeout, _}) -> false; - (_) -> true - end, - ssl_opts(Opts) - ); - ws -> - tcp_opts(Opts) + wss -> tcp_opts(Opts) ++ proplists:delete(handshake_timeout, ssl_opts(Opts)); + ws -> tcp_opts(Opts) end, #{ num_acceptors => NumAcceptors, @@ -974,16 +962,6 @@ quic_listener_optional_settings() -> stateless_operation_expiration_ms ]. -inject_root_fun(#{ssl_options := SslOpts} = Opts) -> - Opts#{ssl_options := emqx_tls_lib:opt_partial_chain(SslOpts)}; -inject_root_fun(Opts) -> - Opts. - -inject_verify_fun(#{ssl_options := SslOpts} = Opts) -> - Opts#{ssl_options := emqx_tls_lib:opt_verify_fun(SslOpts)}; -inject_verify_fun(Opts) -> - Opts. - inject_sni_fun(ListenerId, Conf = #{ssl_options := #{ocsp := #{enable_ocsp_stapling := true}}}) -> emqx_ocsp_cache:inject_sni_fun(ListenerId, Conf); inject_sni_fun(_ListenerId, Conf) -> diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 6c8466aab..a83a13209 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -2178,22 +2178,6 @@ common_ssl_opts_schema(Defaults, Type) -> desc => ?DESC(common_ssl_opts_schema_verify) } )}, - {"partial_chain", - sc( - hoconsc:enum([true, false, two_cacerts_from_cacertfile, cacert_from_cacertfile]), - #{ - default => Df(partial_chain, false), - desc => ?DESC(common_ssl_opts_schema_partial_chain) - } - )}, - {"verify_peer_ext_key_usage", - sc( - string(), - #{ - required => false, - desc => ?DESC(common_ssl_opts_verify_peer_ext_key_usage) - } - )}, {"reuse_sessions", sc( boolean(), diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 09a846832..c524381ad 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -15,7 +15,6 @@ %%-------------------------------------------------------------------- -module(emqx_tls_lib). --elvis([{elvis_style, atom_naming_convention, #{regex => "^([a-z][a-z0-9A-Z]*_?)*(_SUITE)?$"}}]). %% version & cipher suites -export([ @@ -24,8 +23,6 @@ default_ciphers/0, selected_ciphers/1, integral_ciphers/2, - opt_partial_chain/1, - opt_verify_fun/1, all_ciphers_set_cached/0 ]). @@ -688,55 +685,3 @@ ensure_ssl_file_key(SSL, RequiredKeyPaths) -> [] -> ok; Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} end. - -%% @doc enable TLS partial_chain validation if set. --spec opt_partial_chain(SslOpts :: map()) -> NewSslOpts :: map(). -opt_partial_chain(#{partial_chain := false} = SslOpts) -> - maps:remove(partial_chain, SslOpts); -opt_partial_chain(#{partial_chain := true} = SslOpts) -> - SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(1, SslOpts)}; -opt_partial_chain(#{partial_chain := cacert_from_cacertfile} = SslOpts) -> - SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(1, SslOpts)}; -opt_partial_chain(#{partial_chain := two_cacerts_from_cacertfile} = SslOpts) -> - SslOpts#{partial_chain := rootfun_trusted_ca_from_cacertfile(2, SslOpts)}; -opt_partial_chain(SslOpts) -> - SslOpts. - -%% @doc make verify_fun if set. --spec opt_verify_fun(SslOpts :: map()) -> NewSslOpts :: map(). -opt_verify_fun(#{verify_peer_ext_key_usage := V} = SslOpts) when V =/= undefined -> - SslOpts#{verify_fun => emqx_const_v2:make_tls_verify_fun(verify_cert_extKeyUsage, V)}; -opt_verify_fun(SslOpts) -> - SslOpts. - -%% @doc Helper, make TLS root_fun -rootfun_trusted_ca_from_cacertfile(NumOfCerts, #{cacertfile := Cacertfile}) -> - case file:read_file(Cacertfile) of - {ok, PemBin} -> - try - do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, PemBin) - catch - _Error:_Info:ST -> - %% The cacertfile will be checked by OTP SSL as well and OTP choice to be silent on this. - %% We are touching security sutffs, don't leak extra info.. - ?SLOG(error, #{ - msg => "trusted_cacert_not_found_in_cacertfile", stacktrace => ST - }), - throw({error, ?FUNCTION_NAME}) - end; - {error, Reason} -> - throw({error, {read_cacertfile_error, Cacertfile, Reason}}) - end; -rootfun_trusted_ca_from_cacertfile(_NumOfCerts, _SslOpts) -> - throw({error, cacertfile_unset}). - -do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, PemBin) -> - %% The last one or two should be the top parent in the chain if it is a chain - Certs = public_key:pem_decode(PemBin), - Pos = length(Certs) - NumOfCerts + 1, - Trusted = [ - CADer - || {'Certificate', CADer, _} <- - lists:sublist(public_key:pem_decode(PemBin), Pos, NumOfCerts) - ], - emqx_const_v2:make_tls_root_fun(cacert_from_cacertfile, Trusted). diff --git a/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl deleted file mode 100644 index 0b445c939..000000000 --- a/apps/emqx/test/emqx_listener_tls_verify_chain_SUITE.erl +++ /dev/null @@ -1,257 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_listener_tls_verify_chain_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - --import( - emqx_test_tls_certs_helper, - [ - emqx_start_listener/4, - fail_when_ssl_error/1, - fail_when_no_ssl_alert/2, - generate_tls_certs/1 - ] -). - -all() -> emqx_common_test_helpers:all(?MODULE). - -init_per_suite(Config) -> - generate_tls_certs(Config), - application:ensure_all_started(esockd), - [{ssl_config, ssl_config_verify_peer()} | Config]. - -end_per_suite(_Config) -> - application:stop(esockd). - -t_conn_fail_with_intermediate_ca_cert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")}, - {verify, verify_none} - ], - 1000 - ), - - fail_when_no_ssl_alert(Socket, unknown_ca), - ok = ssl:close(Socket). - -t_conn_fail_with_other_intermediate_ca_cert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2.pem")}, - {verify, verify_none} - ], - 1000 - ), - - fail_when_no_ssl_alert(Socket, unknown_ca), - ok = ssl:close(Socket). - -t_conn_success_with_server_client_composed_complete_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% Server has root ca cert - Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "root.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} - ], - %% Client has complete chain - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}, - {verify, verify_none} - ], - 1000 - ), - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_success_with_other_signed_client_composed_complete_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% Server has root ca cert - Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "root.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} - ], - %% Client has partial_chain - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}, - {verify, verify_none} - ], - 1000 - ), - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_success_with_renewed_intermediate_root_bundle(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% Server has root ca cert - Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate1_renewed-root-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")}, - {verify, verify_none} - ], - 1000 - ), - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_success_with_client_complete_cert_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "root.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-complete-bundle.pem")}, - {verify, verify_none} - ], - 1000 - ), - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_fail_with_server_partial_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% imcomplete at server side - Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-complete-bundle.pem")}, - {versions, ['tlsv1.2']}, - {verify, verify_none} - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_fail_without_root_cacert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, [ - {cacertfile, filename:join(DataDir, "intermediate2.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}, - %% stick to tlsv1.2 for consistent error message - {versions, ['tlsv1.2']}, - {cacertfile, filename:join(DataDir, "intermediate2.pem")} - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -ssl_config_verify_peer() -> - [ - {verify, verify_peer}, - {fail_if_no_peer_cert, true} - ]. diff --git a/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl deleted file mode 100644 index 8265a7492..000000000 --- a/apps/emqx/test/emqx_listener_tls_verify_keyusage_SUITE.erl +++ /dev/null @@ -1,372 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_listener_tls_verify_keyusage_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - --import( - emqx_test_tls_certs_helper, - [ - fail_when_ssl_error/1, - fail_when_no_ssl_alert/2, - generate_tls_certs/1, - gen_host_cert/4, - emqx_start_listener/4 - ] -). - -all() -> - [ - {group, full_chain}, - {group, partial_chain} - ]. - -all_tc() -> - emqx_common_test_helpers:all(?MODULE). - -groups() -> - [ - {partial_chain, [], all_tc()}, - {full_chain, [], all_tc()} - ]. - -init_per_suite(Config) -> - generate_tls_certs(Config), - application:ensure_all_started(esockd), - Config. - -end_per_suite(_Config) -> - application:stop(esockd). - -init_per_group(full_chain, Config) -> - [{ssl_config, ssl_config_verify_peer_full_chain(Config)} | Config]; -init_per_group(partial_chain, Config) -> - [{ssl_config, ssl_config_verify_peer_partial_chain(Config)} | Config]; -init_per_group(_, Config) -> - Config. - -end_per_group(_, Config) -> - Config. - -t_conn_success_verify_peer_ext_key_usage_unset(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% Given listener keyusage unset - Options = [{ssl_options, ?config(ssl_config, Config)}], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - %% when client connect with cert without keyusage ext - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")}, - {verify, verify_none} - ], - 1000 - ), - %% Then connection success - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_success_verify_peer_ext_key_usage_undefined(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% Give listener keyusage is set to undefined - Options = [ - {ssl_options, [ - {verify_peer_ext_key_usage, undefined} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - %% when client connect with cert without keyusages ext - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")}, - {verify, verify_none} - ], - 1000 - ), - %% Then connection success - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_success_verify_peer_ext_key_usage_matched_predefined(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% Give listener keyusage is set to clientAuth - Options = [ - {ssl_options, [ - {verify_peer_ext_key_usage, "clientAuth"} - | ?config(ssl_config, Config) - ]} - ], - - %% When client cert has clientAuth that is matched - gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth"), - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, - {verify, verify_none} - ], - 1000 - ), - %% Then connection success - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_success_verify_peer_ext_key_usage_matched_raw_oid(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% Give listener keyusage is set to raw OID - - %% from OTP-PUB-KEY.hrl - Options = [ - {ssl_options, [ - {verify_peer_ext_key_usage, "OID:1.3.6.1.5.5.7.3.2"} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - %% When client cert has keyusage and matched. - gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth"), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, - {verify, verify_none} - ], - 1000 - ), - %% Then connection success - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_success_verify_peer_ext_key_usage_matched_ordered_list(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - - %% Give listener keyusage is clientAuth,serverAuth - Options = [ - {ssl_options, [ - {verify_peer_ext_key_usage, "clientAuth,serverAuth"} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - %% When client cert has the same keyusage ext list - gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth,serverAuth"), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, - {verify, verify_none} - ], - 1000 - ), - %% Then connection success - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_success_verify_peer_ext_key_usage_matched_unordered_list(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% Give listener keyusage is clientAuth,serverAuth - Options = [ - {ssl_options, [ - {verify_peer_ext_key_usage, "serverAuth,clientAuth"} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - %% When client cert has the same keyusage ext list but different order - gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth,serverAuth"), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, - {verify, verify_none} - ], - 1000 - ), - %% Then connection success - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_fail_verify_peer_ext_key_usage_unmatched_raw_oid(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% Give listener keyusage is using OID - Options = [ - {ssl_options, [ - {verify_peer_ext_key_usage, "OID:1.3.6.1.5.5.7.3.1"} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - - %% When client cert has the keyusage but not matching OID - gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "clientAuth"), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, - {verify, verify_none} - ], - 1000 - ), - - %% Then connecion should fail. - fail_when_no_ssl_alert(Socket, handshake_failure), - ok = ssl:close(Socket). - -t_conn_fail_verify_peer_ext_key_usage_empty_str(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, [ - {verify_peer_ext_key_usage, ""} - | ?config(ssl_config, Config) - ]} - ], - %% Give listener keyusage is empty string - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - %% When client connect with cert without keyusage - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")}, - {verify, verify_none} - ], - 1000 - ), - %% Then connecion should fail. - fail_when_no_ssl_alert(Socket, handshake_failure), - ok = ssl:close(Socket). - -t_conn_fail_client_keyusage_unmatch(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - - %% Give listener keyusage is clientAuth - Options = [ - {ssl_options, [ - {verify_peer_ext_key_usage, "clientAuth"} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - %% When client connect with mismatch cert keyusage = codeSigning - gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "codeSigning"), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, client_key_file(DataDir, ?FUNCTION_NAME)}, - {certfile, client_pem_file(DataDir, ?FUNCTION_NAME)}, - {verify, verify_none} - ], - 1000 - ), - %% Then connecion should fail. - fail_when_no_ssl_alert(Socket, handshake_failure), - ok = ssl:close(Socket). - -t_conn_fail_client_keyusage_incomplete(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% Give listener keyusage is codeSigning,clientAuth - Options = [ - {ssl_options, [ - {verify_peer_ext_key_usage, - "serverAuth,clientAuth,codeSigning,emailProtection,timeStamping,ocspSigning"} - | ?config(ssl_config, Config) - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - %% When client connect with cert keyusage = clientAuth - gen_client_cert_ext_keyusage(?FUNCTION_NAME, "intermediate1", DataDir, "codeSigning"), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")}, - {verify, verify_none} - ], - 1000 - ), - %% Then connection should fail - fail_when_no_ssl_alert(Socket, handshake_failure), - ok = ssl:close(Socket). - -%%% -%%% Helpers -%%% -gen_client_cert_ext_keyusage(Name, CA, DataDir, Usage) when is_atom(Name) -> - gen_client_cert_ext_keyusage(atom_to_list(Name), CA, DataDir, Usage); -gen_client_cert_ext_keyusage(Name, CA, DataDir, Usage) -> - gen_host_cert(Name, CA, DataDir, #{ext => "extendedKeyUsage=" ++ Usage}). - -client_key_file(DataDir, Name) -> - filename:join(DataDir, Name) ++ ".key". - -client_pem_file(DataDir, Name) -> - filename:join(DataDir, Name) ++ ".pem". - -ssl_config_verify_peer_full_chain(Config) -> - [ - {cacertfile, filename:join(?config(data_dir, Config), "intermediate1-root-bundle.pem")} - | ssl_config_verify_peer(Config) - ]. -ssl_config_verify_peer_partial_chain(Config) -> - [ - {cacertfile, filename:join(?config(data_dir, Config), "intermediate1.pem")}, - {partial_chain, true} - | ssl_config_verify_peer(Config) - ]. - -ssl_config_verify_peer(Config) -> - DataDir = ?config(data_dir, Config), - [ - {verify, verify_peer}, - {fail_if_no_peer_cert, true}, - {keyfile, filename:join(DataDir, "server1.key")}, - {certfile, filename:join(DataDir, "server1.pem")} - %% , {log_level, debug} - ]. diff --git a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl b/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl deleted file mode 100644 index 1a1963dc9..000000000 --- a/apps/emqx/test/emqx_listener_tls_verify_partial_chain_SUITE.erl +++ /dev/null @@ -1,708 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_listener_tls_verify_partial_chain_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - --import( - emqx_test_tls_certs_helper, - [ - emqx_start_listener/4, - fail_when_ssl_error/1, - fail_when_no_ssl_alert/2, - generate_tls_certs/1 - ] -). - -all() -> emqx_common_test_helpers:all(?MODULE). - -init_per_suite(Config) -> - generate_tls_certs(Config), - application:ensure_all_started(esockd), - [{ssl_config, ssl_config_verify_partial_chain()} | Config]. - -end_per_suite(_Config) -> - application:stop(esockd). - -t_conn_success_with_server_intermediate_cacert_and_client_cert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ssl:close(Socket). - -t_conn_success_with_intermediate_cacert_bundle(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "server1-intermediate1-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ssl:close(Socket). - -t_conn_success_with_renewed_intermediate_cacert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate1_renewed.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ssl:close(Socket). - -t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_complete_bundle(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_bundle(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_renewed_client_cert( - Config -) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")}, - {partial_chain, two_cacerts_from_cacertfile} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2_renewed.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ssl:close(Socket). - -%% Note, this is good to have for usecase coverage -t_conn_success_with_new_intermediate_cacert_and_client_provides_renewed_client_cert_signed_by_old_intermediate( - Config -) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2_renewed.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ssl:close(Socket). - -%% @doc server should build a partial_chain with old version of ca cert. -t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_client_cert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")}, - {partial_chain, two_cacerts_from_cacertfile} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ssl:close(Socket). - -%% @doc verify when config does not allow two versions of certs from same trusted CA. -t_conn_fail_with_renewed_and_old_intermediate_cacert_and_client_using_old_bundle(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -%% @doc verify when config (two_cacerts_from_cacertfile) allows two versions of certs from same trusted CA. -t_001_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_old_bundle( - Config -) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")}, - {partial_chain, two_cacerts_from_cacertfile} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ssl:close(Socket). - -%% @doc: verify even if listener has old/new intermediate2 certs, -%% client1 should not able to connect with old intermediate2 cert. -%% In this case, listener verify_fun returns {trusted_ca, Oldintermediate2Cert} but -%% OTP should still fail the validation since the client1 cert is not signed by -%% Oldintermediate2Cert (trusted CA cert). -%% @end -t_conn_fail_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_all_CAcerts(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")}, - {partial_chain, two_cacerts_from_cacertfile} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "all-CAcerts-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_fail_with_renewed_intermediate_cacert_other_client(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate1_renewed.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_fail_with_intermediate_cacert_bundle_but_incorrect_order(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate1-server1-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_fail_when_singed_by_other_intermediate_ca(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_success_with_complete_chain_that_server_root_cacert_and_client_complete_cert_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "root.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_fail_with_other_client_complete_cert_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_fail_with_server_intermediate_and_other_client_complete_cert_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate1-root-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_success_with_server_intermediate_cacert_and_client_complete_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate2.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-complete-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_fail_with_server_intermediate_chain_and_client_other_incomplete_cert_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_fail_with_server_intermediate_and_other_client_root_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-root-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_success_with_server_intermediate_and_client_root_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate2.pem")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-root-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -%% @doc once rootCA cert present in cacertfile, sibling CA signed Client cert could connect. -t_conn_success_with_server_all_CA_bundle_and_client_root_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "all-CAcerts-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - {ok, Socket} = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-root-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_ssl_error(Socket), - ok = ssl:close(Socket). - -t_conn_fail_with_server_two_IA_bundle_and_client_root_chain(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "two-intermediates-bundle.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client2.key")}, - {certfile, filename:join(DataDir, "client2-root-bundle.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_conn_fail_with_server_partial_chain_false_intermediate_cacert_and_client_cert(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "intermediate1.pem")}, - {certfile, filename:join(DataDir, "server1.pem")}, - {keyfile, filename:join(DataDir, "server1.key")}, - {partial_chain, false} - ]} - ], - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options), - Res = ssl:connect( - {127, 0, 0, 1}, - Port, - [ - {keyfile, filename:join(DataDir, "client1.key")}, - {certfile, filename:join(DataDir, "client1.pem")} - | client_default_tls_opts() - ], - 1000 - ), - fail_when_no_ssl_alert(Res, unknown_ca). - -t_error_handling_invalid_cacertfile(Config) -> - Port = emqx_test_tls_certs_helper:select_free_port(ssl), - DataDir = ?config(data_dir, Config), - %% trigger error - Options = [ - {ssl_options, - ?config(ssl_config, Config) ++ - [ - {cacertfile, filename:join(DataDir, "server2.key")}, - {certfile, filename:join(DataDir, "server2.pem")}, - {keyfile, filename:join(DataDir, "server2.key")} - ]} - ], - ?assertException( - throw, - {error, rootfun_trusted_ca_from_cacertfile}, - emqx_start_listener(?FUNCTION_NAME, ssl, Port, Options) - ). - -ssl_config_verify_partial_chain() -> - [ - {verify, verify_peer}, - {fail_if_no_peer_cert, true}, - {partial_chain, true} - ]. - -client_default_tls_opts() -> - [ - {versions, ['tlsv1.2']}, - {verify, verify_none} - ]. diff --git a/apps/emqx/test/emqx_test_tls_certs_helper.erl b/apps/emqx/test/emqx_test_tls_certs_helper.erl deleted file mode 100644 index 78d51c5e0..000000000 --- a/apps/emqx/test/emqx_test_tls_certs_helper.erl +++ /dev/null @@ -1,319 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_test_tls_certs_helper). --export([ - gen_ca/2, - gen_host_cert/3, - gen_host_cert/4, - - select_free_port/1, - generate_tls_certs/1, - - fail_when_ssl_error/1, - fail_when_ssl_error/2, - fail_when_no_ssl_alert/2, - fail_when_no_ssl_alert/3, - - emqx_start_listener/4 -]). - --include_lib("common_test/include/ct.hrl"). - -%%------------------------------------------------------------------------------- -%% Start Listener -%%------------------------------------------------------------------------------- -emqx_start_listener(Name, Type, Port, Opts) when is_list(Opts) -> - emqx_start_listener(Name, Type, Port, maps:from_list(Opts)); -emqx_start_listener(Name, ssl, Port, #{ssl_options := SslOptions} = Opts0) -> - Opts = Opts0#{ - enable => true, - bind => {{127, 0, 0, 1}, Port}, - mountpoint => <<>>, - zone => default, - ssl_options => maps:from_list(SslOptions) - }, - ct:pal("start listener with ~p ~p", [Name, Opts]), - emqx_listeners:start_listener(ssl, Name, Opts). - -%%------------------------------------------------------------------------------- -%% TLS certs -%%------------------------------------------------------------------------------- -gen_ca(Path, Name) -> - %% Generate ca.pem and ca.key which will be used to generate certs - %% for hosts server and clients - ECKeyFile = eckey_name(Path), - filelib:ensure_dir(ECKeyFile), - os:cmd("openssl ecparam -name secp256r1 > " ++ ECKeyFile), - Cmd = lists:flatten( - io_lib:format( - "openssl req -new -x509 -nodes " - "-newkey ec:~s " - "-keyout ~s -out ~s -days 3650 " - "-addext basicConstraints=CA:TRUE " - "-subj \"/C=SE/O=TEST CA\"", - [ - ECKeyFile, - ca_key_name(Path, Name), - ca_cert_name(Path, Name) - ] - ) - ), - os:cmd(Cmd). - -ca_cert_name(Path, Name) -> - filename(Path, "~s.pem", [Name]). -ca_key_name(Path, Name) -> - filename(Path, "~s.key", [Name]). - -eckey_name(Path) -> - filename(Path, "ec.key", []). - -gen_host_cert(H, CaName, Path) -> - gen_host_cert(H, CaName, Path, #{}). - -gen_host_cert(H, CaName, Path, Opts) -> - ECKeyFile = eckey_name(Path), - CN = str(H), - HKey = filename(Path, "~s.key", [H]), - HCSR = filename(Path, "~s.csr", [H]), - HCSR2 = filename(Path, "~s.csr", [H]), - HPEM = filename(Path, "~s.pem", [H]), - HPEM2 = filename(Path, "~s_renewed.pem", [H]), - HEXT = filename(Path, "~s.extfile", [H]), - PasswordArg = - case maps:get(password, Opts, undefined) of - undefined -> - " -nodes "; - Password -> - io_lib:format(" -passout pass:'~s' ", [Password]) - end, - - create_file( - HEXT, - "keyUsage=digitalSignature,keyAgreement,keyCertSign\n" - "basicConstraints=CA:TRUE \n" - "~s \n" - "subjectAltName=DNS:~s\n", - [maps:get(ext, Opts, ""), CN] - ), - - CSR_Cmd = csr_cmd(PasswordArg, ECKeyFile, HKey, HCSR, CN), - CSR_Cmd2 = csr_cmd(PasswordArg, ECKeyFile, HKey, HCSR2, CN), - - CERT_Cmd = cert_sign_cmd( - HEXT, HCSR, ca_cert_name(Path, CaName), ca_key_name(Path, CaName), HPEM - ), - %% 2nd cert for testing renewed cert. - CERT_Cmd2 = cert_sign_cmd( - HEXT, HCSR2, ca_cert_name(Path, CaName), ca_key_name(Path, CaName), HPEM2 - ), - ct:pal(os:cmd(CSR_Cmd)), - ct:pal(os:cmd(CSR_Cmd2)), - ct:pal(os:cmd(CERT_Cmd)), - ct:pal(os:cmd(CERT_Cmd2)), - file:delete(HEXT). - -cert_sign_cmd(ExtFile, CSRFile, CACert, CAKey, OutputCert) -> - lists:flatten( - io_lib:format( - "openssl x509 -req " - "-extfile ~s " - "-in ~s -CA ~s -CAkey ~s -CAcreateserial " - "-out ~s -days 500", - [ - ExtFile, - CSRFile, - CACert, - CAKey, - OutputCert - ] - ) - ). - -csr_cmd(PasswordArg, ECKeyFile, HKey, HCSR, CN) -> - lists:flatten( - io_lib:format( - "openssl req -new ~s -newkey ec:~s " - "-keyout ~s -out ~s " - "-addext \"subjectAltName=DNS:~s\" " - "-addext basicConstraints=CA:TRUE " - "-addext keyUsage=digitalSignature,keyAgreement,keyCertSign " - "-subj \"/C=SE/O=TEST/CN=~s\"", - [PasswordArg, ECKeyFile, HKey, HCSR, CN, CN] - ) - ). - -filename(Path, F, A) -> - filename:join(Path, str(io_lib:format(F, A))). - -str(Arg) -> - binary_to_list(iolist_to_binary(Arg)). - -create_file(Filename, Fmt, Args) -> - filelib:ensure_dir(Filename), - {ok, F} = file:open(Filename, [write]), - try - io:format(F, Fmt, Args) - after - file:close(F) - end, - ok. - -%% @doc get unused port from OS --spec select_free_port(tcp | udp | ssl | quic) -> inets:port_number(). -select_free_port(tcp) -> - select_free_port(gen_tcp, listen); -select_free_port(udp) -> - select_free_port(gen_udp, open); -select_free_port(ssl) -> - select_free_port(tcp); -select_free_port(quic) -> - select_free_port(udp). - -select_free_port(GenModule, Fun) when - GenModule == gen_tcp orelse - GenModule == gen_udp --> - {ok, S} = GenModule:Fun(0, [{reuseaddr, true}]), - {ok, Port} = inet:port(S), - ok = GenModule:close(S), - case os:type() of - {unix, darwin} -> - %% in MacOS, still get address_in_use after close port - timer:sleep(500); - _ -> - skip - end, - ct:pal("Select free OS port: ~p", [Port]), - Port. - -%% @doc fail the test if ssl_error recvd -%% post check for success conn establishment -fail_when_ssl_error(Socket) -> - fail_when_ssl_error(Socket, 1000). -fail_when_ssl_error(Socket, Timeout) -> - receive - {ssl_error, Socket, _} -> - ct:fail("Handshake failed!") - after Timeout -> - ok - end. - -%% @doc fail the test if no ssl_error -fail_when_no_ssl_alert(Res, Alert) -> - fail_when_no_ssl_alert(Res, Alert, 1000). - -fail_when_no_ssl_alert({error, {tls_alert, {Alert, _}}}, Alert, _Timeout) -> - ok; -fail_when_no_ssl_alert({error, _} = Other, Alert, _Timeout) -> - ct:fail("returned unexpected ssl_error: ~p, expected ~n", [Other, Alert]); -fail_when_no_ssl_alert({ok, Socket}, Alert, Timeout) -> - fail_when_no_ssl_alert(Socket, Alert, Timeout); -fail_when_no_ssl_alert(Socket, Alert, Timeout) -> - receive - {ssl_error, Socket, {tls_alert, {Alert, AlertInfo}}} -> - ct:pal("alert info: ~p~n", [AlertInfo]); - {ssl_error, Socket, Other} -> - ct:fail("recv unexpected ssl_error: ~p~n", [Other]) - after Timeout -> - ct:fail("No expected alert: ~p from Socket: ~p ", [Alert, Socket]) - end. - -%% @doc Generate TLS cert chain for tests -generate_tls_certs(Config) -> - DataDir = ?config(data_dir, Config), - gen_ca(DataDir, "root"), - gen_host_cert("intermediate1", "root", DataDir), - gen_host_cert("intermediate2", "root", DataDir), - gen_host_cert("server1", "intermediate1", DataDir), - gen_host_cert("client1", "intermediate1", DataDir), - gen_host_cert("server2", "intermediate2", DataDir), - gen_host_cert("client2", "intermediate2", DataDir), - - %% Build bundles below - os:cmd( - io_lib:format("cat ~p ~p ~p > ~p", [ - filename:join(DataDir, "client2.pem"), - filename:join(DataDir, "intermediate2.pem"), - filename:join(DataDir, "root.pem"), - filename:join(DataDir, "client2-complete-bundle.pem") - ]) - ), - os:cmd( - io_lib:format("cat ~p ~p > ~p", [ - filename:join(DataDir, "client2.pem"), - filename:join(DataDir, "intermediate2.pem"), - filename:join(DataDir, "client2-intermediate2-bundle.pem") - ]) - ), - os:cmd( - io_lib:format("cat ~p ~p > ~p", [ - filename:join(DataDir, "client2.pem"), - filename:join(DataDir, "root.pem"), - filename:join(DataDir, "client2-root-bundle.pem") - ]) - ), - os:cmd( - io_lib:format("cat ~p ~p > ~p", [ - filename:join(DataDir, "server1.pem"), - filename:join(DataDir, "intermediate1.pem"), - filename:join(DataDir, "server1-intermediate1-bundle.pem") - ]) - ), - os:cmd( - io_lib:format("cat ~p ~p > ~p", [ - filename:join(DataDir, "intermediate1.pem"), - filename:join(DataDir, "server1.pem"), - filename:join(DataDir, "intermediate1-server1-bundle.pem") - ]) - ), - os:cmd( - io_lib:format("cat ~p ~p > ~p", [ - filename:join(DataDir, "intermediate1_renewed.pem"), - filename:join(DataDir, "root.pem"), - filename:join(DataDir, "intermediate1_renewed-root-bundle.pem") - ]) - ), - os:cmd( - io_lib:format("cat ~p ~p > ~p", [ - filename:join(DataDir, "intermediate2.pem"), - filename:join(DataDir, "intermediate2_renewed.pem"), - filename:join(DataDir, "intermediate2_renewed_old-bundle.pem") - ]) - ), - os:cmd( - io_lib:format("cat ~p ~p > ~p", [ - filename:join(DataDir, "intermediate1.pem"), - filename:join(DataDir, "root.pem"), - filename:join(DataDir, "intermediate1-root-bundle.pem") - ]) - ), - os:cmd( - io_lib:format("cat ~p ~p ~p > ~p", [ - filename:join(DataDir, "root.pem"), - filename:join(DataDir, "intermediate2.pem"), - filename:join(DataDir, "intermediate1.pem"), - filename:join(DataDir, "all-CAcerts-bundle.pem") - ]) - ), - os:cmd( - io_lib:format("cat ~p ~p > ~p", [ - filename:join(DataDir, "intermediate2.pem"), - filename:join(DataDir, "intermediate1.pem"), - filename:join(DataDir, "two-intermediates-bundle.pem") - ]) - ). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 3150ec675..8fd9a1519 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -559,8 +559,6 @@ ssl_opts(Name, Opts) -> [ fun ssl_opts_crl_config/2, fun ssl_opts_drop_unsupported/2, - fun ssl_partial_chain/2, - fun ssl_verify_fun/2, fun ssl_server_opts/2 ], SSLOpts, @@ -588,12 +586,6 @@ ssl_server_opts(SSLOpts, ssl_options) -> ssl_server_opts(SSLOpts, dtls_options) -> emqx_tls_lib:to_server_opts(dtls, SSLOpts). -ssl_partial_chain(SSLOpts, _Options) -> - emqx_tls_lib:opt_partial_chain(SSLOpts). - -ssl_verify_fun(SSLOpts, _Options) -> - emqx_tls_lib:opt_verify_fun(SSLOpts). - ranch_opts(Type, ListenOn, Opts) -> NumAcceptors = maps:get(acceptors, Opts, 4), MaxConnections = maps:get(max_connections, Opts, 1024), diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 496192e39..a2d4d21af 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -421,7 +421,6 @@ t_create_webhook_v1_bridges_api(Config) -> <<"enable">> => true, <<"hibernate_after">> => <<"5s">>, <<"log_level">> => <<"notice">>, - <<"partial_chain">> => false, <<"reuse_sessions">> => true, <<"secure_renegotiate">> => true, <<"user_lookup_fun">> => diff --git a/changes/ce/feat-11721.en.md b/changes/ce/feat-11721.en.md deleted file mode 100644 index 37eac8a5f..000000000 --- a/changes/ce/feat-11721.en.md +++ /dev/null @@ -1,22 +0,0 @@ -Enhance TLS listener to support more flexible TLS verifications. - -- partial_chain support - - If the option `partial_chain` is set to `true`, allow connections with incomplete certificate chains. - - Check the configuration manual document for more details. - -- Certificate KeyUsage Validation - - Added support for required Extended Key Usage defined in - [rfc5280](https://www.rfc-editor.org/rfc/rfc5280#section-4.2.1.12). - - Introduced a new option (`verify_peer_ext_key_usage`) to require specific key usages (like "serverAuth") - in peer certificates during the TLS handshake. - This strengthens security by ensuring certificates are used for their intended purposes. - - example: - "serverAuth,OID:1.3.6.1.5.5.7.3.2" - - Check the configuration manual document for more details. - diff --git a/mix.exs b/mix.exs index b31164e65..e81617dbb 100644 --- a/mix.exs +++ b/mix.exs @@ -101,8 +101,7 @@ defmodule EMQXUmbrella.MixProject do {:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.2", override: true}, {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true}, {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true}, - {:ra, "2.7.3", override: true}, - {:mimerl, "1.2.0", override: true} + {:ra, "2.7.3", override: true} ] ++ emqx_apps(profile_info, version) ++ enterprise_deps(profile_info) ++ jq_dep() ++ quicer_dep() diff --git a/rebar.config b/rebar.config index be47f8e4d..51a7ed17c 100644 --- a/rebar.config +++ b/rebar.config @@ -111,8 +111,7 @@ {ssl_verify_fun, "1.1.7"}, {rfc3339, {git, "https://github.com/emqx/rfc3339.git", {tag, "0.2.3"}}}, {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.2"}}}, - {ra, "2.7.3"}, - {mimerl, "1.2.0"} + {ra, "2.7.3"} ]}. {xref_ignores, diff --git a/rel/i18n/emqx_schema.hocon b/rel/i18n/emqx_schema.hocon index c6ec68d63..e80f36817 100644 --- a/rel/i18n/emqx_schema.hocon +++ b/rel/i18n/emqx_schema.hocon @@ -684,49 +684,6 @@ common_ssl_opts_schema_verify.desc: common_ssl_opts_schema_verify.label: """Verify peer""" -common_ssl_opts_schema_partial_chain.desc: -"""Enable or disable peer verification with partial_chain. -When local verifies a peer certificate during the x509 path validation -process, it constructs a certificate chain that starts with the peer -certificate and ends with a trust anchor. -By default, if it is set to `false`, the trust anchor is the -Root CA, and the certificate chain must be complete. -However, if the setting is set to `true` or `cacert_from_cacertfile`, -the last certificate in `cacertfile` will be used as the trust anchor -certificate (intermediate CA). This creates a partial chain -in the path validation. -Alternatively, if it is configured with `two_cacerts_from_cacertfile`, -one of the last two certificates in `cacertfile` will be used as the -trust anchor certificate, forming a partial chain. This option is -particularly useful for intermediate CA certificate rotation. -However, please note that it incurs some additional overhead, so it -should only be used for certificate rotation purposes.""" - -common_ssl_opts_schema_partial_chain.label: -"""Partial chain""" - -common_ssl_opts_verify_peer_ext_key_usage.desc: -"""Verify extended key usage in peer's certificate -For additional peer certificate validation, the value defined here must present in the -'Extended Key Usage' of peer certificate defined in -[rfc5280](https://www.rfc-editor.org/rfc/rfc5280#section-4.2.1.12). - -Allowed values are -- `clientAuth` -- `serverAuth` -- `codeSigning` -- `emailProtection` -- `timeStamping` -- `ocspSigning` -- raw OID, for example: "OID:1.3.6.1.5.5.7.3.2" means `id-pk 2` which is equivalent to `clientAuth` - -Comma-separated string is also supported for validating more than one key usages. - -For example, `"serverAuth,OID:1.3.6.1.5.5.7.3.2"`""" - -common_ssl_opts_verify_peer_ext_key_usage.label: -"""Verify KeyUsage in cert""" - fields_listeners_ssl.desc: """SSL listeners.""" diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index ce08d0f6b..7c888af49 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -310,4 +310,3 @@ ElasticSearch doc_as_upsert upsert aliyun -OID From b86d63174468e011364fdae47a2186db0356b37b Mon Sep 17 00:00:00 2001 From: William Yang Date: Thu, 13 Jun 2024 19:40:12 +0200 Subject: [PATCH 38/39] test: fix tc t_handle_outing_non_utf8_topic --- apps/emqx/test/emqx_connection_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 073496c4c..b025e9d08 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -338,7 +338,7 @@ t_handle_outing_non_utf8_topic(_) -> StrictOff = #{version => 5, max_size => 16#FFFF, strict_mode => false}, StOff = st(#{serialize => StrictOff}), OffResult = emqx_connection:handle_outgoing(Publish, StOff), - ?assertMatch(ok, OffResult), + ?assertMatch({ok, _}, OffResult), StrictOn = #{version => 5, max_size => 16#FFFF, strict_mode => true}, StOn = st(#{serialize => StrictOn}), ?assertError(frame_serialize_error, emqx_connection:handle_outgoing(Publish, StOn)). From 626aae6edfb7676293c8cec5a312d16f31f6d662 Mon Sep 17 00:00:00 2001 From: zmstone Date: Fri, 14 Jun 2024 16:57:53 +0200 Subject: [PATCH 39/39] chore: fix bad conflict resolution --- .../integration_test/emqx_persistent_session_ds_SUITE.erl | 4 ++-- apps/emqx/src/emqx_channel.erl | 2 +- apps/emqx_auth/include/emqx_authn.hrl | 1 + apps/emqx_auth/src/emqx_auth_utils.erl | 2 ++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl b/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl index 4f67443dd..8b0afa0b2 100644 --- a/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl +++ b/apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl @@ -251,7 +251,7 @@ t_session_subscription_idempotency(Config) -> ok end, - fun(Trace) -> + fun(_Trace) -> Session = session_open(Node1, ClientId), ?assertMatch( #{SubTopicFilter := #{}}, @@ -324,7 +324,7 @@ t_session_unsubscription_idempotency(Config) -> ok end, - fun(Trace) -> + fun(_Trace) -> Session = session_open(Node1, ClientId), ?assertEqual( #{}, diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 968ae22b1..3177c1c11 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1738,7 +1738,7 @@ maybe_add_cert(Map, #channel{conninfo = ConnInfo}) -> maybe_add_cert(Map, ConnInfo); maybe_add_cert(Map, #{peercert := PeerCert}) when is_binary(PeerCert) -> %% NOTE: it's raw binary at this point, - %% encoding to PEM (base64) is done lazy in emqx_authn_utils:render_var + %% encoding to PEM (base64) is done lazy in emqx_auth_utils:render_var Map#{cert_pem => PeerCert}; maybe_add_cert(Map, _) -> Map. diff --git a/apps/emqx_auth/include/emqx_authn.hrl b/apps/emqx_auth/include/emqx_authn.hrl index a55b9409d..782bfb9ca 100644 --- a/apps/emqx_auth/include/emqx_authn.hrl +++ b/apps/emqx_auth/include/emqx_authn.hrl @@ -39,6 +39,7 @@ ?VAR_PEERHOST, ?VAR_CERT_SUBJECT, ?VAR_CERT_CN_NAME, + ?VAR_CERT_PEM, ?VAR_NS_CLIENT_ATTRS ]). diff --git a/apps/emqx_auth/src/emqx_auth_utils.erl b/apps/emqx_auth/src/emqx_auth_utils.erl index 5056999d3..ca8c67a9e 100644 --- a/apps/emqx_auth/src/emqx_auth_utils.erl +++ b/apps/emqx_auth/src/emqx_auth_utils.erl @@ -207,6 +207,8 @@ render_var(_, undefined) -> % Any allowed but undefined binding will be replaced with empty string, even when % rendering SQL values. <<>>; +render_var(?VAR_CERT_PEM, Value) -> + base64:encode(Value); render_var(?VAR_PEERHOST, Value) -> inet:ntoa(Value); render_var(?VAR_PASSWORD, Value) ->