From 8717535d323035e61856663924d7fcb33ba6494a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 20 Aug 2022 15:31:34 +0200 Subject: [PATCH] refactor: populate ciphers list at runtime Populating ciphers list when checking schema makes the config file example and the schmea documents quite bloated --- apps/emqx/src/emqx_listeners.erl | 6 +- apps/emqx/src/emqx_schema.erl | 16 +- apps/emqx/src/emqx_tls_lib.erl | 250 +++++++++--------- apps/emqx/test/emqx_schema_tests.erl | 11 +- apps/emqx_gateway/src/emqx_gateway_schema.erl | 6 +- apps/emqx_gateway/src/emqx_gateway_utils.erl | 9 +- rel/emqx_conf.template.en.md | 1 + rel/emqx_conf.template.zh.md | 1 + 8 files changed, 136 insertions(+), 164 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 326661e75..1c9029778 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -583,11 +583,7 @@ enable_authn(Opts) -> maps:get(enable_authn, Opts, true). ssl_opts(Opts) -> - maps:to_list( - emqx_tls_lib:drop_tls13_for_old_otp( - maps:get(ssl_options, Opts, #{}) - ) - ). + emqx_tls_lib:to_server_opts(maps:get(ssl_options, Opts, #{})). tcp_opts(Opts) -> maps:to_list( diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 574305a4f..035096b45 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -102,7 +102,7 @@ -export([namespace/0, roots/0, roots/1, fields/1, desc/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). --export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1, default_ciphers/1]). +-export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1]). -export([sc/2, map/2]). -elvis([{elvis_style, god_modules, disable}]). @@ -2060,19 +2060,15 @@ default_ciphers(Which) -> do_default_ciphers(Which) ). -do_default_ciphers(undefined) -> - do_default_ciphers(tls_all_available); do_default_ciphers(quic) -> [ "TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256", "TLS_CHACHA20_POLY1305_SHA256" ]; -do_default_ciphers(dtls_all_available) -> - %% as of now, dtls does not support tlsv1.3 ciphers - emqx_tls_lib:selected_ciphers(['dtlsv1.2', 'dtlsv1']); -do_default_ciphers(tls_all_available) -> - emqx_tls_lib:default_ciphers(). +do_default_ciphers(_) -> + %% otherwise resolve default ciphers list at runtime + []. %% @private return a list of keys in a parent field -spec keys(string(), hocon:config()) -> [string()]. @@ -2246,8 +2242,8 @@ parse_user_lookup_fun(StrConf) -> {fun Mod:Fun/3, undefined}. validate_ciphers(Ciphers) -> - All = emqx_tls_lib:all_ciphers(), - case lists:filter(fun(Cipher) -> not lists:member(Cipher, All) end, Ciphers) of + Set = emqx_tls_lib:all_ciphers_set_cached(), + case lists:filter(fun(Cipher) -> not sets:is_element(Cipher, Set) end, Ciphers) of [] -> ok; Bad -> {error, {bad_ciphers, Bad}} end. diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index b08270df9..133c51338 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -23,8 +23,7 @@ default_ciphers/0, selected_ciphers/1, integral_ciphers/2, - drop_tls13_for_old_otp/1, - all_ciphers/0 + all_ciphers_set_cached/0 ]). %% SSL files @@ -38,6 +37,7 @@ ]). -export([ + to_server_opts/1, to_client_opts/1 ]). @@ -54,6 +54,63 @@ %% non-empty list of strings -define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso ?IS_STRING(hd(L)))). +%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl) +%% should return when running on otp 23. +%% But we still have to hard-code them because tlsv1.3 on otp 22 is +%% not trustworthy. +-define(TLSV13_EXCLUSIVE_CIPHERS, [ + "TLS_AES_256_GCM_SHA384", + "TLS_AES_128_GCM_SHA256", + "TLS_CHACHA20_POLY1305_SHA256", + "TLS_AES_128_CCM_SHA256", + "TLS_AES_128_CCM_8_SHA256" +]). + +-define(SELECTED_CIPHERS, [ + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES256-SHA384", + "ECDHE-RSA-AES256-SHA384", + "ECDH-ECDSA-AES256-GCM-SHA384", + "ECDH-RSA-AES256-GCM-SHA384", + "ECDH-ECDSA-AES256-SHA384", + "ECDH-RSA-AES256-SHA384", + "DHE-DSS-AES256-GCM-SHA384", + "DHE-DSS-AES256-SHA256", + "AES256-GCM-SHA384", + "AES256-SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-SHA256", + "ECDHE-RSA-AES128-SHA256", + "ECDH-ECDSA-AES128-GCM-SHA256", + "ECDH-RSA-AES128-GCM-SHA256", + "ECDH-ECDSA-AES128-SHA256", + "ECDH-RSA-AES128-SHA256", + "DHE-DSS-AES128-GCM-SHA256", + "DHE-DSS-AES128-SHA256", + "AES128-GCM-SHA256", + "AES128-SHA256", + "ECDHE-ECDSA-AES256-SHA", + "ECDHE-RSA-AES256-SHA", + "DHE-DSS-AES256-SHA", + "ECDH-ECDSA-AES256-SHA", + "ECDH-RSA-AES256-SHA", + "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", + "DHE-DSS-AES128-SHA", + "ECDH-ECDSA-AES128-SHA", + "ECDH-RSA-AES128-SHA", + + %% psk + "RSA-PSK-AES256-GCM-SHA384", + "RSA-PSK-AES256-CBC-SHA384", + "RSA-PSK-AES128-GCM-SHA256", + "RSA-PSK-AES128-CBC-SHA256", + "RSA-PSK-AES256-CBC-SHA", + "RSA-PSK-AES128-CBC-SHA" +]). + %% @doc Returns the default supported tls versions. -spec default_versions() -> [atom()]. default_versions() -> available_versions(). @@ -86,8 +143,19 @@ integral_versions(Desired) -> Filtered end. +%% @doc Return a set of all ciphers +all_ciphers_set_cached() -> + case persistent_term:get(?FUNCTION_NAME, false) of + false -> + S = sets:from_list(all_ciphers()), + persistent_term:put(?FUNCTION_NAME, S); + Set -> + Set + end. + %% @doc Return a list of all supported ciphers. -all_ciphers() -> all_ciphers(default_versions()). +all_ciphers() -> + all_ciphers(default_versions()). %% @doc Return a list of (openssl string format) cipher suites. -spec all_ciphers([ssl:tls_version()]) -> [string()]. @@ -96,23 +164,15 @@ all_ciphers(['tlsv1.3']) -> %% because 'all' returns legacy cipher suites too, %% which does not make sense since tlsv1.3 can not use %% legacy cipher suites. - ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); + ?TLSV13_EXCLUSIVE_CIPHERS; all_ciphers(Versions) -> %% assert non-empty List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]), [_ | _] = dedup(List). %% @doc All Pre-selected TLS ciphers. -%% ssl:cipher_suites(all, V, openssl) is too slow. so we cache default ciphers. default_ciphers() -> - case persistent_term:get(default_ciphers, undefined) of - undefined -> - Default = selected_ciphers(available_versions()), - persistent_term:put(default_ciphers, Default), - Default; - Default -> - Default - end. + selected_ciphers(available_versions()). %% @doc Pre-selected TLS ciphers for given versions.. selected_ciphers(Vsns) -> @@ -126,54 +186,11 @@ selected_ciphers(Vsns) -> do_selected_ciphers('tlsv1.3') -> case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of - true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); + true -> ?TLSV13_EXCLUSIVE_CIPHERS; false -> [] end ++ do_selected_ciphers('tlsv1.2'); do_selected_ciphers(_) -> - [ - "ECDHE-ECDSA-AES256-GCM-SHA384", - "ECDHE-RSA-AES256-GCM-SHA384", - "ECDHE-ECDSA-AES256-SHA384", - "ECDHE-RSA-AES256-SHA384", - "ECDH-ECDSA-AES256-GCM-SHA384", - "ECDH-RSA-AES256-GCM-SHA384", - "ECDH-ECDSA-AES256-SHA384", - "ECDH-RSA-AES256-SHA384", - "DHE-DSS-AES256-GCM-SHA384", - "DHE-DSS-AES256-SHA256", - "AES256-GCM-SHA384", - "AES256-SHA256", - "ECDHE-ECDSA-AES128-GCM-SHA256", - "ECDHE-RSA-AES128-GCM-SHA256", - "ECDHE-ECDSA-AES128-SHA256", - "ECDHE-RSA-AES128-SHA256", - "ECDH-ECDSA-AES128-GCM-SHA256", - "ECDH-RSA-AES128-GCM-SHA256", - "ECDH-ECDSA-AES128-SHA256", - "ECDH-RSA-AES128-SHA256", - "DHE-DSS-AES128-GCM-SHA256", - "DHE-DSS-AES128-SHA256", - "AES128-GCM-SHA256", - "AES128-SHA256", - "ECDHE-ECDSA-AES256-SHA", - "ECDHE-RSA-AES256-SHA", - "DHE-DSS-AES256-SHA", - "ECDH-ECDSA-AES256-SHA", - "ECDH-RSA-AES256-SHA", - "ECDHE-ECDSA-AES128-SHA", - "ECDHE-RSA-AES128-SHA", - "DHE-DSS-AES128-SHA", - "ECDH-ECDSA-AES128-SHA", - "ECDH-RSA-AES128-SHA", - - %% psk - "RSA-PSK-AES256-GCM-SHA384", - "RSA-PSK-AES256-CBC-SHA384", - "RSA-PSK-AES128-GCM-SHA256", - "RSA-PSK-AES128-CBC-SHA256", - "RSA-PSK-AES256-CBC-SHA", - "RSA-PSK-AES128-CBC-SHA" - ]. + ?SELECTED_CIPHERS. %% @doc Ensure version & cipher-suites integrity. -spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()]. @@ -209,9 +226,14 @@ available_versions() -> %% tlsv1.3 is available from OTP-22 but we do not want to use until 23. default_versions(OtpRelease) when OtpRelease >= 23 -> - proplists:get_value(available, ssl:versions()); + availables(); default_versions(_) -> - lists:delete('tlsv1.3', proplists:get_value(available, ssl:versions())). + lists:delete('tlsv1.3', availables()). + +availables() -> + All = ssl:versions(), + proplists:get_value(available, All) ++ + proplists:get_value(available_dtls, All). %% Deduplicate a list without re-ordering the elements. dedup([]) -> @@ -244,6 +266,8 @@ do_parse_versions([V | More], Acc) -> do_parse_versions(More, [Parsed | Acc]) end. +parse_version(<<"dtlsv1.2">>) -> 'dtlsv1.2'; +parse_version(<<"dtlsv1">>) -> dtlsv1; parse_version(<<"tlsv", Vsn/binary>>) -> parse_version(Vsn); parse_version(<<"v", Vsn/binary>>) -> parse_version(Vsn); parse_version(<<"1.3">>) -> 'tlsv1.3'; @@ -259,36 +283,6 @@ split_by_comma(Bin) -> trim_space(Bin) -> hd([I || I <- binary:split(Bin, <<" ">>), I =/= <<>>]). -%% @doc Drop tlsv1.3 version and ciphers from ssl options -%% if running on otp 22 or earlier. -drop_tls13_for_old_otp(SslOpts) -> - case list_to_integer(erlang:system_info(otp_release)) < 23 of - true -> drop_tls13(SslOpts); - false -> SslOpts - end. - -%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl) -%% should return when running on otp 23. -%% But we still have to hard-code them because tlsv1.3 on otp 22 is -%% not trustworthy. --define(TLSV13_EXCLUSIVE_CIPHERS, [ - "TLS_AES_256_GCM_SHA384", - "TLS_AES_128_GCM_SHA256", - "TLS_CHACHA20_POLY1305_SHA256", - "TLS_AES_128_CCM_SHA256", - "TLS_AES_128_CCM_8_SHA256" -]). -drop_tls13(SslOpts0) -> - SslOpts1 = - case maps:find(versions, SslOpts0) of - error -> SslOpts0; - {ok, Vsns} -> SslOpts0#{versions => (Vsns -- ['tlsv1.3'])} - end, - case maps:find(ciphers, SslOpts1) of - error -> SslOpts1; - {ok, Ciphers} -> SslOpts1#{ciphers => Ciphers -- ?TLSV13_EXCLUSIVE_CIPHERS} - end. - %% @doc The input map is a HOCON decoded result of a struct defined as %% emqx_schema:server_ssl_opts_schema. (NOTE: before schema-checked). %% `keyfile', `certfile' and `cacertfile' can be either pem format key or certificates, @@ -498,27 +492,46 @@ do_drop_invalid_certs([Key | Keys], SSL) -> end end. +%% @doc Convert hocon-checked ssl server options (map()) to +%% proplist accepted by ssl library. +to_server_opts(Opts) -> + Versions = integral_versions(maps:get(versions, Opts, undefined)), + Ciphers = integral_ciphers(Versions, maps:get(ciphers, Opts, undefined)), + maps:to_list(Opts#{ + ciphers => Ciphers, + versions => Versions + }). + %% @doc Convert hocon-checked ssl client options (map()) to %% proplist accepted by ssl library. to_client_opts(Opts) -> GetD = fun(Key, Default) -> fuzzy_map_get(Key, Opts, Default) end, Get = fun(Key) -> GetD(Key, undefined) end, - KeyFile = ensure_str(Get(keyfile)), - CertFile = ensure_str(Get(certfile)), - CAFile = ensure_str(Get(cacertfile)), - Verify = GetD(verify, verify_none), - SNI = ensure_sni(Get(server_name_indication)), - Versions = integral_versions(Get(versions)), - Ciphers = integral_ciphers(Versions, Get(ciphers)), - filter([ - {keyfile, KeyFile}, - {certfile, CertFile}, - {cacertfile, CAFile}, - {verify, Verify}, - {server_name_indication, SNI}, - {versions, Versions}, - {ciphers, Ciphers} - ]). + case GetD(enable, false) of + true -> + KeyFile = ensure_str(Get(keyfile)), + CertFile = ensure_str(Get(certfile)), + CAFile = ensure_str(Get(cacertfile)), + Verify = GetD(verify, verify_none), + SNI = ensure_sni(Get(server_name_indication)), + Versions = integral_versions(Get(versions)), + Ciphers = integral_ciphers(Versions, Get(ciphers)), + filter([ + {keyfile, KeyFile}, + {certfile, CertFile}, + {cacertfile, CAFile}, + {verify, Verify}, + {server_name_indication, SNI}, + {versions, Versions}, + {ciphers, Ciphers}, + {reuse_sessions, Get(reuse_sessions)}, + {depth, Get(depth)}, + {password, ensure_str(Get(password))}, + {secure_renegotiate, Get(secure_renegotiate)} + ]); + false -> + [] + end. filter([]) -> []; filter([{_, undefined} | T]) -> filter(T); @@ -556,28 +569,3 @@ ensure_ssl_file_key(SSL, RequiredKeys) -> [] -> ok; Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} end. - --if(?OTP_RELEASE > 22). --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -drop_tls13_test() -> - Versions = default_versions(), - ?assert(lists:member('tlsv1.3', Versions)), - Ciphers = all_ciphers(), - ?assert(has_tlsv13_cipher(Ciphers)), - Opts0 = #{versions => Versions, ciphers => Ciphers, other => true}, - Opts = drop_tls13(Opts0), - ?assertNot(lists:member('tlsv1.3', maps:get(versions, Opts, undefined))), - ?assertNot(has_tlsv13_cipher(maps:get(ciphers, Opts, undefined))). - -drop_tls13_no_versions_cipers_test() -> - Opts0 = #{other => 0, bool => true}, - Opts = drop_tls13(Opts0), - ?_assertEqual(Opts0, Opts). - -has_tlsv13_cipher(Ciphers) -> - lists:any(fun(C) -> lists:member(C, Ciphers) end, ?TLSV13_EXCLUSIVE_CIPHERS). - --endif. --endif. diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index a40026d4c..b081ec996 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -21,8 +21,7 @@ ssl_opts_dtls_test() -> Sc = emqx_schema:server_ssl_opts_schema( #{ - versions => dtls_all_available, - ciphers => dtls_all_available + versions => dtls_all_available }, false ), @@ -30,7 +29,7 @@ ssl_opts_dtls_test() -> ?assertMatch( #{ versions := ['dtlsv1.2', 'dtlsv1'], - ciphers := ["ECDHE-ECDSA-AES256-GCM-SHA384" | _] + ciphers := [] }, Checked ). @@ -42,7 +41,7 @@ ssl_opts_tls_1_3_test() -> ?assertMatch( #{ versions := ['tlsv1.3'], - ciphers := [_ | _] + ciphers := [] }, Checked ). @@ -53,7 +52,7 @@ ssl_opts_tls_for_ranch_test() -> ?assertMatch( #{ versions := ['tlsv1.3'], - ciphers := [_ | _], + ciphers := [], handshake_timeout := _ }, Checked @@ -125,7 +124,7 @@ validate(Schema, Data0) -> ), Checked. -ciperhs_schema_test() -> +ciphers_schema_test() -> Sc = emqx_schema:ciphers_schema(undefined), WSc = #{roots => [{ciphers, Sc}]}, ?assertThrow( diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index dfe937024..8850fa462 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -365,8 +365,7 @@ fields(ssl_server_opts) -> #{ depth => 10, reuse_sessions => true, - versions => tls_all_available, - ciphers => tls_all_available + versions => tls_all_available }, true ); @@ -502,8 +501,7 @@ fields(dtls_opts) -> #{ depth => 10, reuse_sessions => true, - versions => dtls_all_available, - ciphers => dtls_all_available + versions => dtls_all_available }, false ). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 15359dea6..5a38ee0f0 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -455,14 +455,7 @@ esockd_access_rules(StrRules) -> [Access(R) || R <- StrRules]. ssl_opts(Name, Opts) -> - maps:to_list( - emqx_tls_lib:drop_tls13_for_old_otp( - maps:without( - [enable], - maps:get(Name, Opts, #{}) - ) - ) - ). + emqx_tls_lib:to_server_opts(maps:get(Name, Opts, #{})). sock_opts(Name, Opts) -> maps:to_list( diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md index 3e56ea743..76d25680b 100644 --- a/rel/emqx_conf.template.en.md +++ b/rel/emqx_conf.template.en.md @@ -291,6 +291,7 @@ ciphers = "ECDH-ECDSA-AES128-SHA", "ECDH-RSA-AES128-SHA" ] +``` For PSK enabled listeners diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md index fbfa823e0..ac4c5ce39 100644 --- a/rel/emqx_conf.template.zh.md +++ b/rel/emqx_conf.template.zh.md @@ -272,6 +272,7 @@ ciphers = "ECDH-ECDSA-AES128-SHA", "ECDH-RSA-AES128-SHA" ] +``` 配置 PSK 认证的监听器