yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: populate ciphers list at runtime 

Populating ciphers list when checking schema makes the
config file example and the schmea documents quite bloated
This commit is contained in:
Zaiming (Stone) Shi 2022-08-20 15:31:34 +02:00
parent 22f5b62531
commit 8717535d32
8 changed files with 136 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/emqx/src/emqx_listeners.erl
View File

@ -583,11 +583,7 @@ enable_authn(Opts) ->
maps:get(enable_authn, Opts, true). maps:get(enable_authn, Opts, true).
ssl_opts(Opts) -> ssl_opts(Opts) ->
maps:to_list( emqx_tls_lib:to_server_opts(maps:get(ssl_options, Opts, #{})).
emqx_tls_lib:drop_tls13_for_old_otp(
maps:get(ssl_options, Opts, #{})
)
).
tcp_opts(Opts) -> tcp_opts(Opts) ->
maps:to_list( maps:to_list(

16
apps/emqx/src/emqx_schema.erl
View File

@ -102,7 +102,7 @@
-export([namespace/0, roots/0, roots/1, fields/1, desc/1]). -export([namespace/0, roots/0, roots/1, fields/1, desc/1]).
-export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]).
-export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1, default_ciphers/1]). -export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1]).
-export([sc/2, map/2]). -export([sc/2, map/2]).
-elvis([{elvis_style, god_modules, disable}]). -elvis([{elvis_style, god_modules, disable}]).
@ -2060,19 +2060,15 @@ default_ciphers(Which) ->
do_default_ciphers(Which) do_default_ciphers(Which)
). ).
do_default_ciphers(undefined) ->
do_default_ciphers(tls_all_available);
do_default_ciphers(quic) -> do_default_ciphers(quic) ->
[ [
"TLS_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384",
"TLS_AES_128_GCM_SHA256", "TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256" "TLS_CHACHA20_POLY1305_SHA256"
]; ];
do_default_ciphers(dtls_all_available) -> do_default_ciphers(_) ->
%% as of now, dtls does not support tlsv1.3 ciphers %% otherwise resolve default ciphers list at runtime
emqx_tls_lib:selected_ciphers(['dtlsv1.2', 'dtlsv1']); [].
do_default_ciphers(tls_all_available) ->
emqx_tls_lib:default_ciphers().
%% @private return a list of keys in a parent field %% @private return a list of keys in a parent field
-spec keys(string(), hocon:config()) -> [string()]. -spec keys(string(), hocon:config()) -> [string()].
@ -2246,8 +2242,8 @@ parse_user_lookup_fun(StrConf) ->
{fun Mod:Fun/3, undefined}. {fun Mod:Fun/3, undefined}.
validate_ciphers(Ciphers) -> validate_ciphers(Ciphers) ->
All = emqx_tls_lib:all_ciphers(), Set = emqx_tls_lib:all_ciphers_set_cached(),
case lists:filter(fun(Cipher) -> not lists:member(Cipher, All) end, Ciphers) of case lists:filter(fun(Cipher) -> not sets:is_element(Cipher, Set) end, Ciphers) of
[] -> ok; [] -> ok;
Bad -> {error, {bad_ciphers, Bad}} Bad -> {error, {bad_ciphers, Bad}}
end. end.

250
apps/emqx/src/emqx_tls_lib.erl
View File

@ -23,8 +23,7 @@
default_ciphers/0, default_ciphers/0,
selected_ciphers/1, selected_ciphers/1,
integral_ciphers/2, integral_ciphers/2,
drop_tls13_for_old_otp/1, all_ciphers_set_cached/0
all_ciphers/0
]). ]).
%% SSL files %% SSL files
@ -38,6 +37,7 @@
]). ]).
-export([ -export([
to_server_opts/1,
to_client_opts/1 to_client_opts/1
]). ]).
@ -54,6 +54,63 @@
%% non-empty list of strings %% non-empty list of strings
-define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso ?IS_STRING(hd(L)))). -define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso ?IS_STRING(hd(L)))).
%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl)
%% should return when running on otp 23.
%% But we still have to hard-code them because tlsv1.3 on otp 22 is
%% not trustworthy.
-define(TLSV13_EXCLUSIVE_CIPHERS, [
"TLS_AES_256_GCM_SHA384",
"TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256",
"TLS_AES_128_CCM_SHA256",
"TLS_AES_128_CCM_8_SHA256"
]).
-define(SELECTED_CIPHERS, [
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES256-SHA384",
"ECDHE-RSA-AES256-SHA384",
"ECDH-ECDSA-AES256-GCM-SHA384",
"ECDH-RSA-AES256-GCM-SHA384",
"ECDH-ECDSA-AES256-SHA384",
"ECDH-RSA-AES256-SHA384",
"DHE-DSS-AES256-GCM-SHA384",
"DHE-DSS-AES256-SHA256",
"AES256-GCM-SHA384",
"AES256-SHA256",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-AES128-SHA256",
"ECDHE-RSA-AES128-SHA256",
"ECDH-ECDSA-AES128-GCM-SHA256",
"ECDH-RSA-AES128-GCM-SHA256",
"ECDH-ECDSA-AES128-SHA256",
"ECDH-RSA-AES128-SHA256",
"DHE-DSS-AES128-GCM-SHA256",
"DHE-DSS-AES128-SHA256",
"AES128-GCM-SHA256",
"AES128-SHA256",
"ECDHE-ECDSA-AES256-SHA",
"ECDHE-RSA-AES256-SHA",
"DHE-DSS-AES256-SHA",
"ECDH-ECDSA-AES256-SHA",
"ECDH-RSA-AES256-SHA",
"ECDHE-ECDSA-AES128-SHA",
"ECDHE-RSA-AES128-SHA",
"DHE-DSS-AES128-SHA",
"ECDH-ECDSA-AES128-SHA",
"ECDH-RSA-AES128-SHA",
%% psk
"RSA-PSK-AES256-GCM-SHA384",
"RSA-PSK-AES256-CBC-SHA384",
"RSA-PSK-AES128-GCM-SHA256",
"RSA-PSK-AES128-CBC-SHA256",
"RSA-PSK-AES256-CBC-SHA",
"RSA-PSK-AES128-CBC-SHA"
]).
%% @doc Returns the default supported tls versions. %% @doc Returns the default supported tls versions.
-spec default_versions() -> [atom()]. -spec default_versions() -> [atom()].
default_versions() -> available_versions(). default_versions() -> available_versions().
@ -86,8 +143,19 @@ integral_versions(Desired) ->
Filtered Filtered
end. end.
%% @doc Return a set of all ciphers
all_ciphers_set_cached() ->
case persistent_term:get(?FUNCTION_NAME, false) of
false ->
S = sets:from_list(all_ciphers()),
persistent_term:put(?FUNCTION_NAME, S);
Set ->
Set
end.
%% @doc Return a list of all supported ciphers. %% @doc Return a list of all supported ciphers.
all_ciphers() -> all_ciphers(default_versions()). all_ciphers() ->
all_ciphers(default_versions()).
%% @doc Return a list of (openssl string format) cipher suites. %% @doc Return a list of (openssl string format) cipher suites.
-spec all_ciphers([ssl:tls_version()]) -> [string()]. -spec all_ciphers([ssl:tls_version()]) -> [string()].
@ -96,23 +164,15 @@ all_ciphers(['tlsv1.3']) ->
%% because 'all' returns legacy cipher suites too, %% because 'all' returns legacy cipher suites too,
%% which does not make sense since tlsv1.3 can not use %% which does not make sense since tlsv1.3 can not use
%% legacy cipher suites. %% legacy cipher suites.
ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); ?TLSV13_EXCLUSIVE_CIPHERS;
all_ciphers(Versions) -> all_ciphers(Versions) ->
%% assert non-empty %% assert non-empty
List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]), List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]),
[_ | _] = dedup(List). [_ | _] = dedup(List).
%% @doc All Pre-selected TLS ciphers. %% @doc All Pre-selected TLS ciphers.
%% ssl:cipher_suites(all, V, openssl) is too slow. so we cache default ciphers.
default_ciphers() -> default_ciphers() ->
case persistent_term:get(default_ciphers, undefined) of selected_ciphers(available_versions()).
undefined ->
Default = selected_ciphers(available_versions()),
persistent_term:put(default_ciphers, Default),
Default;
Default ->
Default
end.
%% @doc Pre-selected TLS ciphers for given versions.. %% @doc Pre-selected TLS ciphers for given versions..
selected_ciphers(Vsns) -> selected_ciphers(Vsns) ->
@ -126,54 +186,11 @@ selected_ciphers(Vsns) ->
do_selected_ciphers('tlsv1.3') -> do_selected_ciphers('tlsv1.3') ->
case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of
true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); true -> ?TLSV13_EXCLUSIVE_CIPHERS;
false -> [] false -> []
end ++ do_selected_ciphers('tlsv1.2'); end ++ do_selected_ciphers('tlsv1.2');
do_selected_ciphers(_) -> do_selected_ciphers(_) ->
[ ?SELECTED_CIPHERS.
"ECDHE-ECDSA-AES256-GCM-SHA384",
"ECDHE-RSA-AES256-GCM-SHA384",
"ECDHE-ECDSA-AES256-SHA384",
"ECDHE-RSA-AES256-SHA384",
"ECDH-ECDSA-AES256-GCM-SHA384",
"ECDH-RSA-AES256-GCM-SHA384",
"ECDH-ECDSA-AES256-SHA384",
"ECDH-RSA-AES256-SHA384",
"DHE-DSS-AES256-GCM-SHA384",
"DHE-DSS-AES256-SHA256",
"AES256-GCM-SHA384",
"AES256-SHA256",
"ECDHE-ECDSA-AES128-GCM-SHA256",
"ECDHE-RSA-AES128-GCM-SHA256",
"ECDHE-ECDSA-AES128-SHA256",
"ECDHE-RSA-AES128-SHA256",
"ECDH-ECDSA-AES128-GCM-SHA256",
"ECDH-RSA-AES128-GCM-SHA256",
"ECDH-ECDSA-AES128-SHA256",
"ECDH-RSA-AES128-SHA256",
"DHE-DSS-AES128-GCM-SHA256",
"DHE-DSS-AES128-SHA256",
"AES128-GCM-SHA256",
"AES128-SHA256",
"ECDHE-ECDSA-AES256-SHA",
"ECDHE-RSA-AES256-SHA",
"DHE-DSS-AES256-SHA",
"ECDH-ECDSA-AES256-SHA",
"ECDH-RSA-AES256-SHA",
"ECDHE-ECDSA-AES128-SHA",
"ECDHE-RSA-AES128-SHA",
"DHE-DSS-AES128-SHA",
"ECDH-ECDSA-AES128-SHA",
"ECDH-RSA-AES128-SHA",
%% psk
"RSA-PSK-AES256-GCM-SHA384",
"RSA-PSK-AES256-CBC-SHA384",
"RSA-PSK-AES128-GCM-SHA256",
"RSA-PSK-AES128-CBC-SHA256",
"RSA-PSK-AES256-CBC-SHA",
"RSA-PSK-AES128-CBC-SHA"
].
%% @doc Ensure version & cipher-suites integrity. %% @doc Ensure version & cipher-suites integrity.
-spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()]. -spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()].
@ -209,9 +226,14 @@ available_versions() ->
%% tlsv1.3 is available from OTP-22 but we do not want to use until 23. %% tlsv1.3 is available from OTP-22 but we do not want to use until 23.
default_versions(OtpRelease) when OtpRelease >= 23 -> default_versions(OtpRelease) when OtpRelease >= 23 ->
proplists:get_value(available, ssl:versions()); availables();
default_versions(_) -> default_versions(_) ->
lists:delete('tlsv1.3', proplists:get_value(available, ssl:versions())). lists:delete('tlsv1.3', availables()).
availables() ->
All = ssl:versions(),
proplists:get_value(available, All) ++
proplists:get_value(available_dtls, All).
%% Deduplicate a list without re-ordering the elements. %% Deduplicate a list without re-ordering the elements.
dedup([]) -> dedup([]) ->
@ -244,6 +266,8 @@ do_parse_versions([V | More], Acc) ->
do_parse_versions(More, [Parsed | Acc]) do_parse_versions(More, [Parsed | Acc])
end. end.
parse_version(<<"dtlsv1.2">>) -> 'dtlsv1.2';
parse_version(<<"dtlsv1">>) -> dtlsv1;
parse_version(<<"tlsv", Vsn/binary>>) -> parse_version(Vsn); parse_version(<<"tlsv", Vsn/binary>>) -> parse_version(Vsn);
parse_version(<<"v", Vsn/binary>>) -> parse_version(Vsn); parse_version(<<"v", Vsn/binary>>) -> parse_version(Vsn);
parse_version(<<"1.3">>) -> 'tlsv1.3'; parse_version(<<"1.3">>) -> 'tlsv1.3';
@ -259,36 +283,6 @@ split_by_comma(Bin) ->
trim_space(Bin) -> trim_space(Bin) ->
hd([I || I <- binary:split(Bin, <<" ">>), I =/= <<>>]). hd([I || I <- binary:split(Bin, <<" ">>), I =/= <<>>]).
%% @doc Drop tlsv1.3 version and ciphers from ssl options
%% if running on otp 22 or earlier.
drop_tls13_for_old_otp(SslOpts) ->
case list_to_integer(erlang:system_info(otp_release)) < 23 of
true -> drop_tls13(SslOpts);
false -> SslOpts
end.
%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl)
%% should return when running on otp 23.
%% But we still have to hard-code them because tlsv1.3 on otp 22 is
%% not trustworthy.
-define(TLSV13_EXCLUSIVE_CIPHERS, [
"TLS_AES_256_GCM_SHA384",
"TLS_AES_128_GCM_SHA256",
"TLS_CHACHA20_POLY1305_SHA256",
"TLS_AES_128_CCM_SHA256",
"TLS_AES_128_CCM_8_SHA256"
]).
drop_tls13(SslOpts0) ->
SslOpts1 =
case maps:find(versions, SslOpts0) of
error -> SslOpts0;
{ok, Vsns} -> SslOpts0#{versions => (Vsns -- ['tlsv1.3'])}
end,
case maps:find(ciphers, SslOpts1) of
error -> SslOpts1;
{ok, Ciphers} -> SslOpts1#{ciphers => Ciphers -- ?TLSV13_EXCLUSIVE_CIPHERS}
end.
%% @doc The input map is a HOCON decoded result of a struct defined as %% @doc The input map is a HOCON decoded result of a struct defined as
%% emqx_schema:server_ssl_opts_schema. (NOTE: before schema-checked). %% emqx_schema:server_ssl_opts_schema. (NOTE: before schema-checked).
%% `keyfile', `certfile' and `cacertfile' can be either pem format key or certificates, %% `keyfile', `certfile' and `cacertfile' can be either pem format key or certificates,
@ -498,27 +492,46 @@ do_drop_invalid_certs([Key | Keys], SSL) ->
end end
end. end.
%% @doc Convert hocon-checked ssl server options (map()) to
%% proplist accepted by ssl library.
to_server_opts(Opts) ->
Versions = integral_versions(maps:get(versions, Opts, undefined)),
Ciphers = integral_ciphers(Versions, maps:get(ciphers, Opts, undefined)),
maps:to_list(Opts#{
ciphers => Ciphers,
versions => Versions
}).
%% @doc Convert hocon-checked ssl client options (map()) to %% @doc Convert hocon-checked ssl client options (map()) to
%% proplist accepted by ssl library. %% proplist accepted by ssl library.
to_client_opts(Opts) -> to_client_opts(Opts) ->
GetD = fun(Key, Default) -> fuzzy_map_get(Key, Opts, Default) end, GetD = fun(Key, Default) -> fuzzy_map_get(Key, Opts, Default) end,
Get = fun(Key) -> GetD(Key, undefined) end, Get = fun(Key) -> GetD(Key, undefined) end,
KeyFile = ensure_str(Get(keyfile)), case GetD(enable, false) of
CertFile = ensure_str(Get(certfile)), true ->
CAFile = ensure_str(Get(cacertfile)), KeyFile = ensure_str(Get(keyfile)),
Verify = GetD(verify, verify_none), CertFile = ensure_str(Get(certfile)),
SNI = ensure_sni(Get(server_name_indication)), CAFile = ensure_str(Get(cacertfile)),
Versions = integral_versions(Get(versions)), Verify = GetD(verify, verify_none),
Ciphers = integral_ciphers(Versions, Get(ciphers)), SNI = ensure_sni(Get(server_name_indication)),
filter([ Versions = integral_versions(Get(versions)),
{keyfile, KeyFile}, Ciphers = integral_ciphers(Versions, Get(ciphers)),
{certfile, CertFile}, filter([
{cacertfile, CAFile}, {keyfile, KeyFile},
{verify, Verify}, {certfile, CertFile},
{server_name_indication, SNI}, {cacertfile, CAFile},
{versions, Versions}, {verify, Verify},
{ciphers, Ciphers} {server_name_indication, SNI},
]). {versions, Versions},
{ciphers, Ciphers},
{reuse_sessions, Get(reuse_sessions)},
{depth, Get(depth)},
{password, ensure_str(Get(password))},
{secure_renegotiate, Get(secure_renegotiate)}
]);
false ->
[]
end.
filter([]) -> []; filter([]) -> [];
filter([{_, undefined} | T]) -> filter(T); filter([{_, undefined} | T]) -> filter(T);
@ -556,28 +569,3 @@ ensure_ssl_file_key(SSL, RequiredKeys) ->
[] -> ok; [] -> ok;
Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}} Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}}
end. end.
-if(?OTP_RELEASE > 22).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
drop_tls13_test() ->
Versions = default_versions(),
?assert(lists:member('tlsv1.3', Versions)),
Ciphers = all_ciphers(),
?assert(has_tlsv13_cipher(Ciphers)),
Opts0 = #{versions => Versions, ciphers => Ciphers, other => true},
Opts = drop_tls13(Opts0),
?assertNot(lists:member('tlsv1.3', maps:get(versions, Opts, undefined))),
?assertNot(has_tlsv13_cipher(maps:get(ciphers, Opts, undefined))).
drop_tls13_no_versions_cipers_test() ->
Opts0 = #{other => 0, bool => true},
Opts = drop_tls13(Opts0),
?_assertEqual(Opts0, Opts).
has_tlsv13_cipher(Ciphers) ->
lists:any(fun(C) -> lists:member(C, Ciphers) end, ?TLSV13_EXCLUSIVE_CIPHERS).
-endif.
-endif.

11
apps/emqx/test/emqx_schema_tests.erl
View File

@ -21,8 +21,7 @@
ssl_opts_dtls_test() -> ssl_opts_dtls_test() ->
Sc = emqx_schema:server_ssl_opts_schema( Sc = emqx_schema:server_ssl_opts_schema(
#{ #{
versions => dtls_all_available, versions => dtls_all_available
ciphers => dtls_all_available
}, },
false false
), ),
@ -30,7 +29,7 @@ ssl_opts_dtls_test() ->
?assertMatch( ?assertMatch(
#{ #{
versions := ['dtlsv1.2', 'dtlsv1'], versions := ['dtlsv1.2', 'dtlsv1'],
ciphers := ["ECDHE-ECDSA-AES256-GCM-SHA384" | _] ciphers := []
}, },
Checked Checked
). ).
@ -42,7 +41,7 @@ ssl_opts_tls_1_3_test() ->
?assertMatch( ?assertMatch(
#{ #{
versions := ['tlsv1.3'], versions := ['tlsv1.3'],
ciphers := [_ | _] ciphers := []
}, },
Checked Checked
). ).
@ -53,7 +52,7 @@ ssl_opts_tls_for_ranch_test() ->
?assertMatch( ?assertMatch(
#{ #{
versions := ['tlsv1.3'], versions := ['tlsv1.3'],
ciphers := [_ | _], ciphers := [],
handshake_timeout := _ handshake_timeout := _
}, },
Checked Checked
@ -125,7 +124,7 @@ validate(Schema, Data0) ->
), ),
Checked. Checked.
ciperhs_schema_test() -> ciphers_schema_test() ->
Sc = emqx_schema:ciphers_schema(undefined), Sc = emqx_schema:ciphers_schema(undefined),
WSc = #{roots => [{ciphers, Sc}]}, WSc = #{roots => [{ciphers, Sc}]},
?assertThrow( ?assertThrow(

6
apps/emqx_gateway/src/emqx_gateway_schema.erl
View File

@ -365,8 +365,7 @@ fields(ssl_server_opts) ->
#{ #{
depth => 10, depth => 10,
reuse_sessions => true, reuse_sessions => true,
versions => tls_all_available, versions => tls_all_available
ciphers => tls_all_available
}, },
true true
); );
@ -502,8 +501,7 @@ fields(dtls_opts) ->
#{ #{
depth => 10, depth => 10,
reuse_sessions => true, reuse_sessions => true,
versions => dtls_all_available, versions => dtls_all_available
ciphers => dtls_all_available
}, },
false false
). ).

9
apps/emqx_gateway/src/emqx_gateway_utils.erl
View File

@ -455,14 +455,7 @@ esockd_access_rules(StrRules) ->
[Access(R) || R <- StrRules]. [Access(R) || R <- StrRules].
ssl_opts(Name, Opts) -> ssl_opts(Name, Opts) ->
maps:to_list( emqx_tls_lib:to_server_opts(maps:get(Name, Opts, #{})).
emqx_tls_lib:drop_tls13_for_old_otp(
maps:without(
[enable],
maps:get(Name, Opts, #{})
)
)
).
sock_opts(Name, Opts) -> sock_opts(Name, Opts) ->
maps:to_list( maps:to_list(

1
rel/emqx_conf.template.en.md
View File

@ -291,6 +291,7 @@ ciphers =
"ECDH-ECDSA-AES128-SHA", "ECDH-ECDSA-AES128-SHA",
"ECDH-RSA-AES128-SHA" "ECDH-RSA-AES128-SHA"
] ]
```
For PSK enabled listeners For PSK enabled listeners

1
rel/emqx_conf.template.zh.md
View File

@ -272,6 +272,7 @@ ciphers =
"ECDH-ECDSA-AES128-SHA", "ECDH-ECDSA-AES128-SHA",
"ECDH-RSA-AES128-SHA" "ECDH-RSA-AES128-SHA"
] ]
```
配置 PSK 认证的监听器 配置 PSK 认证的监听器