refactor: populate ciphers list at runtime

Populating ciphers list when checking schema makes the
config file example and the schmea documents quite bloated

apps/emqx/src/emqx_listeners.erl

@@ -583,11 +583,7 @@ enable_authn(Opts) ->
     maps:get(enable_authn, Opts, true).
 
 ssl_opts(Opts) ->
-    maps:to_list(
+    emqx_tls_lib:to_server_opts(maps:get(ssl_options, Opts, #{})).
-        emqx_tls_lib:drop_tls13_for_old_otp(
-            maps:get(ssl_options, Opts, #{})
-        )
-    ).
 
 tcp_opts(Opts) ->
     maps:to_list(

apps/emqx/src/emqx_schema.erl

@@ -102,7 +102,7 @@
 
 -export([namespace/0, roots/0, roots/1, fields/1, desc/1]).
 -export([conf_get/2, conf_get/3, keys/2, filter/1]).
--export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1, default_ciphers/1]).
+-export([server_ssl_opts_schema/2, client_ssl_opts_schema/1, ciphers_schema/1]).
 -export([sc/2, map/2]).
 
 -elvis([{elvis_style, god_modules, disable}]).
@@ -2060,19 +2060,15 @@ default_ciphers(Which) ->
         do_default_ciphers(Which)
     ).
 
-do_default_ciphers(undefined) ->
-    do_default_ciphers(tls_all_available);
 do_default_ciphers(quic) ->
     [
         "TLS_AES_256_GCM_SHA384",
         "TLS_AES_128_GCM_SHA256",
         "TLS_CHACHA20_POLY1305_SHA256"
     ];
-do_default_ciphers(dtls_all_available) ->
+do_default_ciphers(_) ->
-    %% as of now, dtls does not support tlsv1.3 ciphers
+    %% otherwise resolve default ciphers list at runtime
-    emqx_tls_lib:selected_ciphers(['dtlsv1.2', 'dtlsv1']);
+    [].
-do_default_ciphers(tls_all_available) ->
-    emqx_tls_lib:default_ciphers().
 
 %% @private return a list of keys in a parent field
 -spec keys(string(), hocon:config()) -> [string()].
@@ -2246,8 +2242,8 @@ parse_user_lookup_fun(StrConf) ->
     {fun Mod:Fun/3, undefined}.
 
 validate_ciphers(Ciphers) ->
-    All = emqx_tls_lib:all_ciphers(),
+    Set = emqx_tls_lib:all_ciphers_set_cached(),
-    case lists:filter(fun(Cipher) -> not lists:member(Cipher, All) end, Ciphers) of
+    case lists:filter(fun(Cipher) -> not sets:is_element(Cipher, Set) end, Ciphers) of
         [] -> ok;
         Bad -> {error, {bad_ciphers, Bad}}
     end.

apps/emqx/src/emqx_tls_lib.erl

@@ -23,8 +23,7 @@
     default_ciphers/0,
     selected_ciphers/1,
     integral_ciphers/2,
-    drop_tls13_for_old_otp/1,
+    all_ciphers_set_cached/0
-    all_ciphers/0
 ]).
 
 %% SSL files
@@ -38,6 +37,7 @@
 ]).
 
 -export([
+    to_server_opts/1,
     to_client_opts/1
 ]).
 
@@ -54,83 +54,19 @@
 %% non-empty list of strings
 -define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso ?IS_STRING(hd(L)))).
 
-%% @doc Returns the default supported tls versions.
+%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl)
--spec default_versions() -> [atom()].
+%% should return when running on otp 23.
-default_versions() -> available_versions().
+%% But we still have to hard-code them because tlsv1.3 on otp 22 is
+%% not trustworthy.
+-define(TLSV13_EXCLUSIVE_CIPHERS, [
+    "TLS_AES_256_GCM_SHA384",
+    "TLS_AES_128_GCM_SHA256",
+    "TLS_CHACHA20_POLY1305_SHA256",
+    "TLS_AES_128_CCM_SHA256",
+    "TLS_AES_128_CCM_8_SHA256"
+]).
 
-%% @doc Validate a given list of desired tls versions.
+-define(SELECTED_CIPHERS, [
-%% raise an error exception if non of them are available.
-%% The input list can be a string/binary of comma separated versions.
--spec integral_versions(undefined | string() | binary() | [ssl:tls_version()]) ->
-    [ssl:tls_version()].
-integral_versions(undefined) ->
-    integral_versions(default_versions());
-integral_versions([]) ->
-    integral_versions(default_versions());
-integral_versions(<<>>) ->
-    integral_versions(default_versions());
-integral_versions(Desired) when ?IS_STRING(Desired) ->
-    integral_versions(iolist_to_binary(Desired));
-integral_versions(Desired) when is_binary(Desired) ->
-    integral_versions(parse_versions(Desired));
-integral_versions(Desired) ->
-    Available = available_versions(),
-    case lists:filter(fun(V) -> lists:member(V, Available) end, Desired) of
-        [] ->
-            erlang:error(#{
-                reason => no_available_tls_version,
-                desired => Desired,
-                available => Available
-            });
-        Filtered ->
-            Filtered
-    end.
-
-%% @doc Return a list of all supported ciphers.
-all_ciphers() -> all_ciphers(default_versions()).
-
-%% @doc Return a list of (openssl string format) cipher suites.
--spec all_ciphers([ssl:tls_version()]) -> [string()].
-all_ciphers(['tlsv1.3']) ->
-    %% When it's only tlsv1.3 wanted, use 'exclusive' here
-    %% because 'all' returns legacy cipher suites too,
-    %% which does not make sense since tlsv1.3 can not use
-    %% legacy cipher suites.
-    ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
-all_ciphers(Versions) ->
-    %% assert non-empty
-    List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]),
-    [_ | _] = dedup(List).
-
-%% @doc All Pre-selected TLS ciphers.
-%% ssl:cipher_suites(all, V, openssl) is too slow. so we cache default ciphers.
-default_ciphers() ->
-    case persistent_term:get(default_ciphers, undefined) of
-        undefined ->
-            Default = selected_ciphers(available_versions()),
-            persistent_term:put(default_ciphers, Default),
-            Default;
-        Default ->
-            Default
-    end.
-
-%% @doc Pre-selected TLS ciphers for given versions..
-selected_ciphers(Vsns) ->
-    All = all_ciphers(Vsns),
-    dedup(
-        lists:filter(
-            fun(Cipher) -> lists:member(Cipher, All) end,
-            lists:flatmap(fun do_selected_ciphers/1, Vsns)
-        )
-    ).
-
-do_selected_ciphers('tlsv1.3') ->
-    case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of
-        true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
-        false -> []
-    end ++ do_selected_ciphers('tlsv1.2');
-do_selected_ciphers(_) ->
-    [
     "ECDHE-ECDSA-AES256-GCM-SHA384",
     "ECDHE-RSA-AES256-GCM-SHA384",
     "ECDHE-ECDSA-AES256-SHA384",
@@ -173,7 +109,88 @@ do_selected_ciphers(_) ->
     "RSA-PSK-AES128-CBC-SHA256",
     "RSA-PSK-AES256-CBC-SHA",
     "RSA-PSK-AES128-CBC-SHA"
-    ].
+]).
+
+%% @doc Returns the default supported tls versions.
+-spec default_versions() -> [atom()].
+default_versions() -> available_versions().
+
+%% @doc Validate a given list of desired tls versions.
+%% raise an error exception if non of them are available.
+%% The input list can be a string/binary of comma separated versions.
+-spec integral_versions(undefined | string() | binary() | [ssl:tls_version()]) ->
+    [ssl:tls_version()].
+integral_versions(undefined) ->
+    integral_versions(default_versions());
+integral_versions([]) ->
+    integral_versions(default_versions());
+integral_versions(<<>>) ->
+    integral_versions(default_versions());
+integral_versions(Desired) when ?IS_STRING(Desired) ->
+    integral_versions(iolist_to_binary(Desired));
+integral_versions(Desired) when is_binary(Desired) ->
+    integral_versions(parse_versions(Desired));
+integral_versions(Desired) ->
+    Available = available_versions(),
+    case lists:filter(fun(V) -> lists:member(V, Available) end, Desired) of
+        [] ->
+            erlang:error(#{
+                reason => no_available_tls_version,
+                desired => Desired,
+                available => Available
+            });
+        Filtered ->
+            Filtered
+    end.
+
+%% @doc Return a set of all ciphers
+all_ciphers_set_cached() ->
+    case persistent_term:get(?FUNCTION_NAME, false) of
+        false ->
+            S = sets:from_list(all_ciphers()),
+            persistent_term:put(?FUNCTION_NAME, S);
+        Set ->
+            Set
+    end.
+
+%% @doc Return a list of all supported ciphers.
+all_ciphers() ->
+    all_ciphers(default_versions()).
+
+%% @doc Return a list of (openssl string format) cipher suites.
+-spec all_ciphers([ssl:tls_version()]) -> [string()].
+all_ciphers(['tlsv1.3']) ->
+    %% When it's only tlsv1.3 wanted, use 'exclusive' here
+    %% because 'all' returns legacy cipher suites too,
+    %% which does not make sense since tlsv1.3 can not use
+    %% legacy cipher suites.
+    ?TLSV13_EXCLUSIVE_CIPHERS;
+all_ciphers(Versions) ->
+    %% assert non-empty
+    List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]),
+    [_ | _] = dedup(List).
+
+%% @doc All Pre-selected TLS ciphers.
+default_ciphers() ->
+    selected_ciphers(available_versions()).
+
+%% @doc Pre-selected TLS ciphers for given versions..
+selected_ciphers(Vsns) ->
+    All = all_ciphers(Vsns),
+    dedup(
+        lists:filter(
+            fun(Cipher) -> lists:member(Cipher, All) end,
+            lists:flatmap(fun do_selected_ciphers/1, Vsns)
+        )
+    ).
+
+do_selected_ciphers('tlsv1.3') ->
+    case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of
+        true -> ?TLSV13_EXCLUSIVE_CIPHERS;
+        false -> []
+    end ++ do_selected_ciphers('tlsv1.2');
+do_selected_ciphers(_) ->
+    ?SELECTED_CIPHERS.
 
 %% @doc Ensure version & cipher-suites integrity.
 -spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()].
@@ -209,9 +226,14 @@ available_versions() ->
 
 %% tlsv1.3 is available from OTP-22 but we do not want to use until 23.
 default_versions(OtpRelease) when OtpRelease >= 23 ->
-    proplists:get_value(available, ssl:versions());
+    availables();
 default_versions(_) ->
-    lists:delete('tlsv1.3', proplists:get_value(available, ssl:versions())).
+    lists:delete('tlsv1.3', availables()).
+
+availables() ->
+    All = ssl:versions(),
+    proplists:get_value(available, All) ++
+        proplists:get_value(available_dtls, All).
 
 %% Deduplicate a list without re-ordering the elements.
 dedup([]) ->
@@ -244,6 +266,8 @@ do_parse_versions([V | More], Acc) ->
             do_parse_versions(More, [Parsed | Acc])
     end.
 
+parse_version(<<"dtlsv1.2">>) -> 'dtlsv1.2';
+parse_version(<<"dtlsv1">>) -> dtlsv1;
 parse_version(<<"tlsv", Vsn/binary>>) -> parse_version(Vsn);
 parse_version(<<"v", Vsn/binary>>) -> parse_version(Vsn);
 parse_version(<<"1.3">>) -> 'tlsv1.3';
@@ -259,36 +283,6 @@ split_by_comma(Bin) ->
 trim_space(Bin) ->
     hd([I || I <- binary:split(Bin, <<" ">>), I =/= <<>>]).
 
-%% @doc Drop tlsv1.3 version and ciphers from ssl options
-%% if running on otp 22 or earlier.
-drop_tls13_for_old_otp(SslOpts) ->
-    case list_to_integer(erlang:system_info(otp_release)) < 23 of
-        true -> drop_tls13(SslOpts);
-        false -> SslOpts
-    end.
-
-%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl)
-%% should return when running on otp 23.
-%% But we still have to hard-code them because tlsv1.3 on otp 22 is
-%% not trustworthy.
--define(TLSV13_EXCLUSIVE_CIPHERS, [
-    "TLS_AES_256_GCM_SHA384",
-    "TLS_AES_128_GCM_SHA256",
-    "TLS_CHACHA20_POLY1305_SHA256",
-    "TLS_AES_128_CCM_SHA256",
-    "TLS_AES_128_CCM_8_SHA256"
-]).
-drop_tls13(SslOpts0) ->
-    SslOpts1 =
-        case maps:find(versions, SslOpts0) of
-            error -> SslOpts0;
-            {ok, Vsns} -> SslOpts0#{versions => (Vsns -- ['tlsv1.3'])}
-        end,
-    case maps:find(ciphers, SslOpts1) of
-        error -> SslOpts1;
-        {ok, Ciphers} -> SslOpts1#{ciphers => Ciphers -- ?TLSV13_EXCLUSIVE_CIPHERS}
-    end.
-
 %% @doc The input map is a HOCON decoded result of a struct defined as
 %% emqx_schema:server_ssl_opts_schema. (NOTE: before schema-checked).
 %% `keyfile', `certfile' and `cacertfile' can be either pem format key or certificates,
@@ -498,11 +492,23 @@ do_drop_invalid_certs([Key | Keys], SSL) ->
             end
     end.
 
+%% @doc Convert hocon-checked ssl server options (map()) to
+%% proplist accepted by ssl library.
+to_server_opts(Opts) ->
+    Versions = integral_versions(maps:get(versions, Opts, undefined)),
+    Ciphers = integral_ciphers(Versions, maps:get(ciphers, Opts, undefined)),
+    maps:to_list(Opts#{
+        ciphers => Ciphers,
+        versions => Versions
+    }).
+
 %% @doc Convert hocon-checked ssl client options (map()) to
 %% proplist accepted by ssl library.
 to_client_opts(Opts) ->
     GetD = fun(Key, Default) -> fuzzy_map_get(Key, Opts, Default) end,
     Get = fun(Key) -> GetD(Key, undefined) end,
+    case GetD(enable, false) of
+        true ->
             KeyFile = ensure_str(Get(keyfile)),
             CertFile = ensure_str(Get(certfile)),
             CAFile = ensure_str(Get(cacertfile)),
@@ -517,8 +523,15 @@ to_client_opts(Opts) ->
                 {verify, Verify},
                 {server_name_indication, SNI},
                 {versions, Versions},
-        {ciphers, Ciphers}
+                {ciphers, Ciphers},
-    ]).
+                {reuse_sessions, Get(reuse_sessions)},
+                {depth, Get(depth)},
+                {password, ensure_str(Get(password))},
+                {secure_renegotiate, Get(secure_renegotiate)}
+            ]);
+        false ->
+            []
+    end.
 
 filter([]) -> [];
 filter([{_, undefined} | T]) -> filter(T);
@@ -556,28 +569,3 @@ ensure_ssl_file_key(SSL, RequiredKeys) ->
         [] -> ok;
         Miss -> {error, #{reason => ssl_file_option_not_found, which_options => Miss}}
     end.
-
--if(?OTP_RELEASE > 22).
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
-
-drop_tls13_test() ->
-    Versions = default_versions(),
-    ?assert(lists:member('tlsv1.3', Versions)),
-    Ciphers = all_ciphers(),
-    ?assert(has_tlsv13_cipher(Ciphers)),
-    Opts0 = #{versions => Versions, ciphers => Ciphers, other => true},
-    Opts = drop_tls13(Opts0),
-    ?assertNot(lists:member('tlsv1.3', maps:get(versions, Opts, undefined))),
-    ?assertNot(has_tlsv13_cipher(maps:get(ciphers, Opts, undefined))).
-
-drop_tls13_no_versions_cipers_test() ->
-    Opts0 = #{other => 0, bool => true},
-    Opts = drop_tls13(Opts0),
-    ?_assertEqual(Opts0, Opts).
-
-has_tlsv13_cipher(Ciphers) ->
-    lists:any(fun(C) -> lists:member(C, Ciphers) end, ?TLSV13_EXCLUSIVE_CIPHERS).
-
--endif.
--endif.

apps/emqx/test/emqx_schema_tests.erl

@@ -21,8 +21,7 @@
 ssl_opts_dtls_test() ->
     Sc = emqx_schema:server_ssl_opts_schema(
         #{
-            versions => dtls_all_available,
+            versions => dtls_all_available
-            ciphers => dtls_all_available
         },
         false
     ),
@@ -30,7 +29,7 @@ ssl_opts_dtls_test() ->
     ?assertMatch(
         #{
             versions := ['dtlsv1.2', 'dtlsv1'],
-            ciphers := ["ECDHE-ECDSA-AES256-GCM-SHA384" | _]
+            ciphers := []
         },
         Checked
     ).
@@ -42,7 +41,7 @@ ssl_opts_tls_1_3_test() ->
     ?assertMatch(
         #{
             versions := ['tlsv1.3'],
-            ciphers := [_ | _]
+            ciphers := []
         },
         Checked
     ).
@@ -53,7 +52,7 @@ ssl_opts_tls_for_ranch_test() ->
     ?assertMatch(
         #{
             versions := ['tlsv1.3'],
-            ciphers := [_ | _],
+            ciphers := [],
             handshake_timeout := _
         },
         Checked
@@ -125,7 +124,7 @@ validate(Schema, Data0) ->
         ),
     Checked.
 
-ciperhs_schema_test() ->
+ciphers_schema_test() ->
     Sc = emqx_schema:ciphers_schema(undefined),
     WSc = #{roots => [{ciphers, Sc}]},
     ?assertThrow(

apps/emqx_gateway/src/emqx_gateway_schema.erl

@@ -365,8 +365,7 @@ fields(ssl_server_opts) ->
         #{
             depth => 10,
             reuse_sessions => true,
-            versions => tls_all_available,
+            versions => tls_all_available
-            ciphers => tls_all_available
         },
         true
     );
@@ -502,8 +501,7 @@ fields(dtls_opts) ->
         #{
             depth => 10,
             reuse_sessions => true,
-            versions => dtls_all_available,
+            versions => dtls_all_available
-            ciphers => dtls_all_available
         },
         false
     ).

apps/emqx_gateway/src/emqx_gateway_utils.erl

@@ -455,14 +455,7 @@ esockd_access_rules(StrRules) ->
     [Access(R) || R <- StrRules].
 
 ssl_opts(Name, Opts) ->
-    maps:to_list(
+    emqx_tls_lib:to_server_opts(maps:get(Name, Opts, #{})).
-        emqx_tls_lib:drop_tls13_for_old_otp(
-            maps:without(
-                [enable],
-                maps:get(Name, Opts, #{})
-            )
-        )
-    ).
 
 sock_opts(Name, Opts) ->
     maps:to_list(

rel/emqx_conf.template.en.md

@@ -291,6 +291,7 @@ ciphers =
     "ECDH-ECDSA-AES128-SHA",
     "ECDH-RSA-AES128-SHA"
   ]
+```
 
 For PSK enabled listeners
 

rel/emqx_conf.template.zh.md

@@ -272,6 +272,7 @@ ciphers =
     "ECDH-ECDSA-AES128-SHA",
     "ECDH-RSA-AES128-SHA"
   ]
+```
 
 配置 PSK 认证的监听器