diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 9b049f74f..d6ca233f7 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -34,6 +34,10 @@ file_content_as_options/1 ]). +-export([ + to_client_opts/1 +]). + -include("logger.hrl"). -define(IS_TRUE(Val), ((Val =:= true) or (Val =:= <<"true">>))). @@ -426,6 +430,51 @@ file_content_as_options([Key | Keys], SSL) -> end end. +%% @doc Convert hocon-checked ssl client options (map()) to +%% proplist accepted by ssl library. +to_client_opts(Opts) -> + GetD = fun(Key, Default) -> fuzzy_map_get(Key, Opts, Default) end, + Get = fun(Key) -> GetD(Key, undefined) end, + KeyFile = ensure_str(Get(keyfile)), + CertFile = ensure_str(Get(certfile)), + CAFile = ensure_str(Get(cacertfile)), + Verify = GetD(verify, verify_none), + SNI = + case GetD(server_name_indication, undefined) of + undefined -> undefined; + SNI0 -> ensure_str(SNI0) + end, + Versions = integral_versions(Get(versions)), + Ciphers = integral_ciphers(Versions, Get(ciphers)), + filter([ + {keyfile, KeyFile}, + {certfile, CertFile}, + {cacertfile, CAFile}, + {verify, Verify}, + {server_name_indication, SNI}, + {versions, Versions}, + {ciphers, Ciphers} + ]). + +filter([]) -> []; +filter([{_, undefined} | T]) -> filter(T); +filter([{_, ""} | T]) -> filter(T); +filter([H | T]) -> [H | filter(T)]. + +-spec fuzzy_map_get(atom() | binary(), map(), any()) -> any(). +fuzzy_map_get(Key, Options, Default) -> + case maps:find(Key, Options) of + {ok, Val} -> + Val; + error when is_atom(Key) -> + fuzzy_map_get(atom_to_binary(Key, utf8), Options, Default); + error -> + Default + end. + +ensure_str(L) when is_list(L) -> L; +ensure_str(B) when is_binary(B) -> unicode:characters_to_list(B, utf8). + -if(?OTP_RELEASE > 22). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index a63ed78bb..8bf3e8ee5 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -145,14 +145,14 @@ on_start(InstId, #{base_url := #{scheme := Scheme, pool_size := PoolSize} = Config) -> ?SLOG(info, #{msg => "starting_http_connector", connector => InstId, config => Config}), - {Transport, TransportOpts} = case Scheme of - http -> - {tcp, []}; - https -> - SSLOpts = emqx_plugin_libs_ssl:save_files_return_opts( - maps:get(ssl, Config), "connectors", InstId), - {tls, SSLOpts} - end, + {Transport, TransportOpts} = + case Scheme of + http -> + {tcp, []}; + https -> + SSLOpts = emqx_tls_lib:to_client_opts(maps:get(ssl, Config)), + {tls, SSLOpts} + end, NTransportOpts = emqx_misc:ipv6_probe(TransportOpts), PoolOpts = [ {host, Host} , {port, Port} diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl index 2416dab7b..3e3bc36ff 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -57,10 +57,7 @@ on_start(InstId, #{servers := Servers0, SslOpts = case maps:get(enable, SSL) of true -> [{ssl, true}, - {sslopts, emqx_plugin_libs_ssl:save_files_return_opts( - SSL, - "connectors", - InstId)} + {sslopts, emqx_tls_lib:to_client_opts(SSL)} ]; false -> [{ssl, false}] end, diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 90945e5b8..35cbfd3c9 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -120,11 +120,7 @@ on_start(InstId, Config = #{mongo_type := Type, SslOpts = case maps:get(enable, SSL) of true -> [{ssl, true}, - {ssl_opts, - emqx_plugin_libs_ssl:save_files_return_opts( - SSL, - "connectors", - InstId)} + {ssl_opts, emqx_tls_lib:to_client_opts(SSL)} ]; false -> [{ssl, false}] end, diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 8fbed74fb..2e939760c 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -67,10 +67,11 @@ on_start(InstId, #{server := {Host, Port}, ?SLOG(info, #{msg => "starting_mysql_connector", connector => InstId, config => Config}), SslOpts = case maps:get(enable, SSL) of - true -> - [{ssl, emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)}]; - false -> [] - end, + true -> + [{ssl, emqx_tls_lib:to_client_opts(SSL)}]; + false -> + [] + end, Options = [{host, Host}, {port, Port}, {user, User}, diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 77509b276..7e556ad81 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -79,8 +79,7 @@ on_start(InstId, #{server := {Host, Port}, SslOpts = case maps:get(enable, SSL) of true -> [{ssl, true}, - {ssl_opts, - emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)}]; + {ssl_opts, emqx_tls_lib:to_client_opts(SSL)}]; false -> [{ssl, false}] end, diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index e8bcc86ac..b237ed6a4 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -109,9 +109,7 @@ on_start(InstId, #{redis_type := Type, Options = case maps:get(enable, SSL) of true -> [{ssl, true}, - {ssl_options, - emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)} - ]; + {ssl_options, emqx_tls_lib:to_client_opts(SSL)}]; false -> [{ssl, false}] end ++ [{sentinel, maps:get(sentinel, Config, undefined)}], PoolName = emqx_plugin_libs_pool:pool_name(InstId), diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl deleted file mode 100644 index 9d1c48ba9..000000000 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl +++ /dev/null @@ -1,177 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_plugin_libs_ssl). - --export([ - save_files_return_opts/2, - save_files_return_opts/3, - save_file/2 -]). - --type file_input_key() :: atom() | binary(). --type file_input() :: #{file_input_key() => binary()}. - -%% options are below paris -%% <<"keyfile">> => file_input() -%% <<"certfile">> => file_input() -%% <<"cafile">> => file_input() %% backward compatible -%% <<"cacertfile">> => file_input() -%% <<"verify">> => verify_none | verify_peer -%% <<"tls_versions">> => binary() -%% <<"ciphers">> => binary() --type opts_key() :: binary() | atom(). --type opts_input() :: #{opts_key() => term()}. - --type opt_key() :: keyfile | certfile | cacertfile | verify | versions | ciphers. --type opt_value() :: term(). --type opts() :: [{opt_key(), opt_value()}]. - --include_lib("emqx/include/logger.hrl"). - -%% @doc Parse ssl options input. -%% If the input contains file content, save the files in the given dir. -%% Returns ssl options for Erlang's ssl application. -%% -%% For SSL files in the input Option, it can either be a file path -%% or a map like `#{filename := FileName, file := Content}`. -%% In case it's a map, the file is saved in EMQX's `data_dir' -%% (unless `SubDir' is an absolute path). -%% NOTE: This function is now deprecated, use emqx_tls_lib:ensure_ssl_files/2 instead. --spec save_files_return_opts( - opts_input(), - atom() | string() | binary(), - string() | binary() -) -> opts(). -save_files_return_opts(Options, SubDir, ResId) -> - Dir = filename:join([emqx:data_dir(), SubDir, ResId]), - save_files_return_opts(Options, Dir). - -%% @doc Parse ssl options input. -%% If the input contains file content, save the files in the given dir. -%% Returns ssl options for Erlang's ssl application. -%% -%% For SSL files in the input Option, it can either be a file path -%% or a map like `#{filename := FileName, file := Content}`. -%% In case it's a map, the file is saved in EMQX's `data_dir' -%% (unless `SubDir' is an absolute path). -%% NOTE: This function is now deprecated, use emqx_tls_lib:ensure_ssl_files/2 instead. --spec save_files_return_opts(opts_input(), file:name_all()) -> opts(). -save_files_return_opts(Options, Dir) -> - GetD = fun(Key, Default) -> fuzzy_map_get(Key, Options, Default) end, - Get = fun(Key) -> GetD(Key, undefined) end, - KeyFile = Get(keyfile), - CertFile = Get(certfile), - CAFile = Get(cacertfile), - Key = maybe_save_file(KeyFile, Dir), - Cert = maybe_save_file(CertFile, Dir), - CA = maybe_save_file(CAFile, Dir), - Verify = GetD(verify, verify_none), - SNI = - case Get(<<"server_name_indication">>) of - undefined -> undefined; - SNI0 -> ensure_str(SNI0) - end, - Versions = emqx_tls_lib:integral_versions(Get(versions)), - Ciphers = emqx_tls_lib:integral_ciphers(Versions, Get(ciphers)), - filter([ - {keyfile, Key}, - {certfile, Cert}, - {cacertfile, CA}, - {verify, Verify}, - {server_name_indication, SNI}, - {versions, Versions}, - {ciphers, Ciphers} - ]). - -%% @doc Save a key or certificate file in data dir, -%% and return path of the saved file. -%% empty string is returned if the input is empty. --spec save_file(file_input(), atom() | string() | binary()) -> string(). -save_file(Param, SubDir) -> - Dir = filename:join([emqx:data_dir(), SubDir]), - maybe_save_file(Param, Dir). - -filter([]) -> []; -filter([{_, undefined} | T]) -> filter(T); -filter([{_, ""} | T]) -> filter(T); -filter([H | T]) -> [H | filter(T)]. - -maybe_save_file(#{filename := FileName, file := Content}, Dir) when - FileName =/= undefined andalso Content =/= undefined --> - maybe_save_file(ensure_str(FileName), iolist_to_binary(Content), Dir); -maybe_save_file(FilePath, _) when is_list(FilePath) -> - FilePath; -maybe_save_file(FilePath, _) when is_binary(FilePath) -> - ensure_str(FilePath); -maybe_save_file(_, _) -> - "". - -%% no filename, ignore -maybe_save_file("", _, _Dir) -> - ""; -%% no content, see if file exists -maybe_save_file(FileName, <<>>, Dir) -> - {ok, Cwd} = file:get_cwd(), - %% NOTE: when FileName is an absolute path, filename:join has no effect - CwdFile = ensure_str(filename:join([Cwd, FileName])), - DataDirFile = ensure_str(filename:join([Dir, FileName])), - Possibles = - case CwdFile =:= DataDirFile of - true -> [CwdFile]; - false -> [CwdFile, DataDirFile] - end, - case find_exist_file(FileName, Possibles) of - false -> erlang:throw({bad_cert_file, Possibles}); - Found -> Found - end; -maybe_save_file(FileName, Content, Dir) -> - FullFilename = filename:join([Dir, FileName]), - ok = filelib:ensure_dir(FullFilename), - case file:write_file(FullFilename, Content) of - ok -> - ensure_str(FullFilename); - {error, Reason} -> - ?SLOG(error, #{ - msg => "failed_to_save_ssl_file", - filename => FullFilename, - reason => Reason - }), - error({"failed_to_save_ssl_file", FullFilename, Reason}) - end. - -ensure_str(L) when is_list(L) -> L; -ensure_str(B) when is_binary(B) -> unicode:characters_to_list(B, utf8). - --spec fuzzy_map_get(atom() | binary(), map(), any()) -> any(). -fuzzy_map_get(Key, Options, Default) -> - case maps:find(Key, Options) of - {ok, Val} -> - Val; - error when is_atom(Key) -> - fuzzy_map_get(atom_to_binary(Key, utf8), Options, Default); - error -> - Default - end. - -find_exist_file(_Name, []) -> - false; -find_exist_file(Name, [F | Rest]) -> - case filelib:is_regular(F) of - true -> F; - false -> find_exist_file(Name, Rest) - end. diff --git a/apps/emqx_plugin_libs/test/emqx_plugin_libs_ssl_tests.erl b/apps/emqx_plugin_libs/test/emqx_plugin_libs_ssl_tests.erl deleted file mode 100644 index afac68166..000000000 --- a/apps/emqx_plugin_libs/test/emqx_plugin_libs_ssl_tests.erl +++ /dev/null @@ -1,86 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2022 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_plugin_libs_ssl_tests). - --include_lib("proper/include/proper.hrl"). --include_lib("eunit/include/eunit.hrl"). - -no_crash_test_() -> - Opts = [{numtests, 1000}, {to_file, user}], - {timeout, 60, - fun() -> ?assert(proper:quickcheck(prop_run(), Opts)) end}. - -prop_run() -> - ?FORALL(Generated, prop_opts_input(), test_opts_input(Generated)). - -%% proper type to generate input value. -prop_opts_input() -> - [{keyfile, prop_file_or_content()}, - {certfile, prop_file_or_content()}, - {cacertfile, prop_file_or_content()}, - {verify, proper_types:oneof([verify_none, verify_peer])}, - {versions, prop_tls_versions()}, - {ciphers, prop_tls_ciphers()}, - {other, proper_types:binary()}]. - -prop_file_or_content() -> - proper_types:oneof([prop_cert_file_name(), - {prop_cert_file_name(), proper_types:binary()}]). - -prop_cert_file_name() -> - File = code:which(?MODULE), %% existing - proper_types:oneof(["", <<>>, undefined, File]). - -prop_tls_versions() -> - proper_types:oneof(["tlsv1.3", - <<"tlsv1.3,tlsv1.2">>, - "tlsv1.2 , tlsv1.1", - "1.2", - "v1.3", - "", - <<>>, - undefined]). - -prop_tls_ciphers() -> - proper_types:oneof(["TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256", - <<>>, - "", - undefined]). - -test_opts_input(Inputs) -> - KF = fun(K) -> {_, V} = lists:keyfind(K, 1, Inputs), V end, - Generated = #{<<"keyfile">> => file_or_content(KF(keyfile)), - <<"certfile">> => file_or_content(KF(certfile)), - <<"cafile">> => file_or_content(KF(cacertfile)), - <<"verify">> => file_or_content(KF(verify)), - <<"tls_versions">> => KF(versions), - <<"ciphers">> => KF(ciphers), - <<"other">> => KF(other)}, - _ = emqx_plugin_libs_ssl:save_files_return_opts(Generated, "test-data"), - true. - -file_or_content({Name, Content}) -> - #{<<"file">> => Content, <<"filename">> => Name}; -file_or_content(Name) -> - Name. - -bad_cert_file_test() -> - Input = #{<<"keyfile">> => - #{<<"filename">> => "notafile", - <<"file">> => ""}}, - ?assertThrow({bad_cert_file, _}, - emqx_plugin_libs_ssl:save_files_return_opts(Input, "test-data")).