From c3b980fde916b2fb3ca27f97a22fcd1764d7ad76 Mon Sep 17 00:00:00 2001 From: Spycsh <757407490@qq.com> Date: Fri, 3 Sep 2021 17:25:05 +0800 Subject: [PATCH 1/2] chore: fix Windows compilation process --- .gitattributes | 1 + .gitignore | 2 ++ apps/emqx_authz/src/emqx_authz_schema.erl | 5 --- apps/emqx_machine/src/emqx_machine.erl | 8 +++-- bin/emqx.cmd | 38 +++++++++++++++-------- 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/.gitattributes b/.gitattributes index 4ed73da9a..434addf38 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,6 @@ * text=auto *.* text eol=lf +*.cmd text eol=crlf *.jpg -text *.png -text *.pdf -text diff --git a/.gitignore b/.gitignore index 57be83882..4188126da 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,5 @@ _upgrade_base/ TAGS erlang_ls.config .els_cache/ +.vs/ +.vscode/ diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index af1c59fdc..50c2ec3ff 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -48,11 +48,6 @@ fields(file) -> , {enable, #{type => boolean(), default => true}} , {path, #{type => string(), - validator => fun(S) -> case filelib:is_file(S) of - true -> ok; - _ -> {error, "File does not exist"} - end - end, desc => "Path to the file which contains the ACL rules." }} ]; diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index 6ea493aa2..5a717c5e7 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -35,8 +35,12 @@ %% @doc EMQ X boot entrypoint. start() -> - os:set_signal(sighup, ignore), - os:set_signal(sigterm, handle), %% default is handle + case os:type() of + {win32, nt} -> ok; + _nix -> + os:set_signal(sighup, ignore), + os:set_signal(sigterm, handle) %% default is handle + end, ok = set_backtrace_depth(), ok = print_otp_version_warning(), diff --git a/bin/emqx.cmd b/bin/emqx.cmd index 768e30d2c..fe0d474c9 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -22,14 +22,19 @@ @set script=%~n0 +@set EPMD_ARG=-start_epmd false -epmd_module ekka_epmd -proto_dist ekka +@set ERL_FLAGS=%EPMD_ARG% + :: Discover the release root directory from the directory :: of this script @set script_dir=%~dp0 @for %%A in ("%script_dir%\..") do @( set rel_root_dir=%%~fA ) + @set rel_dir=%rel_root_dir%\releases\%rel_vsn% @set RUNNER_ROOT_DIR=%rel_root_dir% +@set RUNNER_ETC_DIR=%rel_root_dir%\etc @set etc_dir=%rel_root_dir%\etc @set lib_dir=%rel_root_dir%\lib @@ -46,22 +51,22 @@ @set progname=erl.exe @set clean_boot_script=%rel_root_dir%\bin\start_clean @set erlsrv="%bindir%\erlsrv.exe" -@set epmd="%bindir%\epmd.exe" @set escript="%bindir%\escript.exe" @set werl="%bindir%\werl.exe" @set erl_exe="%bindir%\erl.exe" @set nodetool="%rel_root_dir%\bin\nodetool" @set cuttlefish="%rel_root_dir%\bin\cuttlefish" @set node_type="-name" +@set schema_mod="emqx_machine_schema" :: Extract node name from emqx.conf -@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.name "%emqx_conf%"`) do @( +@for /f "usebackq delims=" %%I in (`"%escript% %nodetool% hocon -s %schema_mod% -c %etc_dir%\emqx.conf get node.name"`) do @( @call :set_trim node_name %%I ) :: Extract node cookie from emqx.conf -@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.cookie "%emqx_conf%"`) do @( - @call :set_trim node_cookie= %%I +@for /f "usebackq delims=" %%I in (`"%escript% %nodetool% hocon -s %schema_mod% -c %etc_dir%\emqx.conf get node.cookie"`) do @( + @call :set_trim node_cookie %%I ) :: Write the erl.ini file to set up paths relative to this script @@ -139,13 +144,23 @@ ) @goto :eof -:generate_app_config -@set gen_config_cmd=%escript% %cuttlefish% -i %rel_dir%\emqx.schema -c %etc_dir%\emqx.conf -d %data_dir%\configs generate -@for /f "delims=" %%A in ('%%gen_config_cmd%%') do @( - set generated_config_args=%%A +:: get the current time with hocon +:get_cur_time +@for /f "usebackq tokens=1-6 delims=." %%a in (`"%escript% %nodetool% hocon now_time"`) do @( + set now_time=%%a.%%b.%%c.%%d.%%e.%%f ) @goto :eof +:generate_app_config +@call :get_cur_time +%escript% %nodetool% hocon -v -t %now_time% -s %schema_mod% -c "%etc_dir%\emqx.conf" -d "%data_dir%\configs" generate +@set generated_config_args=-config %data_dir%\configs\app.%now_time%.config -args_file %data_dir%\configs\vm.%now_time%.args +:: create one new line +@echo.>>%data_dir%\configs\vm.%now_time%.args +:: write the node type and node name in to vm args file +@echo %node_type% %node_name%>>%data_dir%\configs\vm.%now_time%.args +@goto :eof + :: set boot_script variable :set_boot_script_var @if exist "%rel_dir%\%rel_name%.boot" ( @@ -188,13 +203,11 @@ :: relup and reldown goto relup ) - @goto :eof :: Uninstall the Windows service :uninstall @%erlsrv% remove %service_name% -@%epmd% -kill @goto :eof :: Start the Windows service @@ -207,7 +220,7 @@ @echo off cd /d %rel_root_dir% @echo on -@start "%rel_name%" %werl% -boot "%boot_script%" %args% +@start "%rel_name%" %werl% -boot "%boot_script%" -mode embedded %args% @goto :eof :: Stop the Windows service @@ -237,7 +250,7 @@ cd /d %rel_root_dir% @echo off cd /d %rel_root_dir% @echo on -@start "bin\%rel_name% console" %werl% -boot "%boot_script%" %args% +@start "bin\%rel_name% console" %werl% -boot "%boot_script%" -mode embedded %args% @echo emqx is started! @goto :eof @@ -262,4 +275,3 @@ cd /d %rel_root_dir% :set_trim @set %1=%2 @goto :eof - From 697a11ded09a97dd3dbde5e13110ac92887b8db2 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Mon, 27 Sep 2021 20:40:01 +0200 Subject: [PATCH 2/2] fix(tls): drop unsupported ciphers we use a hard-coded list of pre-selected ciphers as config default value. some of them may not be supported by the underlying openssl lib. now moved the pre-selected ciphers to emqx_tls_lib:selected_ciphers which performs a filtering before return. --- apps/emqx/src/emqx_schema.erl | 34 ++-------------- apps/emqx/src/emqx_tls_lib.erl | 58 ++++++++++++++++++++++------ apps/emqx/test/emqx_schema_tests.erl | 7 +--- 3 files changed, 51 insertions(+), 48 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 344a1aa45..202cd4ae9 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1156,39 +1156,11 @@ default_ciphers(quic) -> [ "TLS_AES_128_GCM_SHA256", "TLS_CHACHA20_POLY1305_SHA256" ]; -default_ciphers(tls_all_available) -> - default_ciphers('tlsv1.3') ++ - default_ciphers('tlsv1.2') ++ - default_ciphers(psk); default_ciphers(dtls_all_available) -> %% as of now, dtls does not support tlsv1.3 ciphers - default_ciphers('tlsv1.2') ++ default_ciphers('psk'); -default_ciphers('tlsv1.3') -> - case is_tlsv13_available() of - true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); - false -> [] - end ++ default_ciphers('tlsv1.2'); -default_ciphers('tlsv1.2') -> [ - "ECDHE-ECDSA-AES256-GCM-SHA384", - "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384", - "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384", - "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384", - "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256", - "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256", - "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256", - "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256", - "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256", - "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA", - "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA", - "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA", - "ECDH-RSA-AES128-SHA", "AES128-SHA" - ]; -default_ciphers(psk) -> - [ "RSA-PSK-AES256-GCM-SHA384","RSA-PSK-AES256-CBC-SHA384", - "RSA-PSK-AES128-GCM-SHA256","RSA-PSK-AES128-CBC-SHA256", - "RSA-PSK-AES256-CBC-SHA","RSA-PSK-AES128-CBC-SHA", - "RSA-PSK-DES-CBC3-SHA","RSA-PSK-RC4-SHA" - ]. + emqx_tls_lib:selected_ciphers(['dtlsv1.2', 'dtlsv1']); +default_ciphers(tls_all_available) -> + emqx_tls_lib:default_ciphers(). %% @private return a list of keys in a parent field -spec(keys(string(), hocon:config()) -> [string()]). diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 683166e87..11145f684 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -19,7 +19,7 @@ -export([ default_versions/0 , integral_versions/1 , default_ciphers/0 - , default_ciphers/1 + , selected_ciphers/1 , integral_ciphers/2 , drop_tls13_for_old_otp/1 ]). @@ -59,27 +59,61 @@ integral_versions(Desired) -> Filtered end. -%% @doc Return a list of default (openssl string format) cipher suites. --spec default_ciphers() -> [string()]. -default_ciphers() -> default_ciphers(default_versions()). - %% @doc Return a list of (openssl string format) cipher suites. --spec default_ciphers([ssl:tls_version()]) -> [string()]. -default_ciphers(['tlsv1.3']) -> +-spec all_ciphers([ssl:tls_version()]) -> [string()]. +all_ciphers(['tlsv1.3']) -> %% When it's only tlsv1.3 wanted, use 'exclusive' here %% because 'all' returns legacy cipher suites too, %% which does not make sense since tlsv1.3 can not use %% legacy cipher suites. ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); -default_ciphers(Versions) -> +all_ciphers(Versions) -> %% assert non-empty [_ | _] = dedup(lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions])). + +%% @doc All Pre-selected TLS ciphers. +default_ciphers() -> + selected_ciphers(available_versions()). + +%% @doc Pre-selected TLS ciphers for given versions.. +selected_ciphers(Vsns) -> + All = all_ciphers(Vsns), + dedup(lists:filter(fun(Cipher) -> lists:member(Cipher, All) end, + lists:flatmap(fun do_selected_ciphers/1, Vsns))). + +do_selected_ciphers('tlsv1.3') -> + case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of + true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); + false -> [] + end ++ do_selected_ciphers('tlsv1.2'); +do_selected_ciphers(_) -> + [ "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384", + "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384", + "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384", + "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256", + "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256", + "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256", + "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA", + "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA", + "ECDH-RSA-AES128-SHA", "AES128-SHA", + + %% psk + "RSA-PSK-AES256-GCM-SHA384","RSA-PSK-AES256-CBC-SHA384", + "RSA-PSK-AES128-GCM-SHA256","RSA-PSK-AES128-CBC-SHA256", + "RSA-PSK-AES256-CBC-SHA","RSA-PSK-AES128-CBC-SHA", + "RSA-PSK-DES-CBC3-SHA","RSA-PSK-RC4-SHA" + ]. + %% @doc Ensure version & cipher-suites integrity. -spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()]. integral_ciphers(Versions, Ciphers) when Ciphers =:= [] orelse Ciphers =:= undefined -> %% not configured - integral_ciphers(Versions, default_ciphers(Versions)); + integral_ciphers(Versions, selected_ciphers(Versions)); integral_ciphers(Versions, Ciphers) when ?IS_STRING_LIST(Ciphers) -> %% ensure tlsv1.3 ciphers if none of them is found in Ciphers dedup(ensure_tls13_cipher(lists:member('tlsv1.3', Versions), Ciphers)); @@ -93,7 +127,7 @@ integral_ciphers(Versions, Ciphers) -> %% In case tlsv1.3 is present, ensure tlsv1.3 cipher is added if user %% did not provide it from config --- which is a common mistake ensure_tls13_cipher(true, Ciphers) -> - Tls13Ciphers = default_ciphers(['tlsv1.3']), + Tls13Ciphers = selected_ciphers(['tlsv1.3']), case lists:any(fun(C) -> lists:member(C, Tls13Ciphers) end, Ciphers) of true -> Ciphers; false -> Tls13Ciphers ++ Ciphers @@ -179,10 +213,12 @@ drop_tls13(SslOpts0) -> -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). +all_ciphers() -> all_ciphers(default_versions()). + drop_tls13_test() -> Versions = default_versions(), ?assert(lists:member('tlsv1.3', Versions)), - Ciphers = default_ciphers(), + Ciphers = all_ciphers(), ?assert(has_tlsv13_cipher(Ciphers)), Opts0 = #{versions => Versions, ciphers => Ciphers, other => true}, Opts = drop_tls13(Opts0), diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl index 4585089e2..e2825498d 100644 --- a/apps/emqx/test/emqx_schema_tests.erl +++ b/apps/emqx/test/emqx_schema_tests.erl @@ -62,12 +62,7 @@ ssl_opts_cipher_comma_separated_string_test() -> ssl_opts_tls_psk_test() -> Sc = emqx_schema:server_ssl_opts_schema(#{}, false), Checked = validate(Sc, #{<<"versions">> => [<<"tlsv1.2">>]}), - ?assertMatch(#{versions := ['tlsv1.2']}, Checked), - #{ciphers := Ciphers} = Checked, - PskCiphers = emqx_schema:default_ciphers(psk), - lists:foreach(fun(Cipher) -> - ?assert(lists:member(Cipher, Ciphers)) - end, PskCiphers). + ?assertMatch(#{versions := ['tlsv1.2']}, Checked). bad_cipher_test() -> Sc = emqx_schema:server_ssl_opts_schema(#{}, false),