Merge pull request #5822 from Spycsh/fix-windows-compile

chore: fix Windows compilation process

.gitattributes

@@ -1,5 +1,6 @@
 * text=auto
 *.* text eol=lf
+*.cmd text eol=crlf
 *.jpg -text
 *.png -text
 *.pdf -text

.gitignore

@@ -50,3 +50,5 @@ _upgrade_base/
 TAGS
 erlang_ls.config
 .els_cache/
+.vs/
+.vscode/

apps/emqx/src/emqx_schema.erl

@@ -1156,39 +1156,11 @@ default_ciphers(quic) -> [
     "TLS_AES_128_GCM_SHA256",
     "TLS_CHACHA20_POLY1305_SHA256"
     ];
-default_ciphers(tls_all_available) ->
-    default_ciphers('tlsv1.3') ++
-    default_ciphers('tlsv1.2') ++
-    default_ciphers(psk);
 default_ciphers(dtls_all_available) ->
     %% as of now, dtls does not support tlsv1.3 ciphers
-    default_ciphers('tlsv1.2') ++ default_ciphers('psk');
-default_ciphers('tlsv1.3') ->
-    case is_tlsv13_available() of
-        true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
-        false -> []
-    end ++ default_ciphers('tlsv1.2');
-default_ciphers('tlsv1.2') -> [
-    "ECDHE-ECDSA-AES256-GCM-SHA384",
-    "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384",
-    "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384",
-    "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384",
-    "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256",
-    "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256",
-    "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256",
-    "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256",
-    "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256",
-    "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA",
-    "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA",
-    "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA",
-    "ECDH-RSA-AES128-SHA", "AES128-SHA"
-    ];
-default_ciphers(psk) ->
-    [ "RSA-PSK-AES256-GCM-SHA384","RSA-PSK-AES256-CBC-SHA384",
-      "RSA-PSK-AES128-GCM-SHA256","RSA-PSK-AES128-CBC-SHA256",
-      "RSA-PSK-AES256-CBC-SHA","RSA-PSK-AES128-CBC-SHA",
-      "RSA-PSK-DES-CBC3-SHA","RSA-PSK-RC4-SHA"
-    ].
+    emqx_tls_lib:selected_ciphers(['dtlsv1.2', 'dtlsv1']);
+default_ciphers(tls_all_available) ->
+    emqx_tls_lib:default_ciphers().
 
 %% @private return a list of keys in a parent field
 -spec(keys(string(), hocon:config()) -> [string()]).

apps/emqx/src/emqx_tls_lib.erl

@@ -19,7 +19,7 @@
 -export([ default_versions/0
         , integral_versions/1
         , default_ciphers/0
-        , default_ciphers/1
+        , selected_ciphers/1
         , integral_ciphers/2
         , drop_tls13_for_old_otp/1
         ]).
@@ -59,27 +59,61 @@ integral_versions(Desired) ->
             Filtered
     end.
 
-%% @doc Return a list of default (openssl string format) cipher suites.
--spec default_ciphers() -> [string()].
-default_ciphers() -> default_ciphers(default_versions()).
-
 %% @doc Return a list of (openssl string format) cipher suites.
--spec default_ciphers([ssl:tls_version()]) -> [string()].
-default_ciphers(['tlsv1.3']) ->
+-spec all_ciphers([ssl:tls_version()]) -> [string()].
+all_ciphers(['tlsv1.3']) ->
     %% When it's only tlsv1.3 wanted, use 'exclusive' here
     %% because 'all' returns legacy cipher suites too,
     %% which does not make sense since tlsv1.3 can not use
     %% legacy cipher suites.
     ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
-default_ciphers(Versions) ->
+all_ciphers(Versions) ->
     %% assert non-empty
     [_ | _] = dedup(lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions])).
 
+
+%% @doc All Pre-selected TLS ciphers.
+default_ciphers() ->
+    selected_ciphers(available_versions()).
+
+%% @doc Pre-selected TLS ciphers for given versions..
+selected_ciphers(Vsns) ->
+    All = all_ciphers(Vsns),
+    dedup(lists:filter(fun(Cipher) -> lists:member(Cipher, All) end,
+                       lists:flatmap(fun do_selected_ciphers/1, Vsns))).
+
+do_selected_ciphers('tlsv1.3') ->
+    case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of
+        true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
+        false -> []
+    end ++ do_selected_ciphers('tlsv1.2');
+do_selected_ciphers(_) ->
+    [ "ECDHE-ECDSA-AES256-GCM-SHA384",
+      "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384",
+      "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384",
+      "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384",
+      "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256",
+      "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256",
+      "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256",
+      "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256",
+      "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256",
+      "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA",
+      "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA",
+      "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA",
+      "ECDH-RSA-AES128-SHA", "AES128-SHA",
+
+      %% psk
+      "RSA-PSK-AES256-GCM-SHA384","RSA-PSK-AES256-CBC-SHA384",
+      "RSA-PSK-AES128-GCM-SHA256","RSA-PSK-AES128-CBC-SHA256",
+      "RSA-PSK-AES256-CBC-SHA","RSA-PSK-AES128-CBC-SHA",
+      "RSA-PSK-DES-CBC3-SHA","RSA-PSK-RC4-SHA"
+    ].
+
 %% @doc Ensure version & cipher-suites integrity.
 -spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()].
 integral_ciphers(Versions, Ciphers) when Ciphers =:= [] orelse Ciphers =:= undefined ->
     %% not configured
-    integral_ciphers(Versions, default_ciphers(Versions));
+    integral_ciphers(Versions, selected_ciphers(Versions));
 integral_ciphers(Versions, Ciphers) when ?IS_STRING_LIST(Ciphers) ->
     %% ensure tlsv1.3 ciphers if none of them is found in Ciphers
     dedup(ensure_tls13_cipher(lists:member('tlsv1.3', Versions), Ciphers));
@@ -93,7 +127,7 @@ integral_ciphers(Versions, Ciphers) ->
 %% In case tlsv1.3 is present, ensure tlsv1.3 cipher is added if user
 %% did not provide it from config --- which is a common mistake
 ensure_tls13_cipher(true, Ciphers) ->
-    Tls13Ciphers = default_ciphers(['tlsv1.3']),
+    Tls13Ciphers = selected_ciphers(['tlsv1.3']),
     case lists:any(fun(C) -> lists:member(C, Tls13Ciphers) end, Ciphers) of
         true  -> Ciphers;
         false -> Tls13Ciphers ++ Ciphers
@@ -179,10 +213,12 @@ drop_tls13(SslOpts0) ->
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 
+all_ciphers() -> all_ciphers(default_versions()).
+
 drop_tls13_test() ->
     Versions = default_versions(),
     ?assert(lists:member('tlsv1.3', Versions)),
-    Ciphers = default_ciphers(),
+    Ciphers = all_ciphers(),
     ?assert(has_tlsv13_cipher(Ciphers)),
     Opts0 = #{versions => Versions, ciphers => Ciphers, other => true},
     Opts = drop_tls13(Opts0),

apps/emqx/test/emqx_schema_tests.erl

@@ -62,12 +62,7 @@ ssl_opts_cipher_comma_separated_string_test() ->
 ssl_opts_tls_psk_test() ->
     Sc = emqx_schema:server_ssl_opts_schema(#{}, false),
     Checked = validate(Sc, #{<<"versions">> => [<<"tlsv1.2">>]}),
-    ?assertMatch(#{versions := ['tlsv1.2']}, Checked),
-    #{ciphers := Ciphers} = Checked,
-    PskCiphers = emqx_schema:default_ciphers(psk),
-    lists:foreach(fun(Cipher) ->
-                          ?assert(lists:member(Cipher, Ciphers))
-                  end, PskCiphers).
+    ?assertMatch(#{versions := ['tlsv1.2']}, Checked).
 
 bad_cipher_test() ->
     Sc = emqx_schema:server_ssl_opts_schema(#{}, false),

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -48,11 +48,6 @@ fields(file) ->
     , {enable, #{type => boolean(),
                  default => true}}
     , {path, #{type => string(),
-               validator => fun(S) -> case filelib:is_file(S) of
-                                        true -> ok;
-                                        _ -> {error, "File does not exist"}
-                                      end
-                            end,
                desc => "Path to the file which contains the ACL rules."
               }}
     ];

apps/emqx_machine/src/emqx_machine.erl

@@ -35,8 +35,12 @@
 
 %% @doc EMQ X boot entrypoint.
 start() ->
+    case os:type() of
+        {win32, nt} -> ok;
+        _nix ->
             os:set_signal(sighup, ignore),
-    os:set_signal(sigterm, handle), %% default is handle
+            os:set_signal(sigterm, handle) %% default is handle
+    end,
     ok = set_backtrace_depth(),
     ok = print_otp_version_warning(),
 

bin/emqx.cmd

@@ -22,14 +22,19 @@
 
 @set script=%~n0
 
+@set EPMD_ARG=-start_epmd false -epmd_module ekka_epmd -proto_dist ekka
+@set ERL_FLAGS=%EPMD_ARG%
+
 :: Discover the release root directory from the directory
 :: of this script
 @set script_dir=%~dp0
 @for %%A in ("%script_dir%\..") do @(
   set rel_root_dir=%%~fA
 )
+
 @set rel_dir=%rel_root_dir%\releases\%rel_vsn%
 @set RUNNER_ROOT_DIR=%rel_root_dir%
+@set RUNNER_ETC_DIR=%rel_root_dir%\etc
 
 @set etc_dir=%rel_root_dir%\etc
 @set lib_dir=%rel_root_dir%\lib
@@ -46,22 +51,22 @@
 @set progname=erl.exe
 @set clean_boot_script=%rel_root_dir%\bin\start_clean
 @set erlsrv="%bindir%\erlsrv.exe"
-@set epmd="%bindir%\epmd.exe"
 @set escript="%bindir%\escript.exe"
 @set werl="%bindir%\werl.exe"
 @set erl_exe="%bindir%\erl.exe"
 @set nodetool="%rel_root_dir%\bin\nodetool"
 @set cuttlefish="%rel_root_dir%\bin\cuttlefish"
 @set node_type="-name"
+@set schema_mod="emqx_machine_schema"
 
 :: Extract node name from emqx.conf
-@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.name "%emqx_conf%"`) do @(
+@for /f "usebackq delims=" %%I in (`"%escript% %nodetool% hocon -s %schema_mod% -c %etc_dir%\emqx.conf get node.name"`) do @(
   @call :set_trim node_name %%I
 )
 
 :: Extract node cookie from emqx.conf
-@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.cookie "%emqx_conf%"`) do @(
-  @call :set_trim node_cookie= %%I
+@for /f "usebackq delims=" %%I in (`"%escript% %nodetool% hocon -s %schema_mod% -c %etc_dir%\emqx.conf get node.cookie"`) do @(
+  @call :set_trim node_cookie %%I
 )
 
 :: Write the erl.ini file to set up paths relative to this script
@@ -139,13 +144,23 @@
 )
 @goto :eof
 
-:generate_app_config
-@set gen_config_cmd=%escript% %cuttlefish% -i %rel_dir%\emqx.schema -c %etc_dir%\emqx.conf -d %data_dir%\configs generate
-@for /f "delims=" %%A in ('%%gen_config_cmd%%') do @(
-  set generated_config_args=%%A
+:: get the current time with hocon
+:get_cur_time
+@for /f "usebackq tokens=1-6 delims=." %%a in (`"%escript% %nodetool% hocon now_time"`) do @(
+  set now_time=%%a.%%b.%%c.%%d.%%e.%%f
 )
 @goto :eof
 
+:generate_app_config
+@call :get_cur_time
+%escript% %nodetool% hocon -v -t %now_time% -s %schema_mod% -c "%etc_dir%\emqx.conf" -d "%data_dir%\configs" generate
+@set generated_config_args=-config %data_dir%\configs\app.%now_time%.config -args_file %data_dir%\configs\vm.%now_time%.args
+:: create one new line
+@echo.>>%data_dir%\configs\vm.%now_time%.args
+:: write the node type and node name in to vm args file
+@echo %node_type% %node_name%>>%data_dir%\configs\vm.%now_time%.args
+@goto :eof
+
 :: set boot_script variable
 :set_boot_script_var
 @if exist "%rel_dir%\%rel_name%.boot" (
@@ -188,13 +203,11 @@
   :: relup and reldown
   goto relup
 )
-
 @goto :eof
 
 :: Uninstall the Windows service
 :uninstall
 @%erlsrv% remove %service_name%
-@%epmd% -kill
 @goto :eof
 
 :: Start the Windows service
@@ -207,7 +220,7 @@
 @echo off
 cd /d %rel_root_dir%
 @echo on
-@start "%rel_name%" %werl% -boot "%boot_script%" %args%
+@start "%rel_name%" %werl% -boot  "%boot_script%" -mode embedded %args%
 @goto :eof
 
 :: Stop the Windows service
@@ -237,7 +250,7 @@ cd /d %rel_root_dir%
 @echo off
 cd /d %rel_root_dir%
 @echo on
-@start "bin\%rel_name% console" %werl% -boot "%boot_script%" %args%
+@start "bin\%rel_name% console" %werl% -boot "%boot_script%" -mode embedded %args%
 @echo emqx is started!
 @goto :eof
 
@@ -262,4 +275,3 @@ cd /d %rel_root_dir%
 :set_trim
 @set %1=%2
 @goto :eof
-