From 6dedf4875a5b91182c526330250047878a6b67dc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 21 Feb 2022 19:16:58 +0100 Subject: [PATCH 01/10] ci(windows): build windows package on otp 24 --- .github/workflows/build_packages.yaml | 45 ++++++---------------- .github/workflows/build_slim_packages.yaml | 37 ++++++++++++++++++ Windows.md | 4 +- 3 files changed, 50 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 8798796e7..a2bdc35af 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -52,18 +52,14 @@ jobs: windows: runs-on: windows-2019 - needs: prepare - strategy: fail-fast: false matrix: - profile: # only CE for windows + profile: # for now only CE for windows - emqx otp: - ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows - - 23.2 - + - 24.2.1 steps: - uses: actions/download-artifact@v2 with: @@ -72,8 +68,7 @@ jobs: - name: unzip source code run: Expand-Archive -Path source.zip -DestinationPath ./ - uses: ilammy/msvc-dev-cmd@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang + - uses: erlef/setup-beam@v1 with: otp-version: ${{ matrix.otp }} - name: build @@ -82,40 +77,22 @@ jobs: DIAGNOSTIC: 1 working-directory: source run: | - $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" - - $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) - if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { - $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" - $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.tar.gz" - } - else { - $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.tar.gz" - } - ## We do not build/release bcrypt and quic for windows package - Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ - Remove-Item -Recurse -Force -Path _build/default/lib/quicer/ - if (Test-Path rebar.lock) { - Remove-Item -Force -Path rebar.lock - } - make ensure-rebar3 - copy rebar3 "${{ steps.install_erlang.outputs.erlpath }}\bin" - ls "${{ steps.install_erlang.outputs.erlpath }}\bin" - rebar3 --help - make ${{ matrix.profile }} - mkdir -p _packages/${{ matrix.profile }} - Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name - mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }} - Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256 + # ensure crypto app (openssl) + erl -eval "erlang:display(crypto:info_lib())" -s init stop + make ${{ matrix.profile }}-tgz - name: run emqx - timeout-minutes: 1 + timeout-minutes: 5 working-directory: source run: | ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start Start-Sleep -s 5 + echo "EMQX started" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop + echo "EMQX stopped" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install + echo "EMQX installed" ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall + echo "EMQX uninstalled" - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 0f6317732..90a110146 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -96,6 +96,43 @@ jobs: with: name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} path: _packages/${{ matrix.profile}}/*.tar.gz + + windows: + runs-on: windows-2019 + strategy: + fail-fast: false + matrix: + profile: + - emqx + otp: + - 24.2.1 + steps: + - uses: actions/checkout@v2 + - uses: ilammy/msvc-dev-cmd@v1 + - uses: erlef/setup-beam@v1 + with: + otp-version: ${{ matrix.otp }} + - name: build + env: + PYTHON: python + DIAGNOSTIC: 1 + run: | + # ensure crypto app (openssl) + erl -eval "erlang:display(crypto:info_lib())" -s init stop + make ${{ matrix.profile }}-tgz + - name: run emqx + timeout-minutes: 5 + run: | + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start + Start-Sleep -s 5 + echo "EMQX started" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop + echo "EMQX stopped" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install + echo "EQMX installed" + ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall + echo "EQMX uninstaled" + mac: strategy: fail-fast: false diff --git a/Windows.md b/Windows.md index 7ae0f738c..1e72b1982 100644 --- a/Windows.md +++ b/Windows.md @@ -29,7 +29,7 @@ The second path is for CMD to setup environment variables. ### Erlang/OTP -Install Erlang/OTP 23.2 from https://www.erlang.org/downloads +Install Erlang/OTP 24 from https://www.erlang.org/downloads You may need to edit the `Path` environment variable to allow running Erlang commands such as `erl` from powershell. @@ -45,7 +45,7 @@ e.g. ``` PS C:\Users\zmsto> erl -Eshell V11.1.4 (abort with ^G) +Eshell V12.2.1 (abort with ^G) 1> halt(). ``` From a4e48b197ce25df48cbc9987c2491bdc925d1355 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 21 Feb 2022 19:58:36 +0100 Subject: [PATCH 02/10] build: create sha256 sum right after package is built --- .github/workflows/build_packages.yaml | 20 +++---------- build | 42 ++++++++++++++++++--------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index a2bdc35af..9a2958d5e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -96,7 +96,7 @@ jobs: - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.profile }} + name: ${{ matrix.profile }}-windows path: source/_packages/${{ matrix.profile }}/. mac: @@ -186,7 +186,6 @@ jobs: ./emqx/bin/emqx_ctl status ./emqx/bin/emqx stop rm -rf emqx - openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256 - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: @@ -331,19 +330,6 @@ jobs: --system "${SYSTEM}" \ --builder "ghcr.io/emqx/emqx-builder/5.0-5:${ELIXIR}-${OTP}-${SYSTEM}" done - - - name: create sha256 - env: - PROFILE: ${{ matrix.profile}} - working-directory: source - run: | - if [ -d _packages/$PROFILE ]; then - cd _packages/$PROFILE - for var in $(ls emqx-* ); do - bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" - done - cd - - fi - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: @@ -632,7 +618,9 @@ jobs: - emqx-enterprise otp: - 24.1.5-4 - + include: + - profile: emqx + otp: windows # otp version on windows is rather fixed steps: - uses: actions/checkout@v2 - name: get_version diff --git a/build b/build index 569f3c619..0ab504bc1 100755 --- a/build +++ b/build @@ -78,8 +78,7 @@ make_doc() { } make_rel() { - # shellcheck disable=SC1010 - ./rebar3 as "$PROFILE" do tar + ./rebar3 as "$PROFILE" tar if [ "$("$FIND" "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then echo "gpb should not be included in the release" exit 1 @@ -132,40 +131,55 @@ cp_dyn_libs() { ## It assumes the .tar.gz has been built -- relies on Makefile dependency make_tgz() { local pkgpath="_packages/${PROFILE}" - local tarball + local src_tarball + local target_name local target if [ "${IS_ELIXIR:-no}" = "yes" ] then - # ensure tarball exists + # ensure src_tarball exists ELIXIR_MAKE_TAR=yes make_elixir_rel local relpath="_build/${PROFILE}" - target="${pkgpath}/${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + target_name="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" else - # build the tarball again to ensure relup is included + # build the src_tarball again to ensure relup is included # elixir does not have relup yet. make_rel local relpath="_build/${PROFILE}/rel/emqx" - target="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + target_name="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" fi + target="${pkgpath}/${target_name}" - tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" - tard="/tmp/emqx_untar_${PKG_VSN}" + src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" + tard="tmp/emqx_untar_${PKG_VSN}" rm -rf "${tard}" mkdir -p "${tard}/emqx" - mkdir -p "${pkgpath}" - if [ ! -f "$tarball" ]; then - log "ERROR: $tarball is not found" + if [ ! -f "$src_tarball" ]; then + log "ERROR: $src_tarball is not found" fi - tar zxf "${tarball}" -C "${tard}/emqx" + tar zxf "${src_tarball}" -C "${tard}/emqx" ## try to be portable for tar.gz packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt cp_dyn_libs "${tard}/emqx" - (cd "${tard}" && tar -cz emqx) > "${target}" + ## create tar after change dir (for windows) + pushd "${tard}" >/dev/null + tar -czf "${target_name}" emqx + popd >/dev/null + mv "${tard}/${target_name}" "${target}" + case "$SYSTEM" in + macos*) + # sha256sum may not be available on macos + openssl dgst -sha256 "${target}" | cut -d ' ' -f 2 > "${target}.sha256" + ;; + *) + sha256sum "${target}" | head -c 64 > "${target}.sha256" + ;; + esac log "Tarball successfully repacked: ${target}" + log "Tarball sha256sum: $(cat "${target}.sha256")" } ## This function builds the default docker image based on alpine:3.14 (by default) From c2bd30466f83a5f7f4d49186599af87679d944c0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 16:58:28 +0100 Subject: [PATCH 03/10] build: slient pushd and popd --- scripts/relup-base-packages.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 2f4bea1d8..686fcb6c7 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -51,7 +51,7 @@ if [ "$SYSTEM" = "macos" ]; then fi mkdir -p _upgrade_base -pushd _upgrade_base +pushd _upgrade_base >/dev/null for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.tar.gz" @@ -67,4 +67,4 @@ for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do fi done -popd +popd >/dev/null From 08d1b38bbe436a8d1116c5350630989afa7cc008 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 18:13:55 +0100 Subject: [PATCH 04/10] chore: refine console printout about quic not started --- apps/emqx/src/emqx_listeners.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index a249447d3..1a995e000 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -148,8 +148,8 @@ start_listener(Type, ListenerName, #{bind := Bind} = Conf) -> case do_start_listener(Type, ListenerName, Conf) of {ok, {skipped, Reason}} when Reason =:= listener_disabled; Reason =:= quic_app_missing -> - console_print("- Skip - starting listener ~ts on ~ts ~n due to ~p", - [listener_id(Type, ListenerName), format_addr(Bind), Reason]); + console_print("Listener ~ts is NOT started due to: ~p~n.", + [listener_id(Type, ListenerName), Reason]); {ok, _} -> console_print("Listener ~ts on ~ts started.~n", [listener_id(Type, ListenerName), format_addr(Bind)]); From 5a0e3c1c7b3602125be6d3ee0c734f5bf6352c43 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 18:26:12 +0100 Subject: [PATCH 05/10] fix(windows): fix boot in windows --- bin/emqx.cmd | 41 +++++++---------------------------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/bin/emqx.cmd b/bin/emqx.cmd index 9cf81d69f..26e0220a7 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -44,12 +44,12 @@ @call :find_erts_dir @call :find_vm_args @call :find_sys_config -@call :set_boot_script_var +@set boot_file_name=%rel_dir%\start @set service_name=%rel_name%_%rel_vsn% @set bindir=%erts_dir%\bin @set progname=erl.exe -@set clean_boot_script=%rel_root_dir%\bin\start_clean +@set clean_boot_file_name=%rel_root_dir%\bin\start_clean @set erlsrv="%bindir%\erlsrv.exe" @set escript="%bindir%\escript.exe" @set werl="%bindir%\werl.exe" @@ -57,7 +57,7 @@ @set nodetool="%rel_root_dir%\bin\nodetool" @set cuttlefish="%rel_root_dir%\bin\cuttlefish" @set node_type="-name" -@set schema_mod="emqx_conf_schema" +@set schema_mod=emqx_conf_schema @set conf_path="%etc_dir%\emqx.conf" :: Extract node name from emqx.conf @@ -83,8 +83,6 @@ @if "%1"=="start" @goto start @if "%1"=="stop" @goto stop @if "%1"=="restart" @call :stop && @goto start -::@if "%1"=="upgrade" @goto relup -::@if "%1"=="downgrade" @goto relup @if "%1"=="console" @goto console @if "%1"=="ping" @goto ping @if "%1"=="list" @goto list @@ -143,7 +141,6 @@ @for /f "delims=" %%Z in ('%%create_dir_cmd%%') do @( set mnesia_dir=%%Z ) -@set mnesia_dir="%mnesia_dir%" @goto :eof :: get the current time with hocon @@ -163,15 +160,6 @@ @echo %node_type% %node_name%>>"%data_dir%\configs\vm.%now_time%.args" @goto :eof -:: set boot_script variable -:set_boot_script_var -@if exist "%rel_dir%\%rel_name%.boot" ( - set boot_script=%rel_dir%\%rel_name% -) else ( - set boot_script=%rel_dir%\start -) -@goto :eof - :: Write the erl.ini file :write_ini @set erl_ini=%erts_dir%\bin\erl.ini @@ -194,16 +182,13 @@ @call :create_mnesia_dir @call :generate_app_config :: Install the service -@set args="-boot %boot_script% %sys_config% %generated_config_args% -mnesia dir '%mnesia_dir%'" +@set args="-boot %boot_file_name% %sys_config% %generated_config_args% -mnesia dir '%mnesia_dir%'" @set description=EMQX node %node_name% in %rootdir% @if "" == "%2" ( %erlsrv% add %service_name% %node_type% "%node_name%" -on restart -c "%description%" ^ -i "emqx" -w "%rootdir%" -m %erl_exe% -args %args% ^ -st "init:stop()." sc config emqx start=delayed-auto -) else ( - :: relup and reldown - goto relup ) @goto :eof @@ -222,7 +207,7 @@ @echo off cd /d "%rel_root_dir%" @echo on -@start "%rel_name%" %werl% -boot "%boot_script%" -mode embedded %args% +@start "%rel_name%" %werl% -mode embedded -boot "%boot_file_name%" %args% @goto :eof :: Stop the Windows service @@ -232,18 +217,6 @@ cd /d "%rel_root_dir%" @%escript% %nodetool% %node_type% %node_name% -setcookie %node_cookie% stop @goto :eof -:: Relup and reldown -:relup -@if "" == "%2" ( - echo Missing package argument - echo Usage: %rel_name% %1 {package base name} - echo NOTE {package base name} MUST NOT include the .tar.gz suffix - set ERRORLEVEL=1 - exit /b %ERRORLEVEL% -) -@%escript% "%rootdir%/bin/install_upgrade.escript" "%rel_name%" "%node_name%" "%node_cookie%" "%2" -@goto :eof - :: Start a console :console @call :create_mnesia_dir @@ -252,7 +225,7 @@ cd /d "%rel_root_dir%" @echo off cd /d %rel_root_dir% @echo on -@start "bin\%rel_name% console" %werl% -boot "%boot_script%" -mode embedded %args% +@start "%rel_name% console" %werl% -mode embedded -boot "%boot_file_name%" %args% @echo emqx is started! @goto :eof @@ -269,7 +242,7 @@ cd /d %rel_root_dir% :: Attach to a running node :attach :: @start "%node_name% attach" -@start "%node_name% attach" %werl% -boot "%clean_boot_script%" ^ +@start "%node_name% attach" %werl% -boot "%clean_boot_file_name%" ^ -remsh %node_name% %node_type% console_%node_name% -setcookie %node_cookie% @goto :eof From 893b8444445868b679075fa794659369001a37fc Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 18:58:03 +0100 Subject: [PATCH 06/10] feat(windows): simplify emqx_ctl for windows --- bin/emqx.cmd | 10 +++++- bin/emqx_ctl.cmd | 85 ++---------------------------------------------- 2 files changed, 11 insertions(+), 84 deletions(-) diff --git a/bin/emqx.cmd b/bin/emqx.cmd index 26e0220a7..dc81b25c1 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -8,6 +8,7 @@ :: * restart - run the stop command and start command :: * uninstall - uninstall the service and kill a running node :: * ping - check if the node is running +:: * ctl - run management commands :: * console - start the Erlang release in a `werl` Windows shell :: * attach - connect to a running node and open an interactive console :: * list - display a listing of installed Erlang services @@ -85,6 +86,7 @@ @if "%1"=="restart" @call :stop && @goto start @if "%1"=="console" @goto console @if "%1"=="ping" @goto ping +@if "%1"=="ctl" @goto ctl @if "%1"=="list" @goto list @if "%1"=="attach" @goto attach @if "%1"=="" @goto usage @@ -173,7 +175,7 @@ :: Display usage information :usage -@echo usage: %~n0 ^(install^|uninstall^|start^|stop^|restart^|console^|ping^|list^|attach^) +@echo usage: %~n0 ^(install^|uninstall^|start^|stop^|restart^|console^|ping^|ctl^|list^|attach^) @goto :eof :: Install the release as a Windows service @@ -234,6 +236,12 @@ cd /d %rel_root_dir% @%escript% %nodetool% ping %node_type% "%node_name%" -setcookie "%node_cookie%" @goto :eof +:: ctl to execute management commands +:ctl +@for /f "usebackq tokens=1*" %%i in (`echo %*`) DO @ set params=%%j +@%escript% %nodetool% %node_type% "%node_name%" -setcookie "%node_cookie%" rpc_infinity emqx_ctl run_command %params% +@goto :eof + :: List installed Erlang services :list @%erlsrv% list %service_name% diff --git a/bin/emqx_ctl.cmd b/bin/emqx_ctl.cmd index ae9cad2e7..177c5cc39 100644 --- a/bin/emqx_ctl.cmd +++ b/bin/emqx_ctl.cmd @@ -1,92 +1,11 @@ -:: The batch file for emqx_ctl command +:: The batch file for 'emqx ctl' command @set args=%* -:: Set variables that describe the release -@set rel_name=emqx -@set rel_vsn={{ release_version }} -@set erts_vsn={{ erts_vsn }} -@set erl_opts={{ erl_opts }} - :: Discover the release root directory from the directory :: of this script @set script_dir=%~dp0 @for %%A in ("%script_dir%\..") do @( set rel_root_dir=%%~fA ) -@set rel_dir=%rel_root_dir%\releases\%rel_vsn% -@set emqx_conf=%rel_root_dir%\etc\emqx.conf - -@call :find_erts_dir - -@set bindir=%erts_dir%\bin -@set progname=erl.exe -@set escript="%bindir%\escript.exe" -@set nodetool="%rel_root_dir%\bin\nodetool" -@set node_type="-name" - -:: Extract node name from emqx.conf -@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.name "%emqx_conf%"`) do @( - @call :set_trim node_name %%I -) - -:: Extract node cookie from emqx.conf -@for /f "usebackq delims=\= tokens=2" %%I in (`findstr /b node\.cookie "%emqx_conf%"`) do @( - @call :set_trim node_cookie= %%I -) - -:: Write the erl.ini file to set up paths relative to this script -@call :write_ini - -:: If a start.boot file is not present, copy one from the named .boot file -@if not exist "%rel_dir%\start.boot" ( - copy "%rel_dir%\%rel_name%.boot" "%rel_dir%\start.boot" >nul -) - -@%escript% %nodetool% %node_type% "%node_name%" -setcookie "%node_cookie%" rpc emqx_ctl run_command %args% - -:: Find the ERTS dir -:find_erts_dir -@set possible_erts_dir=%rel_root_dir%\erts-%erts_vsn% -@if exist "%possible_erts_dir%" ( - call :set_erts_dir_from_default -) else ( - call :set_erts_dir_from_erl -) -@goto :eof - -:: Set the ERTS dir from the passed in erts_vsn -:set_erts_dir_from_default -@set erts_dir=%possible_erts_dir% -@set rootdir=%rel_root_dir% -@goto :eof - -:: Set the ERTS dir from erl -:set_erts_dir_from_erl -@for /f "delims=" %%i in ('where erl') do @( - set erl=%%i -) -@set dir_cmd="%erl%" -noshell -eval "io:format(\"~s\", [filename:nativename(code:root_dir())])." -s init stop -@for /f %%i in ('%%dir_cmd%%') do @( - set erl_root=%%i -) -@set erts_dir=%erl_root%\erts-%erts_vsn% -@set rootdir=%erl_root% -@goto :eof - -:: Write the erl.ini file -:write_ini -@set erl_ini=%erts_dir%\bin\erl.ini -@set converted_bindir=%bindir:\=\\% -@set converted_rootdir=%rootdir:\=\\% -@echo [erlang] > "%erl_ini%" -@echo Bindir=%converted_bindir% >> "%erl_ini%" -@echo Progname=%progname% >> "%erl_ini%" -@echo Rootdir=%converted_rootdir% >> "%erl_ini%" -@goto :eof - -:: Trim variable -:set_trim -@set %1=%2 -@goto :eof - +@%rel_root_dir%\bin\emqx.cmd ctl %args% From 0fa7b4a7d46aa5c619132e407c776ba012c7ebd4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 20:47:23 +0100 Subject: [PATCH 07/10] fix(windows): wrong clean start boot file path --- bin/emqx.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/emqx.cmd b/bin/emqx.cmd index dc81b25c1..e4999c21c 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -50,7 +50,7 @@ @set service_name=%rel_name%_%rel_vsn% @set bindir=%erts_dir%\bin @set progname=erl.exe -@set clean_boot_file_name=%rel_root_dir%\bin\start_clean +@set clean_boot_file_name=%rel_dir%\start_clean @set erlsrv="%bindir%\erlsrv.exe" @set escript="%bindir%\escript.exe" @set werl="%bindir%\werl.exe" From d9bb616574018e6a5a3b065f860c10e5ccfa66ff Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 21:31:40 +0100 Subject: [PATCH 08/10] fix(windows): attach remote_console --- bin/emqx.cmd | 66 ++++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/bin/emqx.cmd b/bin/emqx.cmd index e4999c21c..6c6fa86ea 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -11,6 +11,7 @@ :: * ctl - run management commands :: * console - start the Erlang release in a `werl` Windows shell :: * attach - connect to a running node and open an interactive console +:: * remote_console - same as attach :: * list - display a listing of installed Erlang services :: * usage - display available commands @@ -23,7 +24,9 @@ @set script=%~n0 +:: for remote_console @set EPMD_ARG=-start_epmd false -epmd_module ekka_epmd -proto_dist ekka +:: for erl command @set ERL_FLAGS=%EPMD_ARG% :: Discover the release root directory from the directory @@ -33,31 +36,31 @@ set rel_root_dir=%%~fA ) -@set rel_dir=%rel_root_dir%\releases\%rel_vsn% -@set RUNNER_ROOT_DIR=%rel_root_dir% -@set RUNNER_ETC_DIR=%rel_root_dir%\etc +@set "rel_dir=%rel_root_dir%\releases\%rel_vsn%" +@set "RUNNER_ROOT_DIR=%rel_root_dir%" +@set "RUNNER_ETC_DIR=%rel_root_dir%\etc" -@set etc_dir=%rel_root_dir%\etc -@set lib_dir=%rel_root_dir%\lib -@set data_dir=%rel_root_dir%\data -@set emqx_conf=%etc_dir%\emqx.conf +@set "etc_dir=%rel_root_dir%\etc" +@set "lib_dir=%rel_root_dir%\lib" +@set "data_dir=%rel_root_dir%\data" +@set "emqx_conf=%etc_dir%\emqx.conf" @call :find_erts_dir @call :find_vm_args @call :find_sys_config -@set boot_file_name=%rel_dir%\start -@set service_name=%rel_name%_%rel_vsn% -@set bindir=%erts_dir%\bin +@set "boot_file_name=%rel_dir%\start" +@set "service_name=%rel_name%_%rel_vsn%" +@set "bindir=%erts_dir%\bin" @set progname=erl.exe -@set clean_boot_file_name=%rel_dir%\start_clean -@set erlsrv="%bindir%\erlsrv.exe" -@set escript="%bindir%\escript.exe" -@set werl="%bindir%\werl.exe" -@set erl_exe="%bindir%\erl.exe" -@set nodetool="%rel_root_dir%\bin\nodetool" -@set cuttlefish="%rel_root_dir%\bin\cuttlefish" -@set node_type="-name" +@set "clean_boot_file_name=%rel_dir%\start_clean" +@set "erlsrv=%bindir%\erlsrv.exe" +@set "escript=%bindir%\escript.exe" +@set "werl=%bindir%\werl.exe" +@set "erl_exe=%bindir%\erl.exe" +@set "nodetool=%rel_root_dir%\bin\nodetool" +@set "cuttlefish=%rel_root_dir%\bin\cuttlefish" +@set node_type=-name @set schema_mod=emqx_conf_schema @set conf_path="%etc_dir%\emqx.conf" @@ -65,11 +68,13 @@ @for /f "usebackq delims=" %%I in (`"%escript% %nodetool% hocon -s %schema_mod% -c %conf_path% get node.name"`) do @( @call :set_trim node_name %%I ) +@set node_name=%node_name:"=% :: Extract node cookie from emqx.conf @for /f "usebackq delims=" %%I in (`"%escript% %nodetool% hocon -s %schema_mod% -c %conf_path% get node.cookie"`) do @( @call :set_trim node_cookie %%I ) +@set node_cookie=%node_cookie:"=% :: Write the erl.ini file to set up paths relative to this script @call :write_ini @@ -89,6 +94,7 @@ @if "%1"=="ctl" @goto ctl @if "%1"=="list" @goto list @if "%1"=="attach" @goto attach +@if "%1"=="remote_console" @goto attach @if "%1"=="" @goto usage @echo Unknown command: "%1" @@ -96,7 +102,7 @@ :: Find the ERTS dir :find_erts_dir -@set possible_erts_dir=%rel_root_dir%\erts-%erts_vsn% +@set "possible_erts_dir=%rel_root_dir%\erts-%erts_vsn%" @if exist "%possible_erts_dir%" ( call :set_erts_dir_from_default ) else ( @@ -106,8 +112,8 @@ :: Set the ERTS dir from the passed in erts_vsn :set_erts_dir_from_default -@set erts_dir=%possible_erts_dir% -@set rootdir=%rel_root_dir% +@set "erts_dir=%possible_erts_dir%" +@set "rootdir=%rel_root_dir%" @goto :eof :: Set the ERTS dir from erl @@ -117,14 +123,14 @@ ) @set dir_cmd="%erl%" -noshell -eval "io:format(\"~s\", [filename:nativename(code:root_dir())])." -s init stop @for /f %%i in ('%%dir_cmd%%') do @( - set erl_root=%%i + set erl_root=%%i" ) -@set erts_dir=%erl_root%\erts-%erts_vsn% -@set rootdir=%erl_root% +@set "erts_dir=%erl_root%\erts-%erts_vsn%" +@set "rootdir=%erl_root%" @goto :eof :find_vm_args -@set possible_vm=%etc_dir%\vm.args +@set "possible_vm=%etc_dir%\vm.args" @if exist "%possible_vm%" ( set args_file=-args_file "%possible_vm%" ) @@ -132,7 +138,7 @@ :: Find the sys.config file :find_sys_config -@set possible_sys=%etc_dir%\sys.config +@set "possible_sys=%etc_dir%\sys.config" @if exist "%possible_sys%" ( set sys_config=-config "%possible_sys%" ) @@ -164,7 +170,7 @@ :: Write the erl.ini file :write_ini -@set erl_ini=%erts_dir%\bin\erl.ini +@set "erl_ini=%erts_dir%\bin\erl.ini" @set converted_bindir=%bindir:\=\\% @set converted_rootdir=%rootdir:\=\\% @echo [erlang] > "%erl_ini%" @@ -175,7 +181,7 @@ :: Display usage information :usage -@echo usage: %~n0 ^(install^|uninstall^|start^|stop^|restart^|console^|ping^|ctl^|list^|attach^) +@echo usage: %~n0 ^(install^|uninstall^|start^|stop^|restart^|console^|ping^|ctl^|list^|remote_console^|attach^) @goto :eof :: Install the release as a Windows service @@ -249,9 +255,7 @@ cd /d %rel_root_dir% :: Attach to a running node :attach -:: @start "%node_name% attach" -@start "%node_name% attach" %werl% -boot "%clean_boot_file_name%" ^ - -remsh %node_name% %node_type% console_%node_name% -setcookie %node_cookie% +@start "remsh_%nodename%" %werl% -hidden -remsh "%node_name%" -boot "%clean_boot_file_name%" "%node_type%" "remsh_%node_name%" -setcookie "%node_cookie%" @goto :eof :: Trim variable From 88343df95a5229af6d7616580249fe87386eea21 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 23:10:42 +0100 Subject: [PATCH 09/10] fix(windows): delete stale find functions some of the find_ functions are generated from the old rebar template which will never be used since we: 1. always release erts 2. always generate sys.config 3. always generate vm.args --- bin/emqx.cmd | 67 ++++++++++------------------------------------------ 1 file changed, 13 insertions(+), 54 deletions(-) diff --git a/bin/emqx.cmd b/bin/emqx.cmd index 6c6fa86ea..236324e85 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -36,19 +36,22 @@ set rel_root_dir=%%~fA ) +@set "erts_dir=%rel_root_dir%\erts-%erts_vsn%" +@set "rootdir=%rel_root_dir%" @set "rel_dir=%rel_root_dir%\releases\%rel_vsn%" @set "RUNNER_ROOT_DIR=%rel_root_dir%" +:: hard code etc dir @set "RUNNER_ETC_DIR=%rel_root_dir%\etc" - @set "etc_dir=%rel_root_dir%\etc" @set "lib_dir=%rel_root_dir%\lib" -@set "data_dir=%rel_root_dir%\data" +:: allow setting data dir +@if "%RUNNER_DATA_DIR%"=="" ( + @set "data_dir=%rel_root_dir%\data" +) else ( + @set "data_dir=%RUNNER_DATA_DIR%" +) @set "emqx_conf=%etc_dir%\emqx.conf" -@call :find_erts_dir -@call :find_vm_args -@call :find_sys_config - @set "boot_file_name=%rel_dir%\start" @set "service_name=%rel_name%_%rel_vsn%" @set "bindir=%erts_dir%\bin" @@ -100,50 +103,6 @@ @goto :eof -:: Find the ERTS dir -:find_erts_dir -@set "possible_erts_dir=%rel_root_dir%\erts-%erts_vsn%" -@if exist "%possible_erts_dir%" ( - call :set_erts_dir_from_default -) else ( - call :set_erts_dir_from_erl -) -@goto :eof - -:: Set the ERTS dir from the passed in erts_vsn -:set_erts_dir_from_default -@set "erts_dir=%possible_erts_dir%" -@set "rootdir=%rel_root_dir%" -@goto :eof - -:: Set the ERTS dir from erl -:set_erts_dir_from_erl -@for /f "delims=" %%i in ('where erl') do @( - set erl=%%i -) -@set dir_cmd="%erl%" -noshell -eval "io:format(\"~s\", [filename:nativename(code:root_dir())])." -s init stop -@for /f %%i in ('%%dir_cmd%%') do @( - set erl_root=%%i" -) -@set "erts_dir=%erl_root%\erts-%erts_vsn%" -@set "rootdir=%erl_root%" -@goto :eof - -:find_vm_args -@set "possible_vm=%etc_dir%\vm.args" -@if exist "%possible_vm%" ( - set args_file=-args_file "%possible_vm%" -) -@goto :eof - -:: Find the sys.config file -:find_sys_config -@set "possible_sys=%etc_dir%\sys.config" -@if exist "%possible_sys%" ( - set sys_config=-config "%possible_sys%" -) -@goto :eof - :create_mnesia_dir @set create_dir_cmd=%escript% %nodetool% mnesia_dir "%data_dir%\mnesia" %node_name% @for /f "delims=" %%Z in ('%%create_dir_cmd%%') do @( @@ -160,7 +119,7 @@ :generate_app_config @call :get_cur_time -%escript% %nodetool% hocon -v -t %now_time% -s %schema_mod% -c "%etc_dir%\emqx.conf" -d "%data_dir%\configs" generate +@%escript% %nodetool% hocon -v -t %now_time% -s %schema_mod% -c "%etc_dir%\emqx.conf" -d "%data_dir%\configs" generate @set generated_config_args=-config "%data_dir%\configs\app.%now_time%.config" -args_file "%data_dir%\configs\vm.%now_time%.args" :: create one new line @echo.>>"%data_dir%\configs\vm.%now_time%.args" @@ -190,7 +149,7 @@ @call :create_mnesia_dir @call :generate_app_config :: Install the service -@set args="-boot %boot_file_name% %sys_config% %generated_config_args% -mnesia dir '%mnesia_dir%'" +@set args="-boot %boot_file_name% %generated_config_args% -mnesia dir '%mnesia_dir%'" @set description=EMQX node %node_name% in %rootdir% @if "" == "%2" ( %erlsrv% add %service_name% %node_type% "%node_name%" -on restart -c "%description%" ^ @@ -211,7 +170,7 @@ :: @%erlsrv% start %service_name% @call :create_mnesia_dir @call :generate_app_config -@set args=-detached %sys_config% %generated_config_args% -mnesia dir '%mnesia_dir%' +@set args=-detached %generated_config_args% -mnesia dir '%mnesia_dir%' @echo off cd /d "%rel_root_dir%" @echo on @@ -229,7 +188,7 @@ cd /d "%rel_root_dir%" :console @call :create_mnesia_dir @call :generate_app_config -@set args=%sys_config% %generated_config_args% -mnesia dir '%mnesia_dir%' +@set args=%generated_config_args% -mnesia dir '%mnesia_dir%' @echo off cd /d %rel_root_dir% @echo on From 271fda3d66b254fce8899b09ee18466a18daec36 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 23:39:26 +0100 Subject: [PATCH 10/10] fix(windows): deal with space in install path --- bin/emqx.cmd | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/emqx.cmd b/bin/emqx.cmd index 236324e85..0e418f782 100644 --- a/bin/emqx.cmd +++ b/bin/emqx.cmd @@ -36,6 +36,12 @@ set rel_root_dir=%%~fA ) +:: If release dir has space, change dir +@if not "%rel_root_dir%"=="%rel_root_dir: =%" ( + @chdir /d "%rel_root_dir%" + @set rel_root_dir=. +) + @set "erts_dir=%rel_root_dir%\erts-%erts_vsn%" @set "rootdir=%rel_root_dir%" @set "rel_dir=%rel_root_dir%\releases\%rel_vsn%"