From 8a92b9dafa2f70e877dccf3a30ce2b483ed3f4ba Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 10 Jan 2022 09:46:34 +0100 Subject: [PATCH 1/4] docs: update Windows build instructions --- .github/workflows/build_packages.yaml | 2 +- Windows.md | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index d72053f49..8a285b645 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -59,6 +59,7 @@ jobs: profile: # only CE for windows - emqx otp: + ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows - 23.2 steps: @@ -71,7 +72,6 @@ jobs: - uses: ilammy/msvc-dev-cmd@v1 - uses: gleam-lang/setup-erlang@v1.1.2 id: install_erlang - ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows with: otp-version: ${{ matrix.otp }} - name: build diff --git a/Windows.md b/Windows.md index 5e947a22e..09bc7e950 100644 --- a/Windows.md +++ b/Windows.md @@ -25,7 +25,7 @@ C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build Depending on your visual studio version and OS, the paths may differ. The first path is for rebar3 port compiler to find `cl.exe` and `link.exe` -The second path is for Powershell or CMD to setup environment variables. +The second path is for CMD to setup environment variables. ### Erlang/OTP @@ -73,6 +73,7 @@ Some of the unix world tools are required to build EMQ X. Including: * git * curl * make +* cmake * jq * zip / unzip @@ -81,14 +82,14 @@ We recommend using [scoop](https://scoop.sh/), or [Chocolatey](https://chocolate When using scoop: ``` -scoop install git curl make jq zip unzip +scoop install git curl make cmake jq zip unzip ``` ## Build EMQ X source code * Clone the repo: `git clone https://github.com/emqx/emqx.git` -* Start CMD or Powershell +* Start CMD console * Execute `vcvarsall.bat x86_amd64` to load environment variables From 4897e00aab9d8a15381a476a8374e7917d405490 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 10 Jan 2022 10:54:24 +0100 Subject: [PATCH 2/4] chore(scripts): get-distro support windows --- scripts/get-distro.sh | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index 89eafc4ee..962a9a4b6 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -5,18 +5,27 @@ set -euo pipefail -if [ "$(uname -s)" = 'Darwin' ]; then - DIST='macos' - VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') - SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" -elif [ "$(uname -s)" = 'Linux' ]; then - if grep -q -i 'centos' /etc/*-release; then - DIST='centos' - VERSION_ID="$(rpm --eval '%{centos_ver}')" - else - DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" - VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" - fi - SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" -fi +UNAME="$(uname -s)" + +case "$UNAME" in + Darwin) + DIST='macos' + VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') + SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" + ;; + Linux) + if grep -q -i 'centos' /etc/*-release; then + DIST='centos' + VERSION_ID="$(rpm --eval '%{centos_ver}')" + else + DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" + VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" + fi + SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" + ;; + CYGWIN*|MSYS*|MINGW*) + SYSTEM="windows" + ;; +esac + echo "$SYSTEM" From b2684b995d996f4b5b17989355d6f3fd3e5a9432 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 10 Jan 2022 07:52:18 +0100 Subject: [PATCH 3/4] build: doc for win --- Makefile | 7 +++++-- build | 28 +++++++++++++++++----------- rebar.config.erl | 12 ++++++------ scripts/find-apps.sh | 9 ++++++++- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 0a1fafbdd..d9383bbe6 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none + FIND=/usr/bin/find +else + FIND=find endif PROFILE ?= emqx @@ -103,8 +106,8 @@ $(PROFILES:%=clean-%): @if [ -d _build/$(@:clean-%=%) ]; then \ rm rebar.lock \ rm -rf _build/$(@:clean-%=%)/rel; \ - find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ - find _build/$(@:clean-%=%) -type l -delete; \ + $(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ + $(FIND) _build/$(@:clean-%=%) -type l -delete; \ fi .PHONY: clean-all diff --git a/build b/build index 15fcee226..5f7459583 100755 --- a/build +++ b/build @@ -45,6 +45,13 @@ if [ "$(uname -s)" = 'Linux' ]; then esac fi +if [ "${SYSTEM}" = 'windows' ]; then + # windows does not like the find + FIND="/usr/bin/find" +else + FIND='find' +fi + log() { local msg="$1" # rebar3 prints ===>, so we print ===< @@ -53,21 +60,20 @@ log() { make_doc() { local libs_dir1 libs_dir2 - libs_dir1="$(find "_build/default/lib/" -maxdepth 2 -name ebin -type d)" - libs_dir2="$(find "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)" - - local conf_doc_md - # TODO render md as html - conf_doc_md="$(pwd)/_build/${PROFILE}/lib/emqx_dashboard/priv/config.md" - echo "===< Generating config document $conf_doc_md" + libs_dir1="$("$FIND" "_build/default/lib/" -maxdepth 2 -name ebin -type d)" + libs_dir2="$("$FIND" "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)" # shellcheck disable=SC2086 - erl -noshell -pa $libs_dir1 $libs_dir2 -eval "ok = emqx_conf:gen_doc(\"${conf_doc_md}\"), halt(0)." + erl -noshell -pa $libs_dir1 $libs_dir2 -eval \ + "F = filename:join(['_build', '${PROFILE}', lib, emqx_dashboard, priv, 'config.md']), \ + io:format(\"===< Generating: ~s~n\", [F]), + ok = emqx_conf:gen_doc(F), \ + halt(0)." } make_rel() { # shellcheck disable=SC1010 ./rebar3 as "$PROFILE" do tar - if [ "$(find "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then + if [ "$("$FIND" "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then echo "gpb should not be included in the release" exit 1 fi @@ -93,7 +99,7 @@ make_relup() { rm -rf "$tmp_dir" fi releases+=( "$base_vsn" ) - done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f) fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -113,7 +119,7 @@ cp_dyn_libs() { mkdir -p "$target_dir" while read -r so_file; do cp -L "$so_file" "$target_dir/" - done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ + done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ | xargs -0 ldd \ | grep -E '(libcrypto)|(libtinfo)' \ | awk '{print $3}' \ diff --git a/rebar.config.erl b/rebar.config.erl index d0f123059..95e16835b 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -122,42 +122,42 @@ profiles() -> , {relx, relx(Vsn, cloud, bin, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} - , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx doc"}]} + , {post_hooks, [{compile, "bash build emqx doc"}]} ]} , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} - , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx-pkg doc"}]} + , {post_hooks, [{compile, "bash build emqx-pkg doc"}]} ]} , {'emqx-enterprise', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin, ee)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ee)} - , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx-enterprise doc"}]} + , {post_hooks, [{compile, "bash build emqx-enterprise doc"}]} ]} , {'emqx-enterprise-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg, ee)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ee)} - , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx-enterprise-pkg doc"}]} + , {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]} ]} , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} - , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx-edge doc"}]} + , {post_hooks, [{compile, "bash build emqx-edge doc"}]} ]} , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} - , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx-edge-pkg doc"}]} + , {post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]} ]} , {check, [ {erl_opts, common_compile_opts()} diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 7d7e5d33c..7a337f664 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -5,9 +5,16 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")/.." +if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then + # Otherwise windows may resolve to find.exe + FIND="/usr/bin/find" +else + FIND='find' +fi + find_app() { local appdir="$1" - find "${appdir}" -mindepth 1 -maxdepth 1 -type d + "$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d } find_app 'apps' From 39d60121d14facf3d82cf9b7ceb908526aed5247 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Mon, 10 Jan 2022 12:12:20 +0100 Subject: [PATCH 4/4] chore: ensure lf but not crlf for build scripts --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitattributes b/.gitattributes index 434addf38..b14bc05ff 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,7 @@ * text=auto *.* text eol=lf +./build text eol=lf +scripts/* text eol=lf *.cmd text eol=crlf *.jpg -text *.png -text