Merge pull request #6687 from zmstone/build-doc-for-the-win

fix: [5.0] windows build

.gitattributes

@@ -1,5 +1,7 @@
 * text=auto
 *.* text eol=lf
+./build text eol=lf
+scripts/* text eol=lf
 *.cmd text eol=crlf
 *.jpg -text
 *.png -text

.github/workflows/build_packages.yaml

@@ -59,6 +59,7 @@ jobs:
         profile: # only CE for windows
           - emqx
         otp:
+          ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows
           - 23.2
 
     steps:
@@ -71,7 +72,6 @@ jobs:
     - uses: ilammy/msvc-dev-cmd@v1
     - uses: gleam-lang/setup-erlang@v1.1.2
       id: install_erlang
-      ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows
       with:
         otp-version: ${{ matrix.otp }}
     - name: build

Makefile

@@ -12,6 +12,9 @@ export DOCKERFILE := deploy/docker/Dockerfile
 export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
+	FIND=/usr/bin/find
+else
+	FIND=find
 endif
 
 PROFILE ?= emqx
@@ -103,8 +106,8 @@ $(PROFILES:%=clean-%):
 	@if [ -d _build/$(@:clean-%=%) ]; then \
 		rm rebar.lock \
 		rm -rf _build/$(@:clean-%=%)/rel; \
-		find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
+		$(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
-		find _build/$(@:clean-%=%)  -type l -delete; \
+		$(FIND) _build/$(@:clean-%=%)  -type l -delete; \
 	fi
 
 .PHONY: clean-all

Windows.md

@@ -25,7 +25,7 @@ C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build
 
 Depending on your visual studio version and OS, the paths may differ.
 The first path is for rebar3 port compiler to find `cl.exe` and `link.exe`
-The second path is for Powershell or CMD to setup environment variables.
+The second path is for CMD to setup environment variables.
 
 ### Erlang/OTP
 
@@ -73,6 +73,7 @@ Some of the unix world tools are required to build EMQ X.  Including:
 * git
 * curl
 * make
+* cmake
 * jq
 * zip / unzip
 
@@ -81,14 +82,14 @@ We recommend using [scoop](https://scoop.sh/), or [Chocolatey](https://chocolate
 When using scoop:
 
 ```
-scoop install git curl make jq zip unzip
+scoop install git curl make cmake jq zip unzip
 ```
 
 ## Build EMQ X source code
 
 * Clone the repo: `git clone https://github.com/emqx/emqx.git`
 
-* Start CMD or Powershell
+* Start CMD console
 
 * Execute `vcvarsall.bat x86_amd64` to load environment variables
 

build

@@ -45,6 +45,13 @@ if [ "$(uname -s)" = 'Linux' ]; then
     esac
 fi
 
+if [ "${SYSTEM}" = 'windows' ]; then
+    # windows does not like the find
+    FIND="/usr/bin/find"
+else
+    FIND='find'
+fi
+
 log() {
     local msg="$1"
     # rebar3 prints ===>, so we print ===<
@@ -53,21 +60,20 @@ log() {
 
 make_doc() {
     local libs_dir1 libs_dir2
-    libs_dir1="$(find "_build/default/lib/" -maxdepth 2 -name ebin -type d)"
+    libs_dir1="$("$FIND" "_build/default/lib/" -maxdepth 2 -name ebin -type d)"
-    libs_dir2="$(find "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)"
+    libs_dir2="$("$FIND" "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)"
-
-    local conf_doc_md
-    # TODO render md as html
-    conf_doc_md="$(pwd)/_build/${PROFILE}/lib/emqx_dashboard/priv/config.md"
-    echo "===< Generating config document $conf_doc_md"
     # shellcheck disable=SC2086
-    erl -noshell -pa $libs_dir1 $libs_dir2 -eval "ok = emqx_conf:gen_doc(\"${conf_doc_md}\"), halt(0)."
+    erl -noshell -pa $libs_dir1 $libs_dir2 -eval \
+        "F = filename:join(['_build', '${PROFILE}', lib, emqx_dashboard, priv, 'config.md']), \
+         io:format(\"===< Generating: ~s~n\", [F]),
+         ok = emqx_conf:gen_doc(F), \
+         halt(0)."
 }
 
 make_rel() {
     # shellcheck disable=SC1010
     ./rebar3 as "$PROFILE" do tar
-    if [ "$(find "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then
+    if [ "$("$FIND" "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then
         echo "gpb should not be included in the release"
         exit 1
     fi
@@ -93,7 +99,7 @@ make_relup() {
                 rm -rf "$tmp_dir"
             fi
             releases+=( "$base_vsn" )
-        done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f)
+        done < <("$FIND" _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f)
     fi
     if [ ${#releases[@]} -eq 0 ]; then
         log "No upgrade base found, relup ignored"
@@ -113,7 +119,7 @@ cp_dyn_libs() {
     mkdir -p "$target_dir"
     while read -r so_file; do
         cp -L "$so_file" "$target_dir/"
-    done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
+    done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
         | xargs -0 ldd \
         | grep -E '(libcrypto)|(libtinfo)' \
         | awk '{print $3}' \

rebar.config.erl

@@ -122,42 +122,42 @@ profiles() ->
        , {relx, relx(Vsn, cloud, bin, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
-       , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx doc"}]}
+       , {post_hooks, [{compile, "bash build emqx doc"}]}
        ]}
     , {'emqx-pkg',
        [ {erl_opts, prod_compile_opts()}
        , {relx, relx(Vsn, cloud, pkg, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
-       , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx-pkg doc"}]}
+       , {post_hooks, [{compile, "bash build emqx-pkg doc"}]}
        ]}
     , {'emqx-enterprise',
        [ {erl_opts, prod_compile_opts()}
        , {relx, relx(Vsn, cloud, bin, ee)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ee)}
-       , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx-enterprise doc"}]}
+       , {post_hooks, [{compile, "bash build emqx-enterprise doc"}]}
        ]}
     , {'emqx-enterprise-pkg',
        [ {erl_opts, prod_compile_opts()}
        , {relx, relx(Vsn, cloud, pkg, ee)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ee)}
-       , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx-enterprise-pkg doc"}]}
+       , {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]}
        ]}
     , {'emqx-edge',
        [ {erl_opts, prod_compile_opts()}
        , {relx, relx(Vsn, edge, bin, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
-       , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx-edge doc"}]}
+       , {post_hooks, [{compile, "bash build emqx-edge doc"}]}
        ]}
     , {'emqx-edge-pkg',
        [ {erl_opts, prod_compile_opts()}
        , {relx, relx(Vsn, edge, pkg, ce)}
        , {overrides, prod_overrides()}
        , {project_app_dirs, project_app_dirs(ce)}
-       , {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "./build emqx-edge-pkg doc"}]}
+       , {post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]}
        ]}
     , {check,
        [ {erl_opts, common_compile_opts()}

scripts/find-apps.sh

@@ -5,9 +5,16 @@ set -euo pipefail
 # ensure dir
 cd -P -- "$(dirname -- "$0")/.."
 
+if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then
+    # Otherwise windows may resolve to find.exe
+    FIND="/usr/bin/find"
+else
+    FIND='find'
+fi
+
 find_app() {
     local appdir="$1"
-    find "${appdir}" -mindepth 1 -maxdepth 1 -type d
+    "$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d
 }
 
 find_app 'apps'

scripts/get-distro.sh

@@ -5,18 +5,27 @@
 
 set -euo pipefail
 
-if [ "$(uname -s)" = 'Darwin' ]; then
+UNAME="$(uname -s)"
-    DIST='macos'
+
-    VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g')
+case "$UNAME" in
-    SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')"
+    Darwin)
-elif [ "$(uname -s)" = 'Linux' ]; then
+        DIST='macos'
-    if grep -q -i 'centos' /etc/*-release; then
+        VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g')
-        DIST='centos'
+        SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')"
-        VERSION_ID="$(rpm --eval '%{centos_ver}')"
+        ;;
-    else
+    Linux)
-        DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')"
+        if grep -q -i 'centos' /etc/*-release; then
-        VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
+            DIST='centos'
-    fi
+            VERSION_ID="$(rpm --eval '%{centos_ver}')"
-    SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')"
+        else
-fi
+            DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')"
+            VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
+        fi
+        SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')"
+        ;;
+    CYGWIN*|MSYS*|MINGW*)
+        SYSTEM="windows"
+        ;;
+esac
+
 echo "$SYSTEM"