From 20ad5d52db683e9a1eb9e31aa165a1f1f324fde0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 20 Feb 2022 09:19:56 +0100 Subject: [PATCH] build: fix windows build --- .ci/build_packages/tests.sh | 76 +++++++++++---------------- .gitattributes | 1 + .github/workflows/build_packages.yaml | 17 +----- Makefile | 7 ++- build | 65 ++++++++++------------- deploy/packages/deb/Makefile | 2 + deploy/packages/rpm/Makefile | 2 + scripts/ensure-rebar3.sh | 29 +++++----- scripts/find-apps.sh | 11 +++- scripts/get-distro.sh | 36 ++++++++----- scripts/get-pkg-name.sh | 46 ++++++++++++++++ 11 files changed, 162 insertions(+), 130 deletions(-) create mode 100755 scripts/get-pkg-name.sh diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index c5b6ec539..d9ca89bee 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -15,7 +15,7 @@ if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then exit 1 fi -PROFILE="${1}" +export PROFILE="${1}" PACKAGE_TYPE="${2}" export CODE_PATH=${CODE_PATH:-"/emqx"} @@ -24,8 +24,6 @@ export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" # export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1" # export EMQX_NODE_COOKIE=$(date +%s%N) -PKG_VSN="$($CODE_PATH/pkg-vsn.sh)" -OTP_VSN="$($CODE_PATH/scripts/get-otp-vsn.sh)" SYSTEM="$($CODE_PATH/scripts/get-distro.sh)" if [ "$PACKAGE_TYPE" = 'zip' ]; then @@ -41,21 +39,8 @@ else esac fi -ARCH="$(uname -m)" -case "$ARCH" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm - ;; -esac -export ARCH - -PACKAGE_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +PACKAGE_NAME="$($CODE_PATH/scripts/get-pkg-name.sh)" +OLD_PACKAGE_PATTERN="$($CODE_PATH/scripts/get-pkg-name.sh 'vsn_matcher')" PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" @@ -178,34 +163,33 @@ running_test(){ relup_test(){ TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh)" - if [ -d "${RELUP_PACKAGE_PATH}" ];then - cd "${RELUP_PACKAGE_PATH}" - - find . -maxdepth 1 -name "${PROFILE}-*-${ARCH}.zip" | - while read -r pkg; do - packagename=$(basename "${pkg}") - unzip -q "$packagename" - if ! ./emqx/bin/emqx start; then - cat emqx/log/erlang.log.1 || true - cat emqx/log/emqx.log.1 || true - exit 1 - fi - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*-"${ARCH}".zip ./emqx/releases - ./emqx/bin/emqx install "${TARGET_VERSION}" - [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 - export EMQX_WAIT_FOR_STOP=300 - ./emqx/bin/emqx_ctl status - if ! ./emqx/bin/emqx stop; then - cat emqx/log/erlang.log.1 || true - cat emqx/log/emqx.log.1 || true - echo "failed to stop emqx" - exit 1 - fi - rm -rf emqx - done - fi + if [ ! -d "${RELUP_PACKAGE_PATH}" ];then + return 0 + fi + cd "${RELUP_PACKAGE_PATH}" + while read -r pkg; do + packagename=$(basename "${pkg}") + unzip -q "$packagename" + if ! ./emqx/bin/emqx start; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + exit 1 + fi + ./emqx/bin/emqx_ctl status + ./emqx/bin/emqx versions + cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*.zip ./emqx/releases/ + ./emqx/bin/emqx install "${TARGET_VERSION}" + [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 + export EMQX_WAIT_FOR_STOP=300 + ./emqx/bin/emqx_ctl status + if ! ./emqx/bin/emqx stop; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + echo "failed to stop emqx" + exit 1 + fi + rm -rf emqx + done < <(find . -maxdepth 1 -name "${OLD_PACKAGE_PATTERN}.zip") } emqx_prepare diff --git a/.gitattributes b/.gitattributes index 4ed73da9a..8ecb2ae1a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ +build text eol=lf * text=auto *.* text eol=lf *.jpg -text diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 39b387e1c..6ce28c028 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -95,15 +95,6 @@ jobs: run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" erl -eval "erlang:display(crypto:info_lib())" -s init stop - - $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) - if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { - $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" - $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip" - } - else { - $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" - } ## We do not build/release bcrypt for windows package Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ if (Test-Path rebar.lock) { @@ -115,11 +106,7 @@ jobs: head -2 rebar3 which rebar3 rebar3 --help - make ${{ matrix.profile }} - mkdir -p _packages/${{ matrix.profile }} - Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name - mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }} - sha256sum "_packages/${{ matrix.profile }}/$pkg_name" | head -c 64 > "_packages/${{ matrix.profile }}/${pkg_name}.sha256" + make ${{ matrix.profile }}-zip - name: run emqx timeout-minutes: 1 working-directory: source @@ -214,8 +201,6 @@ jobs: exit 1 fi rm -rf emqx - #sha256sum $pkg_name | head -c64 > $pkg_name.sha256 - openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256 - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: diff --git a/Makefile b/Makefile index 30e8b4c0a..703a2d551 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ export DOCKERFILE := deploy/docker/Dockerfile export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none + FIND=/usr/bin/find +else + FIND=find endif PROFILE ?= emqx @@ -91,8 +94,8 @@ $(PROFILES:%=clean-%): @if [ -d _build/$(@:clean-%=%) ]; then \ rm rebar.lock \ rm -rf _build/$(@:clean-%=%)/rel; \ - find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ - find _build/$(@:clean-%=%) -type l -delete; \ + $(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ + $(FIND) _build/$(@:clean-%=%) -type l -delete; \ fi .PHONY: clean-all diff --git a/build b/build index 2aa20b7a3..f7bdbf472 100755 --- a/build +++ b/build @@ -6,7 +6,7 @@ set -euo pipefail -PROFILE="$1" +export PROFILE="$1" ARTIFACT="$2" # ensure dir @@ -17,20 +17,6 @@ export PKG_VSN SYSTEM="$(./scripts/get-distro.sh)" -ARCH="$(uname -m)" -case "$ARCH" in - x86_64) - ARCH='amd64' - ;; - aarch64) - ARCH='arm64' - ;; - arm*) - ARCH=arm - ;; -esac -export ARCH - ## ## Support RPM and Debian based linux systems ## @@ -45,6 +31,13 @@ if [ "$(uname -s)" = 'Linux' ]; then esac fi +if [ "${SYSTEM}" = 'windows' ]; then + # windows does not like the find + FIND="/usr/bin/find" +else + FIND='find' +fi + log() { local msg="$1" # rebar3 prints ===>, so we print ===< @@ -60,7 +53,8 @@ make_rel() { make_relup() { local lib_dir="_build/$PROFILE/rel/emqx/lib" local releases_dir="_build/$PROFILE/rel/emqx/releases" - mkdir -p "$lib_dir" "$releases_dir" + local name_pattern="$(./scripts/get-pkg-name.sh 'vsn_matcher')" + mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base' local releases=() if [ -d "$releases_dir" ]; then while read -r zip; do @@ -76,7 +70,7 @@ make_relup() { rm -rf "$tmp_dir" fi releases+=( "$base_vsn" ) - done < <(find _upgrade_base -maxdepth 1 -name "${PROFILE}-*-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" -type f) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.zip" -type f) fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -96,7 +90,7 @@ cp_dyn_libs() { mkdir -p "$target_dir" while read -r so_file; do cp -L "$so_file" "$target_dir/" - done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ + done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ | xargs -0 ldd \ | grep -E '(libcrypto)|(libtinfo)' \ | awk '{print $3}' \ @@ -106,26 +100,25 @@ cp_dyn_libs() { ## make_zip turns .tar.gz into a .zip with a slightly different name. ## It assumes the .tar.gz has been built -- relies on Makefile dependency make_zip() { - # build the tarball again to ensure relup is included - make_rel - - tard="/tmp/emqx_untar_${PKG_VSN}" - rm -rf "${tard}" - mkdir -p "${tard}/emqx" - local relpath="_build/${PROFILE}/rel/emqx" + local relpath="_build/${PROFILE}/rel/" local pkgpath="_packages/${PROFILE}" - mkdir -p "${pkgpath}" - local tarball="${relpath}/emqx-${PKG_VSN}.tar.gz" - if [ ! -f "$tarball" ]; then - log "ERROR: $tarball is not found" - fi - local zipball - zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" - tar zxf "${tarball}" -C "${tard}/emqx" + local pkgname="$(./scripts/get-pkg-name.sh).zip" + local pkgpathname="${pkgpath}/${pkgname}" + # ensure release dir is clean + rm -rf "${relpath}/emqx" + # just make release, no tar + ./rebar3 as "$PROFILE" release ## try to be portable for zip packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt - cp_dyn_libs "${tard}/emqx" - (cd "${tard}" && zip -qr - emqx) > "${zipball}" + cp_dyn_libs "${relpath}/emqx" + pushd "$relpath" >/dev/null + zip -qr "${pkgname}" emqx + popd >/dev/null + mv "${relpath}/${pkgname}" "${pkgpathname}" + # sha256sum may not be available in some os + openssl dgst -sha256 "${pkgpathname}" | cut -d ' ' -f 2 > ${pkgpathname}.sha256 + log "Zip package successfully created: ${pkgpathname}" + log "Zip package sha256sum: $(cat ${pkgpathname}.sha256)" } ## This function builds the default docker image based on alpine:3.14 (by default) @@ -170,7 +163,7 @@ make_docker_testing() { fi EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" local defaultzip - defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" + defaultzip="_packages/${PROFILE}/$(scripts/get-pkg-name.sh).zip" local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}" if [ ! -f "$zip" ]; then log "ERROR: $zip not built?" diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 2cb3679ee..9c7eaa438 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -22,6 +22,8 @@ all: | $(BUILT) cd $(SRCDIR) && dpkg-buildpackage -us -uc mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb + openssl dgst -sha256 $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb | cut -d ' ' -f 2 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb.sha256 $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index acee8b51c..657c5192e 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -48,6 +48,8 @@ all: | $(BUILT) emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm + openssl dgst -sha256 $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm | cut -d ' ' -f 2 > \ + $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm.sha256 $(BUILT): mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index 89058a298..e52a82821 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -2,7 +2,21 @@ set -euo pipefail -VERSION="3.14.3-emqx-8" +## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5. +## we have to use an otp24-compiled rebar3 because the defination of record #application{} +## in systools.hrl is changed in otp24. +case ${OTP_VSN} in + 23*) + VERSION="3.14.3-emqx-8" + ;; + 24*) + VERSION="3.18.0-emqx-1" + ;; + *) + echo "Unsupporetd Erlang/OTP version $OTP_VSN" + exit 1 + ;; +esac # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." @@ -14,10 +28,6 @@ download() { curl -f -L "${DOWNLOAD_URL}/${VERSION}/rebar3" -o ./rebar3 } -version_gte() { - test "$(printf '%s\n' "$1" "$2" | sort -V | head -n 1)" = "$2" -} - # get the version number from the second line of the escript # because command `rebar3 -v` tries to load rebar.config # which is slow and may print some logs @@ -25,16 +35,7 @@ version() { head -n 2 ./rebar3 | tail -n 1 | tr ' ' '\n' | grep -E '^.+-emqx-.+' } -echo "OTP_VSN: ${OTP_VSN}" -if version_gte "${OTP_VSN}" "24.0"; then - ## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5. - ## we have to use an otp24-compiled rebar3 because the defination of record #application{} - ## in systools.hrl is changed in otp24. - VERSION="3.18.0-emqx-1" -fi - if [ -f 'rebar3' ] && [ "$(version)" = "$VERSION" ]; then - echo "rebar3 ${VERSION} already exists" exit 0 fi diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index fabec239e..c4a0eec62 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -5,9 +5,16 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")/.." +if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then + # Otherwise windows may resolve to find.exe + FIND="/usr/bin/find" +else + FIND='find' +fi + find_app() { local appdir="$1" - find "${appdir}" -mindepth 1 -maxdepth 1 -type d + "$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d } # append emqx application first @@ -23,4 +30,4 @@ fi ## find directories in lib-extra find_app 'lib-extra' ## find symlinks in lib-extra -find 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print +"$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh index f4ac5c88c..d017a7d9d 100755 --- a/scripts/get-distro.sh +++ b/scripts/get-distro.sh @@ -5,18 +5,26 @@ set -euo pipefail -if [ "$(uname -s)" = 'Darwin' ]; then - DIST='macos' - VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') - SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" -elif [ "$(uname -s)" = 'Linux' ]; then - if grep -q -i 'rhel' /etc/*-release; then - DIST='el' - VERSION_ID="$(rpm --eval '%{rhel}')" - else - DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" - VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" - fi - SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" -fi +UNAME="$(uname -s)" + +case "$UNAME" in + Darwin) + DIST='macos' + VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') + SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" + ;; + Linux) + if grep -q -i 'rhel' /etc/*-release; then + DIST='el' + VERSION_ID="$(rpm --eval '%{rhel}')" + else + DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" + VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" + fi + SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" + ;; + CYGWIN*|MSYS*|MINGW*) + SYSTEM="windows" + ;; +esac echo "$SYSTEM" diff --git a/scripts/get-pkg-name.sh b/scripts/get-pkg-name.sh new file mode 100755 index 000000000..6c2dcde5a --- /dev/null +++ b/scripts/get-pkg-name.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +## This script print the package name based on the +## current build environment +## NOTE: the package name does not include .zip, .rpm or .deb suffix + +## Arg 1 is either 'vsn_exact' (default) or 'vsn_matcher' +## when 'vsn_exact' is given, the version number is the output of pkg-vsn.sh +## otherwise '*' is used for 'find' command to find old versions (as upgrade base) + +set -euo pipefail + +VSN_MATCH="${1:-vsn_exact}" + +if [ "${VSN_MATCH}" = 'vsn_exact' ]; then + PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" +elif [ "${VSN_MATCH}" = 'vsn_matcher' ]; then + PKG_VSN='*' +else + echo "$0 ERROR: badarg" + exit 1 +fi + +if [ -z "${PROFILE}" ]; then + echo "environment variable PROFILE not set" + exit 2 +fi + +OTP_VSN="${OTP_VSN:-$(./scrippts/get-otp-vsn.sh)}" +SYSTEM="$(./scripts/get-distro.sh)" + +ARCH="$(uname -m)" +case "$ARCH" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac +export ARCH + +echo "${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"