build: fix windows build

This commit is contained in:
Zaiming (Stone) Shi 2022-02-20 09:19:56 +01:00
parent e12065f361
commit 20ad5d52db
11 changed files with 162 additions and 130 deletions

View File

@ -15,7 +15,7 @@ if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then
exit 1
fi
PROFILE="${1}"
export PROFILE="${1}"
PACKAGE_TYPE="${2}"
export CODE_PATH=${CODE_PATH:-"/emqx"}
@ -24,8 +24,6 @@ export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
# export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1"
# export EMQX_NODE_COOKIE=$(date +%s%N)
PKG_VSN="$($CODE_PATH/pkg-vsn.sh)"
OTP_VSN="$($CODE_PATH/scripts/get-otp-vsn.sh)"
SYSTEM="$($CODE_PATH/scripts/get-distro.sh)"
if [ "$PACKAGE_TYPE" = 'zip' ]; then
@ -41,21 +39,8 @@ else
esac
fi
ARCH="$(uname -m)"
case "$ARCH" in
x86_64)
ARCH='amd64'
;;
aarch64)
ARCH='arm64'
;;
arm*)
ARCH=arm
;;
esac
export ARCH
PACKAGE_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
PACKAGE_NAME="$($CODE_PATH/scripts/get-pkg-name.sh)"
OLD_PACKAGE_PATTERN="$($CODE_PATH/scripts/get-pkg-name.sh 'vsn_matcher')"
PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}"
PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}"
@ -178,34 +163,33 @@ running_test(){
relup_test(){
TARGET_VERSION="$("$CODE_PATH"/pkg-vsn.sh)"
if [ -d "${RELUP_PACKAGE_PATH}" ];then
cd "${RELUP_PACKAGE_PATH}"
find . -maxdepth 1 -name "${PROFILE}-*-${ARCH}.zip" |
while read -r pkg; do
packagename=$(basename "${pkg}")
unzip -q "$packagename"
if ! ./emqx/bin/emqx start; then
cat emqx/log/erlang.log.1 || true
cat emqx/log/emqx.log.1 || true
exit 1
fi
./emqx/bin/emqx_ctl status
./emqx/bin/emqx versions
cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*-"${ARCH}".zip ./emqx/releases
./emqx/bin/emqx install "${TARGET_VERSION}"
[ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
export EMQX_WAIT_FOR_STOP=300
./emqx/bin/emqx_ctl status
if ! ./emqx/bin/emqx stop; then
cat emqx/log/erlang.log.1 || true
cat emqx/log/emqx.log.1 || true
echo "failed to stop emqx"
exit 1
fi
rm -rf emqx
done
fi
if [ ! -d "${RELUP_PACKAGE_PATH}" ];then
return 0
fi
cd "${RELUP_PACKAGE_PATH}"
while read -r pkg; do
packagename=$(basename "${pkg}")
unzip -q "$packagename"
if ! ./emqx/bin/emqx start; then
cat emqx/log/erlang.log.1 || true
cat emqx/log/emqx.log.1 || true
exit 1
fi
./emqx/bin/emqx_ctl status
./emqx/bin/emqx versions
cp "${PACKAGE_PATH}/${PROFILE}-${TARGET_VERSION}"-*.zip ./emqx/releases/
./emqx/bin/emqx install "${TARGET_VERSION}"
[ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
export EMQX_WAIT_FOR_STOP=300
./emqx/bin/emqx_ctl status
if ! ./emqx/bin/emqx stop; then
cat emqx/log/erlang.log.1 || true
cat emqx/log/emqx.log.1 || true
echo "failed to stop emqx"
exit 1
fi
rm -rf emqx
done < <(find . -maxdepth 1 -name "${OLD_PACKAGE_PATTERN}.zip")
}
emqx_prepare

1
.gitattributes vendored
View File

@ -1,3 +1,4 @@
build text eol=lf
* text=auto
*.* text eol=lf
*.jpg -text

View File

@ -95,15 +95,6 @@ jobs:
run: |
$env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH"
erl -eval "erlang:display(crypto:info_lib())" -s init stop
$version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
$regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
$pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip"
}
else {
$pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip"
}
## We do not build/release bcrypt for windows package
Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
if (Test-Path rebar.lock) {
@ -115,11 +106,7 @@ jobs:
head -2 rebar3
which rebar3
rebar3 --help
make ${{ matrix.profile }}
mkdir -p _packages/${{ matrix.profile }}
Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name
mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }}
sha256sum "_packages/${{ matrix.profile }}/$pkg_name" | head -c 64 > "_packages/${{ matrix.profile }}/${pkg_name}.sha256"
make ${{ matrix.profile }}-zip
- name: run emqx
timeout-minutes: 1
working-directory: source
@ -214,8 +201,6 @@ jobs:
exit 1
fi
rm -rf emqx
#sha256sum $pkg_name | head -c64 > $pkg_name.sha256
openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256
- uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/')
with:

View File

@ -12,6 +12,9 @@ export DOCKERFILE := deploy/docker/Dockerfile
export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
ifeq ($(OS),Windows_NT)
export REBAR_COLOR=none
FIND=/usr/bin/find
else
FIND=find
endif
PROFILE ?= emqx
@ -91,8 +94,8 @@ $(PROFILES:%=clean-%):
@if [ -d _build/$(@:clean-%=%) ]; then \
rm rebar.lock \
rm -rf _build/$(@:clean-%=%)/rel; \
find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
find _build/$(@:clean-%=%) -type l -delete; \
$(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
$(FIND) _build/$(@:clean-%=%) -type l -delete; \
fi
.PHONY: clean-all

65
build
View File

@ -6,7 +6,7 @@
set -euo pipefail
PROFILE="$1"
export PROFILE="$1"
ARTIFACT="$2"
# ensure dir
@ -17,20 +17,6 @@ export PKG_VSN
SYSTEM="$(./scripts/get-distro.sh)"
ARCH="$(uname -m)"
case "$ARCH" in
x86_64)
ARCH='amd64'
;;
aarch64)
ARCH='arm64'
;;
arm*)
ARCH=arm
;;
esac
export ARCH
##
## Support RPM and Debian based linux systems
##
@ -45,6 +31,13 @@ if [ "$(uname -s)" = 'Linux' ]; then
esac
fi
if [ "${SYSTEM}" = 'windows' ]; then
# windows does not like the find
FIND="/usr/bin/find"
else
FIND='find'
fi
log() {
local msg="$1"
# rebar3 prints ===>, so we print ===<
@ -60,7 +53,8 @@ make_rel() {
make_relup() {
local lib_dir="_build/$PROFILE/rel/emqx/lib"
local releases_dir="_build/$PROFILE/rel/emqx/releases"
mkdir -p "$lib_dir" "$releases_dir"
local name_pattern="$(./scripts/get-pkg-name.sh 'vsn_matcher')"
mkdir -p "$lib_dir" "$releases_dir" '_upgrade_base'
local releases=()
if [ -d "$releases_dir" ]; then
while read -r zip; do
@ -76,7 +70,7 @@ make_relup() {
rm -rf "$tmp_dir"
fi
releases+=( "$base_vsn" )
done < <(find _upgrade_base -maxdepth 1 -name "${PROFILE}-*-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" -type f)
done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.zip" -type f)
fi
if [ ${#releases[@]} -eq 0 ]; then
log "No upgrade base found, relup ignored"
@ -96,7 +90,7 @@ cp_dyn_libs() {
mkdir -p "$target_dir"
while read -r so_file; do
cp -L "$so_file" "$target_dir/"
done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
| xargs -0 ldd \
| grep -E '(libcrypto)|(libtinfo)' \
| awk '{print $3}' \
@ -106,26 +100,25 @@ cp_dyn_libs() {
## make_zip turns .tar.gz into a .zip with a slightly different name.
## It assumes the .tar.gz has been built -- relies on Makefile dependency
make_zip() {
# build the tarball again to ensure relup is included
make_rel
tard="/tmp/emqx_untar_${PKG_VSN}"
rm -rf "${tard}"
mkdir -p "${tard}/emqx"
local relpath="_build/${PROFILE}/rel/emqx"
local relpath="_build/${PROFILE}/rel/"
local pkgpath="_packages/${PROFILE}"
mkdir -p "${pkgpath}"
local tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
if [ ! -f "$tarball" ]; then
log "ERROR: $tarball is not found"
fi
local zipball
zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
tar zxf "${tarball}" -C "${tard}/emqx"
local pkgname="$(./scripts/get-pkg-name.sh).zip"
local pkgpathname="${pkgpath}/${pkgname}"
# ensure release dir is clean
rm -rf "${relpath}/emqx"
# just make release, no tar
./rebar3 as "$PROFILE" release
## try to be portable for zip packages.
## for DEB and RPM packages the dependencies are resoved by yum and apt
cp_dyn_libs "${tard}/emqx"
(cd "${tard}" && zip -qr - emqx) > "${zipball}"
cp_dyn_libs "${relpath}/emqx"
pushd "$relpath" >/dev/null
zip -qr "${pkgname}" emqx
popd >/dev/null
mv "${relpath}/${pkgname}" "${pkgpathname}"
# sha256sum may not be available in some os
openssl dgst -sha256 "${pkgpathname}" | cut -d ' ' -f 2 > ${pkgpathname}.sha256
log "Zip package successfully created: ${pkgpathname}"
log "Zip package sha256sum: $(cat ${pkgpathname}.sha256)"
}
## This function builds the default docker image based on alpine:3.14 (by default)
@ -170,7 +163,7 @@ make_docker_testing() {
fi
EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}"
local defaultzip
defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
defaultzip="_packages/${PROFILE}/$(scripts/get-pkg-name.sh).zip"
local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}"
if [ ! -f "$zip" ]; then
log "ERROR: $zip not built?"

View File

@ -22,6 +22,8 @@ all: | $(BUILT)
cd $(SRCDIR) && dpkg-buildpackage -us -uc
mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)
cp $(SRCDIR)/../$(SOURCE_PKG).deb $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb
openssl dgst -sha256 $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb | cut -d ' ' -f 2 > \
$(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).deb.sha256
$(BUILT):
mkdir -p $(TOPDIR) $(SRCDIR)

View File

@ -48,6 +48,8 @@ all: | $(BUILT)
emqx.spec
mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME)
cp $(TOPDIR)/RPMS/$(shell uname -m)/$(SOURCE_PKG).rpm $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm
openssl dgst -sha256 $(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm | cut -d ' ' -f 2 > \
$(EMQX_REL)/_packages/$(EMQX_NAME)/$(TARGET_PKG).rpm.sha256
$(BUILT):
mkdir -p $(TOPDIR) $(SRCDIR) $(SRCDIR)/BUILT

View File

@ -2,7 +2,21 @@
set -euo pipefail
VERSION="3.14.3-emqx-8"
## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5.
## we have to use an otp24-compiled rebar3 because the defination of record #application{}
## in systools.hrl is changed in otp24.
case ${OTP_VSN} in
23*)
VERSION="3.14.3-emqx-8"
;;
24*)
VERSION="3.18.0-emqx-1"
;;
*)
echo "Unsupporetd Erlang/OTP version $OTP_VSN"
exit 1
;;
esac
# ensure dir
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
@ -14,10 +28,6 @@ download() {
curl -f -L "${DOWNLOAD_URL}/${VERSION}/rebar3" -o ./rebar3
}
version_gte() {
test "$(printf '%s\n' "$1" "$2" | sort -V | head -n 1)" = "$2"
}
# get the version number from the second line of the escript
# because command `rebar3 -v` tries to load rebar.config
# which is slow and may print some logs
@ -25,16 +35,7 @@ version() {
head -n 2 ./rebar3 | tail -n 1 | tr ' ' '\n' | grep -E '^.+-emqx-.+'
}
echo "OTP_VSN: ${OTP_VSN}"
if version_gte "${OTP_VSN}" "24.0"; then
## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5.
## we have to use an otp24-compiled rebar3 because the defination of record #application{}
## in systools.hrl is changed in otp24.
VERSION="3.18.0-emqx-1"
fi
if [ -f 'rebar3' ] && [ "$(version)" = "$VERSION" ]; then
echo "rebar3 ${VERSION} already exists"
exit 0
fi

View File

@ -5,9 +5,16 @@ set -euo pipefail
# ensure dir
cd -P -- "$(dirname -- "$0")/.."
if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then
# Otherwise windows may resolve to find.exe
FIND="/usr/bin/find"
else
FIND='find'
fi
find_app() {
local appdir="$1"
find "${appdir}" -mindepth 1 -maxdepth 1 -type d
"$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d
}
# append emqx application first
@ -23,4 +30,4 @@ fi
## find directories in lib-extra
find_app 'lib-extra'
## find symlinks in lib-extra
find 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print
"$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print

View File

@ -5,18 +5,26 @@
set -euo pipefail
if [ "$(uname -s)" = 'Darwin' ]; then
DIST='macos'
VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g')
SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')"
elif [ "$(uname -s)" = 'Linux' ]; then
if grep -q -i 'rhel' /etc/*-release; then
DIST='el'
VERSION_ID="$(rpm --eval '%{rhel}')"
else
DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')"
VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
fi
SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')"
fi
UNAME="$(uname -s)"
case "$UNAME" in
Darwin)
DIST='macos'
VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g')
SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')"
;;
Linux)
if grep -q -i 'rhel' /etc/*-release; then
DIST='el'
VERSION_ID="$(rpm --eval '%{rhel}')"
else
DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')"
VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
fi
SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')"
;;
CYGWIN*|MSYS*|MINGW*)
SYSTEM="windows"
;;
esac
echo "$SYSTEM"

46
scripts/get-pkg-name.sh Executable file
View File

@ -0,0 +1,46 @@
#!/usr/bin/env bash
## This script print the package name based on the
## current build environment
## NOTE: the package name does not include .zip, .rpm or .deb suffix
## Arg 1 is either 'vsn_exact' (default) or 'vsn_matcher'
## when 'vsn_exact' is given, the version number is the output of pkg-vsn.sh
## otherwise '*' is used for 'find' command to find old versions (as upgrade base)
set -euo pipefail
VSN_MATCH="${1:-vsn_exact}"
if [ "${VSN_MATCH}" = 'vsn_exact' ]; then
PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
elif [ "${VSN_MATCH}" = 'vsn_matcher' ]; then
PKG_VSN='*'
else
echo "$0 ERROR: badarg"
exit 1
fi
if [ -z "${PROFILE}" ]; then
echo "environment variable PROFILE not set"
exit 2
fi
OTP_VSN="${OTP_VSN:-$(./scrippts/get-otp-vsn.sh)}"
SYSTEM="$(./scripts/get-distro.sh)"
ARCH="$(uname -m)"
case "$ARCH" in
x86_64)
ARCH='amd64'
;;
aarch64)
ARCH='arm64'
;;
arm*)
ARCH=arm
;;
esac
export ARCH
echo "${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"