yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: change to build .tar.gz instead of zip

This commit is contained in:
Zaiming (Stone) Shi 2022-01-12 21:13:31 +01:00
parent d9946ffa5b
commit 638a9d1af7
4 changed files with 51 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
Makefile
View File

@ -152,17 +152,17 @@ define gen-relup-target
$1-relup: $(COMMON_DEPS)
@$(BUILD) $1 relup
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-relup-target,$(zt))))
ALL_TGZS = $(REL_PROFILES)
$(foreach zt,$(ALL_TGZS),$(eval $(call gen-relup-target,$(zt))))
## zip target is to create a release package .zip with relup
.PHONY: $(REL_PROFILES:%=%-zip)
define gen-zip-target
$1-zip: $1-relup
@$(BUILD) $1 zip
## tgz target is to create a release package .tar.gz with relup
.PHONY: $(REL_PROFILES:%=%-tgz)
define gen-tgz-target
$1-tgz: $1-relup
@$(BUILD) $1 tgz
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-zip-target,$(zt))))
ALL_TGZS = $(REL_PROFILES)
$(foreach zt,$(ALL_TGZS),$(eval $(call gen-tgz-target,$(zt))))
## A pkg target depend on a regular release
.PHONY: $(PKG_PROFILES)
@ -185,20 +185,20 @@ define gen-docker-target
$1-docker: $(COMMON_DEPS)
@$(BUILD) $1 docker
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
ALL_TGZS = $(REL_PROFILES)
$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target,$(zt))))
## emqx-docker-testing
## emqx-enterprise-docker-testing
## is to directly copy a unzipped zip-package to a
## is to directly copy a extracted tgz-package to a
## base image such as ubuntu20.04. Mostly for testing
.PHONY: $(REL_PROFILES:%=%-docker-testing)
define gen-docker-target-testing
$1-docker-testing: $(COMMON_DEPS)
@$(BUILD) $1 docker-testing
endef
ALL_ZIPS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt))))
ALL_TGZS = $(REL_PROFILES)
$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt))))
conf-segs:
@scripts/merge-config.escript

62
build
View File

@ -2,7 +2,7 @@
# This script helps to build release artifacts.
# arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
# arg2: artifact, e.g. rel | relup | zip | pkg
# arg2: artifact, e.g. rel | relup | tgz | pkg
set -euo pipefail
@ -79,28 +79,18 @@ make_rel() {
fi
}
## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup
## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup
make_relup() {
local lib_dir="_build/$PROFILE/rel/emqx/lib"
local releases_dir="_build/$PROFILE/rel/emqx/releases"
mkdir -p "$lib_dir" "$releases_dir"
local rel_dir="_build/$PROFILE/rel/emqx"
mkdir -p "${rel_dir}/lib"
mkdir -p "${rel_dir}/releases"
local releases=()
if [ -d "$releases_dir" ]; then
while read -r zip; do
while read -r tgzfile ; do
local base_vsn
base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+|(-(alpha|beta)\.[0-9]))(-[0-9a-e]{8})?")"
if [ ! -d "$releases_dir/$base_vsn" ]; then
local tmp_dir
tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir"
unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir"
cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir"
cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir"
rm -rf "$tmp_dir"
fi
base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.[0-9])?(-[0-9a-f]{8})?" | head -1)"
tar -C "$rel_dir" -zxf ---keep-old-files "$tgzfile" emqx/releases emqx/lib
releases+=( "$base_vsn" )
done < <("$FIND" _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f)
fi
done < <("$FIND" _upgrade_base -maxdepth 1 -name "$PROFILE*${SYSTEM}-${ARCH}.tar.gz" -type f)
if [ ${#releases[@]} -eq 0 ]; then
log "No upgrade base found, relup ignored"
return 0
@ -126,9 +116,9 @@ cp_dyn_libs() {
| sort -u)
}
## make_zip turns .tar.gz into a .zip with a slightly different name.
## Re-pack the relx assembled .tar.gz to EMQ X's package naming scheme
## It assumes the .tar.gz has been built -- relies on Makefile dependency
make_zip() {
make_tgz() {
# build the tarball again to ensure relup is included
make_rel
@ -142,14 +132,14 @@ make_zip() {
if [ ! -f "$tarball" ]; then
log "ERROR: $tarball is not found"
fi
local zipball
zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
local target
target="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
tar zxf "${tarball}" -C "${tard}/emqx"
## try to be portable for zip packages.
## try to be portable for tar.gz packages.
## for DEB and RPM packages the dependencies are resoved by yum and apt
cp_dyn_libs "${tard}/emqx"
(cd "${tard}" && zip -qr - emqx) > "${zipball}"
log "Zip package successfully created: ${zipball}"
(cd "${tard}" && tar -cz emqx) > "${target}"
log "Tarball successfully repacked: ${target}"
}
## This function builds the default docker image based on alpine:3.14 (by default)
@ -166,7 +156,7 @@ make_docker() {
}
## This function accepts any base docker image,
## a emqx zip-image, and a image tag (for the image to be built),
## a emqx tgz-image, and a image tag (for the image to be built),
## to build a docker image which runs EMQ X
##
## Export below variables to quickly build an image
@ -174,7 +164,7 @@ make_docker() {
## Name Default Example
## ---------------------------------------------------------------------
## EMQX_BASE_IMAGE current os centos:7
## EMQX_ZIP_PACKAGE _packages/<current-zip-target> /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.zip
## EMQX_TGZ_packages/<current-tgz-target> /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.tar.gz
## EMQX_IMAGE_TAG emqx/emqx:<current-vns-rel> emqx/emqx:testing-tag
##
make_docker_testing() {
@ -193,17 +183,17 @@ make_docker_testing() {
esac
fi
EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}"
local defaultzip
defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip"
local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}"
if [ ! -f "$zip" ]; then
log "ERROR: $zip not built?"
local default_tgz
default_tgz="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
local tgz="${EMQX_TGZ_PACKAGE:-$default_tgz}"
if [ ! -f "$tgz" ]; then
log "ERROR: $tgz not built?"
exit 1
fi
set -x
docker build \
--build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \
--build-arg EMQX_ZIP_PACKAGE="${zip}" \
--build-arg EMQX_TGZ_PACKAGE="${tgz}" \
--tag "$EMQX_IMAGE_TAG" \
-f "${DOCKERFILE_TESTING}" .
}
@ -220,8 +210,8 @@ case "$ARTIFACT" in
relup)
make_relup
;;
zip)
make_zip
tgz)
make_tgz
;;
pkg)
if [ -z "${PKGERDIR:-}" ]; then

9
scripts/buildx.sh
View File

@ -8,7 +8,7 @@
## i.e. will not work if docker command has to be executed with sudo
## example:
## ./scripts/buildx.sh --profile emqx --pkgtype zip --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10
## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10
set -euo pipefail
@ -16,7 +16,8 @@ help() {
echo
echo "-h|--help: To display this usage information"
echo "--profile <PROFILE>: EMQ X profile to build, e.g. emqx, emqx-edge"
echo "--pkgtype zip|pkg: Specify which package to build, zip for .zip and pkg for .rpm or .deb"
echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz"
echo " and pkg for .rpm or .deb"
echo "--arch amd64|arm64: Target arch to build the EMQ X package for"
echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull"
@ -62,8 +63,8 @@ if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [
exit 1
fi
if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then
echo "Bad --pkgtype option, should be zip or pkg"
if [ "$PKGTYPE" != 'tgz' ] && [ "$PKGTYPE" != 'pkg' ]; then
echo "Bad --pkgtype option, should be tgz or pkg"
exit 1
fi

8
scripts/update_appup.escript
View File

@ -32,8 +32,8 @@ Options:
--make-command A command used to assemble the release
--release-dir Release directory
--src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/'
--binary-rel-url Binary release URL pattern. %TAG% variable is substituted with the release tag.
E.g. \"https://github.com/emqx/emqx/releases/download/v%TAG%/emqx-centos7-%TAG%-amd64.zip\"
--binary-rel-url Binary release URL pattern. %VSN% variable is substituted with the version in release tag.
E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.1.5-3-centos7-amd64.tar.gz\"
".
-record(app,
@ -172,8 +172,8 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) -
Dir = filename:basename(Repo, ".git") ++ [$-|Tag],
Filename = filename:join(BaseDir, Dir),
Script = "mkdir -p ${OUTFILE} &&
wget -c -O ${OUTFILE}.zip ${URL} &&
unzip -n -d ${OUTFILE} ${OUTFILE}.zip",
wget -c -O ${OUTFILE}.tar.gz ${URL} &&
tar -zxf ${OUTFILE} ${OUTFILE}.tar.gz",
Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}],
bash(Script, Env),
{ok, Filename}.