yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: change to build .tar.gz instead of zip

This commit is contained in:
Zaiming (Stone) Shi 2022-01-12 21:13:31 +01:00
parent d9946ffa5b
commit 638a9d1af7
4 changed files with 51 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
Makefile
View File

@ -152,17 +152,17 @@ define gen-relup-target
$1-relup: $(COMMON_DEPS) $1-relup: $(COMMON_DEPS)
@$(BUILD) $1 relup @$(BUILD) $1 relup
endef endef
ALL_ZIPS = $(REL_PROFILES) ALL_TGZS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-relup-target,$(zt)))) $(foreach zt,$(ALL_TGZS),$(eval $(call gen-relup-target,$(zt))))
## zip target is to create a release package .zip with relup ## tgz target is to create a release package .tar.gz with relup
.PHONY: $(REL_PROFILES:%=%-zip) .PHONY: $(REL_PROFILES:%=%-tgz)
define gen-zip-target define gen-tgz-target
$1-zip: $1-relup $1-tgz: $1-relup
@$(BUILD) $1 zip @$(BUILD) $1 tgz
endef endef
ALL_ZIPS = $(REL_PROFILES) ALL_TGZS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-zip-target,$(zt)))) $(foreach zt,$(ALL_TGZS),$(eval $(call gen-tgz-target,$(zt))))
## A pkg target depend on a regular release ## A pkg target depend on a regular release
.PHONY: $(PKG_PROFILES) .PHONY: $(PKG_PROFILES)
@ -185,20 +185,20 @@ define gen-docker-target
$1-docker: $(COMMON_DEPS) $1-docker: $(COMMON_DEPS)
@$(BUILD) $1 docker @$(BUILD) $1 docker
endef endef
ALL_ZIPS = $(REL_PROFILES) ALL_TGZS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) $(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target,$(zt))))
## emqx-docker-testing ## emqx-docker-testing
## emqx-enterprise-docker-testing ## emqx-enterprise-docker-testing
## is to directly copy a unzipped zip-package to a ## is to directly copy a extracted tgz-package to a
## base image such as ubuntu20.04. Mostly for testing ## base image such as ubuntu20.04. Mostly for testing
.PHONY: $(REL_PROFILES:%=%-docker-testing) .PHONY: $(REL_PROFILES:%=%-docker-testing)
define gen-docker-target-testing define gen-docker-target-testing
$1-docker-testing: $(COMMON_DEPS) $1-docker-testing: $(COMMON_DEPS)
@$(BUILD) $1 docker-testing @$(BUILD) $1 docker-testing
endef endef
ALL_ZIPS = $(REL_PROFILES) ALL_TGZS = $(REL_PROFILES)
$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt)))) $(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt))))
conf-segs: conf-segs:
@scripts/merge-config.escript @scripts/merge-config.escript

66
build
View File

@ -2,7 +2,7 @@
# This script helps to build release artifacts. # This script helps to build release artifacts.
# arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
# arg2: artifact, e.g. rel | relup | zip | pkg # arg2: artifact, e.g. rel | relup | tgz | pkg
set -euo pipefail set -euo pipefail
@ -79,28 +79,18 @@ make_rel() {
fi fi
} }
## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup ## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup
make_relup() { make_relup() {
local lib_dir="_build/$PROFILE/rel/emqx/lib" local rel_dir="_build/$PROFILE/rel/emqx"
local releases_dir="_build/$PROFILE/rel/emqx/releases" mkdir -p "${rel_dir}/lib"
mkdir -p "$lib_dir" "$releases_dir" mkdir -p "${rel_dir}/releases"
local releases=() local releases=()
if [ -d "$releases_dir" ]; then while read -r tgzfile ; do
while read -r zip; do local base_vsn
local base_vsn base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.[0-9])?(-[0-9a-f]{8})?" | head -1)"
base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+|(-(alpha|beta)\.[0-9]))(-[0-9a-e]{8})?")" tar -C "$rel_dir" -zxf ---keep-old-files "$tgzfile" emqx/releases emqx/lib
if [ ! -d "$releases_dir/$base_vsn" ]; then releases+=( "$base_vsn" )
local tmp_dir done < <("$FIND" _upgrade_base -maxdepth 1 -name "$PROFILE*${SYSTEM}-${ARCH}.tar.gz" -type f)
tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir"
unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir"
cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir"
cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir"
rm -rf "$tmp_dir"
fi
releases+=( "$base_vsn" )
done < <("$FIND" _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f)
fi
if [ ${#releases[@]} -eq 0 ]; then if [ ${#releases[@]} -eq 0 ]; then
log "No upgrade base found, relup ignored" log "No upgrade base found, relup ignored"
return 0 return 0
@ -126,9 +116,9 @@ cp_dyn_libs() {
| sort -u) | sort -u)
} }
## make_zip turns .tar.gz into a .zip with a slightly different name. ## Re-pack the relx assembled .tar.gz to EMQ X's package naming scheme
## It assumes the .tar.gz has been built -- relies on Makefile dependency ## It assumes the .tar.gz has been built -- relies on Makefile dependency
make_zip() { make_tgz() {
# build the tarball again to ensure relup is included # build the tarball again to ensure relup is included
make_rel make_rel
@ -142,14 +132,14 @@ make_zip() {
if [ ! -f "$tarball" ]; then if [ ! -f "$tarball" ]; then
log "ERROR: $tarball is not found" log "ERROR: $tarball is not found"
fi fi
local zipball local target
zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" target="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
tar zxf "${tarball}" -C "${tard}/emqx" tar zxf "${tarball}" -C "${tard}/emqx"
## try to be portable for zip packages. ## try to be portable for tar.gz packages.
## for DEB and RPM packages the dependencies are resoved by yum and apt ## for DEB and RPM packages the dependencies are resoved by yum and apt
cp_dyn_libs "${tard}/emqx" cp_dyn_libs "${tard}/emqx"
(cd "${tard}" && zip -qr - emqx) > "${zipball}" (cd "${tard}" && tar -cz emqx) > "${target}"
log "Zip package successfully created: ${zipball}" log "Tarball successfully repacked: ${target}"
} }
## This function builds the default docker image based on alpine:3.14 (by default) ## This function builds the default docker image based on alpine:3.14 (by default)
@ -166,7 +156,7 @@ make_docker() {
} }
## This function accepts any base docker image, ## This function accepts any base docker image,
## a emqx zip-image, and a image tag (for the image to be built), ## a emqx tgz-image, and a image tag (for the image to be built),
## to build a docker image which runs EMQ X ## to build a docker image which runs EMQ X
## ##
## Export below variables to quickly build an image ## Export below variables to quickly build an image
@ -174,7 +164,7 @@ make_docker() {
## Name Default Example ## Name Default Example
## --------------------------------------------------------------------- ## ---------------------------------------------------------------------
## EMQX_BASE_IMAGE current os centos:7 ## EMQX_BASE_IMAGE current os centos:7
## EMQX_ZIP_PACKAGE _packages/<current-zip-target> /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.zip ## EMQX_TGZ_packages/<current-tgz-target> /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.tar.gz
## EMQX_IMAGE_TAG emqx/emqx:<current-vns-rel> emqx/emqx:testing-tag ## EMQX_IMAGE_TAG emqx/emqx:<current-vns-rel> emqx/emqx:testing-tag
## ##
make_docker_testing() { make_docker_testing() {
@ -193,17 +183,17 @@ make_docker_testing() {
esac esac
fi fi
EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}"
local defaultzip local default_tgz
defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" default_tgz="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}" local tgz="${EMQX_TGZ_PACKAGE:-$default_tgz}"
if [ ! -f "$zip" ]; then if [ ! -f "$tgz" ]; then
log "ERROR: $zip not built?" log "ERROR: $tgz not built?"
exit 1 exit 1
fi fi
set -x set -x
docker build \ docker build \
--build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \ --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \
--build-arg EMQX_ZIP_PACKAGE="${zip}" \ --build-arg EMQX_TGZ_PACKAGE="${tgz}" \
--tag "$EMQX_IMAGE_TAG" \ --tag "$EMQX_IMAGE_TAG" \
-f "${DOCKERFILE_TESTING}" . -f "${DOCKERFILE_TESTING}" .
} }
@ -220,8 +210,8 @@ case "$ARTIFACT" in
relup) relup)
make_relup make_relup
;; ;;
zip) tgz)
make_zip make_tgz
;; ;;
pkg) pkg)
if [ -z "${PKGERDIR:-}" ]; then if [ -z "${PKGERDIR:-}" ]; then

9
scripts/buildx.sh
View File

@ -8,7 +8,7 @@
## i.e. will not work if docker command has to be executed with sudo ## i.e. will not work if docker command has to be executed with sudo
## example: ## example:
## ./scripts/buildx.sh --profile emqx --pkgtype zip --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10
set -euo pipefail set -euo pipefail
@ -16,7 +16,8 @@ help() {
echo echo
echo "-h|--help: To display this usage information" echo "-h|--help: To display this usage information"
echo "--profile <PROFILE>: EMQ X profile to build, e.g. emqx, emqx-edge" echo "--profile <PROFILE>: EMQ X profile to build, e.g. emqx, emqx-edge"
echo "--pkgtype zip|pkg: Specify which package to build, zip for .zip and pkg for .rpm or .deb" echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz"
echo " and pkg for .rpm or .deb"
echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--arch amd64|arm64: Target arch to build the EMQ X package for"
echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD" echo "--src_dir <SRC_DIR>: EMQ X source ode in this dir, default to PWD"
echo "--builder <BUILDER>: Builder image to pull" echo "--builder <BUILDER>: Builder image to pull"
@ -62,8 +63,8 @@ if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [
exit 1 exit 1
fi fi
if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then if [ "$PKGTYPE" != 'tgz' ] && [ "$PKGTYPE" != 'pkg' ]; then
echo "Bad --pkgtype option, should be zip or pkg" echo "Bad --pkgtype option, should be tgz or pkg"
exit 1 exit 1
fi fi

8
scripts/update_appup.escript
View File

@ -32,8 +32,8 @@ Options:
--make-command A command used to assemble the release --make-command A command used to assemble the release
--release-dir Release directory --release-dir Release directory
--src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/'
--binary-rel-url Binary release URL pattern. %TAG% variable is substituted with the release tag. --binary-rel-url Binary release URL pattern. %VSN% variable is substituted with the version in release tag.
E.g. \"https://github.com/emqx/emqx/releases/download/v%TAG%/emqx-centos7-%TAG%-amd64.zip\" E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.1.5-3-centos7-amd64.tar.gz\"
". ".
-record(app, -record(app,
@ -172,8 +172,8 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) -
Dir = filename:basename(Repo, ".git") ++ [$-|Tag], Dir = filename:basename(Repo, ".git") ++ [$-|Tag],
Filename = filename:join(BaseDir, Dir), Filename = filename:join(BaseDir, Dir),
Script = "mkdir -p ${OUTFILE} && Script = "mkdir -p ${OUTFILE} &&
wget -c -O ${OUTFILE}.zip ${URL} && wget -c -O ${OUTFILE}.tar.gz ${URL} &&
unzip -n -d ${OUTFILE} ${OUTFILE}.zip", tar -zxf ${OUTFILE} ${OUTFILE}.tar.gz",
Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}], Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}],
bash(Script, Env), bash(Script, Env),
{ok, Filename}. {ok, Filename}.