From d9946ffa5b4c1dfb6f0df9afa612ba5f5d8b12a4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 12 Jan 2022 22:42:07 +0100 Subject: [PATCH 1/6] chore: Update EMQ X description in RPM and DEB packages --- deploy/packages/deb/debian/control | 2 +- deploy/packages/rpm/emqx.spec | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/packages/deb/debian/control b/deploy/packages/deb/debian/control index b3794036c..66b58047f 100644 --- a/deploy/packages/deb/debian/control +++ b/deploy/packages/deb/debian/control @@ -9,4 +9,4 @@ Homepage: https://www.emqx.com Package: emqx Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} -Description: EMQX, a distributed, massively scalable, highly extensible MQTT message broker written in Erlang/OTP +Description: EMQ X, a distributed, massively scalable, highly extensible MQTT message broker. diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index c6eb56a6f..c6fc60e64 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -14,7 +14,7 @@ Release: %{_release}%{?dist} Summary: emqx Group: System Environment/Daemons License: Apache License Version 2.0 -URL: https://www.emqx.io +URL: https://www.emqx.com BuildRoot: %{_tmppath}/%{_name}-%{_version}-root Provides: %{_name} AutoReq: 0 @@ -26,7 +26,7 @@ Requires: libatomic %endif %description -EMQX, a distributed, massively scalable, highly extensible MQTT message broker written in Erlang/OTP. +EMQ X, a distributed, massively scalable, highly extensible MQTT message broker. %prep From 638a9d1af7f059b82108ed32b2a55099b1ec2a15 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 12 Jan 2022 21:13:31 +0100 Subject: [PATCH 2/6] build: change to build .tar.gz instead of zip --- Makefile | 28 +++++++-------- build | 66 +++++++++++++++--------------------- scripts/buildx.sh | 9 ++--- scripts/update_appup.escript | 8 ++--- 4 files changed, 51 insertions(+), 60 deletions(-) diff --git a/Makefile b/Makefile index 33052d6f9..6dc1d0664 100644 --- a/Makefile +++ b/Makefile @@ -152,17 +152,17 @@ define gen-relup-target $1-relup: $(COMMON_DEPS) @$(BUILD) $1 relup endef -ALL_ZIPS = $(REL_PROFILES) -$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-relup-target,$(zt)))) +ALL_TGZS = $(REL_PROFILES) +$(foreach zt,$(ALL_TGZS),$(eval $(call gen-relup-target,$(zt)))) -## zip target is to create a release package .zip with relup -.PHONY: $(REL_PROFILES:%=%-zip) -define gen-zip-target -$1-zip: $1-relup - @$(BUILD) $1 zip +## tgz target is to create a release package .tar.gz with relup +.PHONY: $(REL_PROFILES:%=%-tgz) +define gen-tgz-target +$1-tgz: $1-relup + @$(BUILD) $1 tgz endef -ALL_ZIPS = $(REL_PROFILES) -$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-zip-target,$(zt)))) +ALL_TGZS = $(REL_PROFILES) +$(foreach zt,$(ALL_TGZS),$(eval $(call gen-tgz-target,$(zt)))) ## A pkg target depend on a regular release .PHONY: $(PKG_PROFILES) @@ -185,20 +185,20 @@ define gen-docker-target $1-docker: $(COMMON_DEPS) @$(BUILD) $1 docker endef -ALL_ZIPS = $(REL_PROFILES) -$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) +ALL_TGZS = $(REL_PROFILES) +$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target,$(zt)))) ## emqx-docker-testing ## emqx-enterprise-docker-testing -## is to directly copy a unzipped zip-package to a +## is to directly copy a extracted tgz-package to a ## base image such as ubuntu20.04. Mostly for testing .PHONY: $(REL_PROFILES:%=%-docker-testing) define gen-docker-target-testing $1-docker-testing: $(COMMON_DEPS) @$(BUILD) $1 docker-testing endef -ALL_ZIPS = $(REL_PROFILES) -$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt)))) +ALL_TGZS = $(REL_PROFILES) +$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt)))) conf-segs: @scripts/merge-config.escript diff --git a/build b/build index a30d7629d..a0c7b4f85 100755 --- a/build +++ b/build @@ -2,7 +2,7 @@ # This script helps to build release artifacts. # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg -# arg2: artifact, e.g. rel | relup | zip | pkg +# arg2: artifact, e.g. rel | relup | tgz | pkg set -euo pipefail @@ -79,28 +79,18 @@ make_rel() { fi } -## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup +## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup make_relup() { - local lib_dir="_build/$PROFILE/rel/emqx/lib" - local releases_dir="_build/$PROFILE/rel/emqx/releases" - mkdir -p "$lib_dir" "$releases_dir" + local rel_dir="_build/$PROFILE/rel/emqx" + mkdir -p "${rel_dir}/lib" + mkdir -p "${rel_dir}/releases" local releases=() - if [ -d "$releases_dir" ]; then - while read -r zip; do - local base_vsn - base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+|(-(alpha|beta)\.[0-9]))(-[0-9a-e]{8})?")" - if [ ! -d "$releases_dir/$base_vsn" ]; then - local tmp_dir - tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" - unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir" - unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir" - cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" - cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" - rm -rf "$tmp_dir" - fi - releases+=( "$base_vsn" ) - done < <("$FIND" _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f) - fi + while read -r tgzfile ; do + local base_vsn + base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.[0-9])?(-[0-9a-f]{8})?" | head -1)" + tar -C "$rel_dir" -zxf ---keep-old-files "$tgzfile" emqx/releases emqx/lib + releases+=( "$base_vsn" ) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "$PROFILE*${SYSTEM}-${ARCH}.tar.gz" -type f) if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" return 0 @@ -126,9 +116,9 @@ cp_dyn_libs() { | sort -u) } -## make_zip turns .tar.gz into a .zip with a slightly different name. +## Re-pack the relx assembled .tar.gz to EMQ X's package naming scheme ## It assumes the .tar.gz has been built -- relies on Makefile dependency -make_zip() { +make_tgz() { # build the tarball again to ensure relup is included make_rel @@ -142,14 +132,14 @@ make_zip() { if [ ! -f "$tarball" ]; then log "ERROR: $tarball is not found" fi - local zipball - zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" + local target + target="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" tar zxf "${tarball}" -C "${tard}/emqx" - ## try to be portable for zip packages. + ## try to be portable for tar.gz packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt cp_dyn_libs "${tard}/emqx" - (cd "${tard}" && zip -qr - emqx) > "${zipball}" - log "Zip package successfully created: ${zipball}" + (cd "${tard}" && tar -cz emqx) > "${target}" + log "Tarball successfully repacked: ${target}" } ## This function builds the default docker image based on alpine:3.14 (by default) @@ -166,7 +156,7 @@ make_docker() { } ## This function accepts any base docker image, -## a emqx zip-image, and a image tag (for the image to be built), +## a emqx tgz-image, and a image tag (for the image to be built), ## to build a docker image which runs EMQ X ## ## Export below variables to quickly build an image @@ -174,7 +164,7 @@ make_docker() { ## Name Default Example ## --------------------------------------------------------------------- ## EMQX_BASE_IMAGE current os centos:7 -## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.zip +## EMQX_TGZ_packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.tar.gz ## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag ## make_docker_testing() { @@ -193,17 +183,17 @@ make_docker_testing() { esac fi EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" - local defaultzip - defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" - local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}" - if [ ! -f "$zip" ]; then - log "ERROR: $zip not built?" + local default_tgz + default_tgz="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + local tgz="${EMQX_TGZ_PACKAGE:-$default_tgz}" + if [ ! -f "$tgz" ]; then + log "ERROR: $tgz not built?" exit 1 fi set -x docker build \ --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \ - --build-arg EMQX_ZIP_PACKAGE="${zip}" \ + --build-arg EMQX_TGZ_PACKAGE="${tgz}" \ --tag "$EMQX_IMAGE_TAG" \ -f "${DOCKERFILE_TESTING}" . } @@ -220,8 +210,8 @@ case "$ARTIFACT" in relup) make_relup ;; - zip) - make_zip + tgz) + make_tgz ;; pkg) if [ -z "${PKGERDIR:-}" ]; then diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 7c49ce909..1ed0fd56d 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 +## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 set -euo pipefail @@ -16,7 +16,8 @@ help() { echo echo "-h|--help: To display this usage information" echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" - echo "--pkgtype zip|pkg: Specify which package to build, zip for .zip and pkg for .rpm or .deb" + echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz" + echo " and pkg for .rpm or .deb" echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" @@ -62,8 +63,8 @@ if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ exit 1 fi -if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then - echo "Bad --pkgtype option, should be zip or pkg" +if [ "$PKGTYPE" != 'tgz' ] && [ "$PKGTYPE" != 'pkg' ]; then + echo "Bad --pkgtype option, should be tgz or pkg" exit 1 fi diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 262ff906f..a713ca8b0 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -32,8 +32,8 @@ Options: --make-command A command used to assemble the release --release-dir Release directory --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' - --binary-rel-url Binary release URL pattern. %TAG% variable is substituted with the release tag. - E.g. \"https://github.com/emqx/emqx/releases/download/v%TAG%/emqx-centos7-%TAG%-amd64.zip\" + --binary-rel-url Binary release URL pattern. %VSN% variable is substituted with the version in release tag. + E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.1.5-3-centos7-amd64.tar.gz\" ". -record(app, @@ -172,8 +172,8 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) - Dir = filename:basename(Repo, ".git") ++ [$-|Tag], Filename = filename:join(BaseDir, Dir), Script = "mkdir -p ${OUTFILE} && - wget -c -O ${OUTFILE}.zip ${URL} && - unzip -n -d ${OUTFILE} ${OUTFILE}.zip", + wget -c -O ${OUTFILE}.tar.gz ${URL} && + tar -zxf ${OUTFILE} ${OUTFILE}.tar.gz", Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}], bash(Script, Env), {ok, Filename}. From 1abac892c928c01a49d86624609bc230bfca3eac Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 12 Jan 2022 23:19:51 +0100 Subject: [PATCH 3/6] refactor: install_upgrade script no need to re-pack OTP release handler only works with .tar.gz as a result, we had to unpack .zip and re-pack .tar.gz just to make OTP happy. Now since we will be building .tar.gz packages there is no need for this re-pack step anymore --- bin/emqx | 4 ---- bin/install_upgrade.escript | 17 ----------------- 2 files changed, 21 deletions(-) diff --git a/bin/emqx b/bin/emqx index c62e42e66..8bd3640fa 100755 --- a/bin/emqx +++ b/bin/emqx @@ -149,7 +149,6 @@ usage() { echo "release package tarball has already been deployed at one" echo "of the following locations:" echo " releases/-.tar.gz" - echo " releases/-.zip" ;; install) echo "Usage: $REL_NAME install [VERSION]" @@ -157,7 +156,6 @@ usage() { echo "release package tarball has already been deployed at one" echo "of the following locations:" echo " releases/-.tar.gz" - echo " releases/-.zip" echo "" echo " --no-permanent Install release package VERSION but" echo " don't make it permanent" @@ -173,7 +171,6 @@ usage() { echo "that a release package tarball has already been deployed at one" echo "of the following locations:" echo " releases/-.tar.gz" - echo " releases/-.zip" echo "" echo " --no-permanent Install release package VERSION but" echo " don't make it permanent" @@ -184,7 +181,6 @@ usage() { echo "that a release package tarball has already been deployed at one" echo "of the following locations:" echo " releases/-.tar.gz" - echo " releases/-.zip" echo "" echo " --no-permanent Install release package VERSION but" echo " don't make it permanent" diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 74b6da6e2..bcefaa641 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -184,7 +184,6 @@ find_and_link_release_package(Version, RelName) -> %% we've found where the actual release package is located ReleaseLink = filename:join(["releases", Version, RelNameStr ++ ".tar.gz"]), - ok = unpack_zipballs(RelNameStr, Version), TarBalls = [ filename:join(["releases", RelNameStr ++ "-" ++ Version ++ ".tar.gz"]), @@ -219,22 +218,6 @@ find_and_link_release_package(Version, RelName) -> {Filename, ReleaseHandlerPackageLink} end. -unpack_zipballs(RelNameStr, Version) -> - {ok, Cwd} = file:get_cwd(), - GzFile = filename:absname(filename:join(["releases", RelNameStr ++ "-" ++ Version ++ ".tar.gz"])), - ZipFiles = filelib:wildcard(filename:join(["releases", RelNameStr ++ "-*" ++ Version ++ "*.zip"])), - ?INFO("unzip ~p", [ZipFiles]), - [begin - TmdTarD="/tmp/emqx_untar_" ++ integer_to_list(erlang:system_time()), - ok = filelib:ensure_dir(filename:join([TmdTarD, "dummy"])), - {ok, _} = file:copy(Zip, filename:join([TmdTarD, "emqx.zip"])), - ok = file:set_cwd(filename:join([TmdTarD])), - {ok, _FileList} = zip:unzip("emqx.zip"), - ok = file:set_cwd(filename:join([TmdTarD, "emqx"])), - ok = erl_tar:create(GzFile, filelib:wildcard("*"), [compressed]) - end || Zip <- ZipFiles], - file:set_cwd(Cwd). - first_value(_Fun, []) -> no_value; first_value(Fun, [Value | Rest]) -> case Fun(Value) of From ecd7ea0a2f62c9dd237ab14d08a67deff264c101 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 12 Jan 2022 23:18:30 +0100 Subject: [PATCH 4/6] ci: change to build and test with .tar.gz instead of .zip packages --- .ci/build_packages/tests.sh | 23 +++++++------- .ci/fvt_tests/relup.lux | 6 ++-- .github/workflows/build_packages.yaml | 28 ++++++++--------- .github/workflows/build_slim_packages.yaml | 16 +++++----- .github/workflows/run_api_tests.yaml | 35 ++++++++++++---------- .github/workflows/run_relup_tests.yaml | 8 ++--- 6 files changed, 60 insertions(+), 56 deletions(-) diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 5fead0139..73622495d 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -4,12 +4,12 @@ set -euo pipefail set -x if [ -z "${1:-}" ]; then - echo "Usage $0 zip|pkg" + echo "Usage $0 tgz|pkg" exit 1 fi -if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then - echo "Usage $0 zip|pkg" +if [ "${2:-}" != 'tgz' ] && [ "${2:-}" != 'pkg' ]; then + echo "Usage $0 tgz|pkg" exit 1 fi @@ -22,8 +22,8 @@ export EMQX_NAME=${EMQX_NAME:-"emqx"} export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" -if [ "$PACKAGE_TYPE" = 'zip' ]; then - PKG_SUFFIX="zip" +if [ "$PACKAGE_TYPE" = 'tgz' ]; then + PKG_SUFFIX="tar.gz" else SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)" case "${SYSTEM:-}" in @@ -37,9 +37,10 @@ else fi PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" -PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}.${PKG_SUFFIX}" +PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" if ! [ -f "$PACKAGE_FILE" ]; then echo "$PACKAGE_FILE is not a file" + exit 1 fi case "$(uname -m)" in @@ -68,8 +69,8 @@ emqx_test(){ cd "${PACKAGE_PATH}" local packagename="${PACKAGE_FILE_NAME}" case "$PKG_SUFFIX" in - "zip") - unzip -q "${PACKAGE_PATH}/${packagename}" + "tar.gz") + tar -zxf "${PACKAGE_PATH}/${packagename}" export EMQX_ZONES__DEFAULT__MQTT__SERVER_KEEPALIVE=60 export EMQX_MQTT__MAX_TOPIC_ALIAS=10 export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug @@ -205,10 +206,10 @@ relup_test(){ if [ -d "${RELUP_PACKAGE_PATH}" ];then cd "${RELUP_PACKAGE_PATH}" - find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" | + find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.tar.gz" | while read -r pkg; do packagename=$(basename "${pkg}") - unzip "$packagename" + tar -zxf "$packagename" if ! ./emqx/bin/emqx start; then cat emqx/log/erlang.log.1 || true cat emqx/log/emqx.log.1 || true @@ -216,7 +217,7 @@ relup_test(){ fi ./emqx/bin/emqx_ctl status ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases + cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".tar.gz ./emqx/releases ./emqx/bin/emqx install "${TARGET_VERSION}" [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 ./emqx/bin/emqx_ctl status diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index f5b36c648..4a8894476 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -22,7 +22,7 @@ [shell emqx] !cd $PACKAGE_PATH - !unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed -r 's/[v|e]//g')-amd64.zip + !tar -zxf ${PROFILE}-$(echo $old_vsn | sed -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz ?SH-PROMPT !cd emqx @@ -82,7 +82,7 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/ !./bin/emqx install $VSN ?Made release permanent: "$VSN" @@ -107,7 +107,7 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/ !./bin/emqx install $VSN ?Made release permanent: "$VSN" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 6076b49a5..956cfbc0e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -85,11 +85,11 @@ jobs: $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" - $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip" - } + $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.tar.gz" + } else { - $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" - } + $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.tar.gz" + } ## We do not build/release bcrypt and quic for windows package Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ Remove-Item -Recurse -Force -Path _build/default/lib/quicer/ @@ -192,12 +192,12 @@ jobs: make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 rm -rf _build/${{ matrix.profile }}/lib - make ${{ matrix.profile }}-zip + make ${{ matrix.profile }}-tgz - name: test working-directory: source run: | - pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) - unzip -q $pkg_name + pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz) + tar -zxf $pkg_name # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' @@ -297,7 +297,7 @@ jobs: with: path: source/_build/default/lib/quicer/ key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} - - name: downloads old emqx zip packages + - name: download old emqx tgz packages env: OTP_VSN: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} @@ -320,10 +320,10 @@ jobs: old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" - if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256 - echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1 + if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz) | grep -oE "^[23]+")" ]; then + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz.sha256 + echo "$(cat $package_name.tar.gz.sha256) $package_name.tar.gz" | sha256sum -c || exit 1 fi done - name: build emqx packages @@ -336,10 +336,10 @@ jobs: run: | ./scripts/buildx.sh \ --profile "${PROFILE}" \ - --pkgtype "zip" \ + --pkgtype "tgz" \ --arch "${ARCH}" \ --builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}" - ## the pkg build is incremental on the zip build + ## the pkg build is incremental on the tgz build ./scripts/buildx.sh \ --profile "${PROFILE}" \ --pkgtype "pkg" \ diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index dd8d2568d..1694c177a 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -63,10 +63,10 @@ jobs: with: path: _build/default/lib/quicer/ key: ${{ matrix.os }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} - - name: build and test zip package + - name: build and test tgz package run: | - make ${EMQX_NAME}-zip - .ci/build_packages/tests.sh "$EMQX_PKG_NAME" zip + make ${EMQX_NAME}-tgz + .ci/build_packages/tests.sh "$EMQX_PKG_NAME" tgz - name: run static checks if: contains(matrix.os, 'ubuntu') run: | @@ -78,7 +78,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} - path: _packages/**/*.zip + path: _packages/${{ matrix.profile}}/*.tar.gz mac: strategy: fail-fast: false @@ -140,11 +140,11 @@ jobs: . $HOME/.kerl/${{ matrix.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 - make ${{ matrix.profile }}-zip + make ${{ matrix.profile }}-tgz - name: test run: | - pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) - unzip -q $pkg_name + pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz) + tar -zxf $pkg_name # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' @@ -166,4 +166,4 @@ jobs: - uses: actions/upload-artifact@v2 with: name: macos - path: _packages/**/*.zip + path: _packages/**/*.tar.gz diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index 084704537..d3dd485b1 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -18,21 +18,18 @@ jobs: steps: - uses: actions/checkout@v2 - - name: zip emqx-broker - if: endsWith(github.repository, 'emqx') + - name: build EMQ X CE and EE tgz package run: | - make emqx-zip - - name: zip emqx-broker - if: endsWith(github.repository, 'enterprise') - run: | - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - make emqx-enterprise-zip + make emqx-tgz + make emqx-enterprise-tgz - uses: actions/upload-artifact@v2 with: - name: emqx-broker - path: _packages/**/*.zip - + name: emqx-ce + path: _packages/emqx/*.tar.gz + - uses: actions/upload-artifact@v2 + with: + name: emqx-ee + path: _packages/emqx-enterprise/*.tar.gz api-test: needs: build runs-on: ubuntu-latest @@ -57,6 +54,9 @@ jobs: - api_topic_rewrite - api_event_message - api_stats + edition: + - emqx-ce + #TODO: add emqx-ee steps: - uses: actions/checkout@v2 with: @@ -70,13 +70,13 @@ jobs: architecture: x64 # (x64 or x86) - defaults to x64 - uses: actions/download-artifact@v2 with: - name: emqx-broker + name: ${{ matrix.edition }} path: . - - name: start emqx-broker + - name: start EMQ X service env: EMQX_LISTENERS__WSS__DEFAULT__BIND: "0.0.0.0:8085" run: | - unzip ./emqx/*.zip + tar -zxf ./*.tar.gz ./emqx/bin/emqx start - name: install jmeter timeout-minutes: 10 @@ -117,4 +117,7 @@ jobs: steps: - uses: geekyeggo/delete-artifact@v1 with: - name: emqx-broker + name: emqx-ce + - uses: geekyeggo/delete-artifact@v1 + with: + name: emqx-ee diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 3b796d476..766484e36 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -85,10 +85,10 @@ jobs: cd emqx/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for old_vsn in ${old_vsns[@]}; do - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.tar.gz done - name: build emqx - run: make -C emqx ${PROFILE}-zip + run: make -C emqx ${PROFILE}-tgz - name: build emqtt-bench run: make -C emqtt-bench - name: build lux @@ -105,8 +105,8 @@ jobs: set -e -x -u if [ -n "$OLD_VSNS" ]; then mkdir -p packages - cp emqx/_packages/${PROFILE}/*.zip packages - cp emqx/_upgrade_base/*.zip packages + cp emqx/_packages/${PROFILE}/*.tar.gz packages + cp emqx/_upgrade_base/*.tar.gz packages lux \ --case_timeout infinity \ --var PROFILE=$PROFILE \ From 2faa3fb4cd5cad05dc73bac1f67173e463e7932e Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 13 Jan 2022 17:14:36 +0100 Subject: [PATCH 5/6] ci: build test package from latest builder image --- .github/workflows/run_api_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index d3dd485b1..e6558b74b 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -14,7 +14,7 @@ on: jobs: build: runs-on: ubuntu-latest - container: "ghcr.io/emqx/emqx-builder/5.0:23.2.7.2-emqx-2-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-3:24.1.5-3-ubuntu20.04" steps: - uses: actions/checkout@v2 From 5263f59f0e7728551ca4887c2056318c16ac5a82 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 13 Jan 2022 20:28:21 +0100 Subject: [PATCH 6/6] ci: rename artifact id in api tests flow --- .github/workflows/run_api_tests.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index e6558b74b..76e5867f4 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -24,11 +24,11 @@ jobs: make emqx-enterprise-tgz - uses: actions/upload-artifact@v2 with: - name: emqx-ce + name: emqx-community path: _packages/emqx/*.tar.gz - uses: actions/upload-artifact@v2 with: - name: emqx-ee + name: emqx-enterprise path: _packages/emqx-enterprise/*.tar.gz api-test: needs: build @@ -55,8 +55,8 @@ jobs: - api_event_message - api_stats edition: - - emqx-ce - #TODO: add emqx-ee + - emqx-community + #TODO: add emqx-enterprise steps: - uses: actions/checkout@v2 with: @@ -117,7 +117,7 @@ jobs: steps: - uses: geekyeggo/delete-artifact@v1 with: - name: emqx-ce + name: emqx-community - uses: geekyeggo/delete-artifact@v1 with: - name: emqx-ee + name: emqx-enterprise