yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ci: change to build and test with .tar.gz instead of .zip packages

This commit is contained in:
Zaiming (Stone) Shi 2022-01-12 23:18:30 +01:00
parent 1abac892c9
commit ecd7ea0a2f
6 changed files with 60 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
.ci/build_packages/tests.sh
View File

@ -4,12 +4,12 @@ set -euo pipefail
set -x set -x
if [ -z "${1:-}" ]; then if [ -z "${1:-}" ]; then
echo "Usage $0 <PACKAGE_NAME> zip|pkg" echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
exit 1 exit 1
fi fi
if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then if [ "${2:-}" != 'tgz' ] && [ "${2:-}" != 'pkg' ]; then
echo "Usage $0 <PACKAGE_NAME> zip|pkg" echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
exit 1 exit 1
fi fi
@ -22,8 +22,8 @@ export EMQX_NAME=${EMQX_NAME:-"emqx"}
export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
if [ "$PACKAGE_TYPE" = 'zip' ]; then if [ "$PACKAGE_TYPE" = 'tgz' ]; then
PKG_SUFFIX="zip" PKG_SUFFIX="tar.gz"
else else
SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)" SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)"
case "${SYSTEM:-}" in case "${SYSTEM:-}" in
@ -37,9 +37,10 @@ else
fi fi
PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}"
PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}.${PKG_SUFFIX}" PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}"
if ! [ -f "$PACKAGE_FILE" ]; then if ! [ -f "$PACKAGE_FILE" ]; then
echo "$PACKAGE_FILE is not a file" echo "$PACKAGE_FILE is not a file"
exit 1
fi fi
case "$(uname -m)" in case "$(uname -m)" in
@ -68,8 +69,8 @@ emqx_test(){
cd "${PACKAGE_PATH}" cd "${PACKAGE_PATH}"
local packagename="${PACKAGE_FILE_NAME}" local packagename="${PACKAGE_FILE_NAME}"
case "$PKG_SUFFIX" in case "$PKG_SUFFIX" in
"zip") "tar.gz")
unzip -q "${PACKAGE_PATH}/${packagename}" tar -zxf "${PACKAGE_PATH}/${packagename}"
export EMQX_ZONES__DEFAULT__MQTT__SERVER_KEEPALIVE=60 export EMQX_ZONES__DEFAULT__MQTT__SERVER_KEEPALIVE=60
export EMQX_MQTT__MAX_TOPIC_ALIAS=10 export EMQX_MQTT__MAX_TOPIC_ALIAS=10
export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug
@ -205,10 +206,10 @@ relup_test(){
if [ -d "${RELUP_PACKAGE_PATH}" ];then if [ -d "${RELUP_PACKAGE_PATH}" ];then
cd "${RELUP_PACKAGE_PATH}" cd "${RELUP_PACKAGE_PATH}"
find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" | find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.tar.gz" |
while read -r pkg; do while read -r pkg; do
packagename=$(basename "${pkg}") packagename=$(basename "${pkg}")
unzip "$packagename" tar -zxf "$packagename"
if ! ./emqx/bin/emqx start; then if ! ./emqx/bin/emqx start; then
cat emqx/log/erlang.log.1 || true cat emqx/log/erlang.log.1 || true
cat emqx/log/emqx.log.1 || true cat emqx/log/emqx.log.1 || true
@ -216,7 +217,7 @@ relup_test(){
fi fi
./emqx/bin/emqx_ctl status ./emqx/bin/emqx_ctl status
./emqx/bin/emqx versions ./emqx/bin/emqx versions
cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".tar.gz ./emqx/releases
./emqx/bin/emqx install "${TARGET_VERSION}" ./emqx/bin/emqx install "${TARGET_VERSION}"
[ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
./emqx/bin/emqx_ctl status ./emqx/bin/emqx_ctl status

6
.ci/fvt_tests/relup.lux
View File

@ -22,7 +22,7 @@
[shell emqx] [shell emqx]
!cd $PACKAGE_PATH !cd $PACKAGE_PATH
!unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed -r 's/[v|e]//g')-amd64.zip !tar -zxf ${PROFILE}-$(echo $old_vsn | sed -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
?SH-PROMPT ?SH-PROMPT
!cd emqx !cd emqx
@ -82,7 +82,7 @@
!echo "" > log/emqx.log.1 !echo "" > log/emqx.log.1
?SH-PROMPT ?SH-PROMPT
!cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/
!./bin/emqx install $VSN !./bin/emqx install $VSN
?Made release permanent: "$VSN" ?Made release permanent: "$VSN"
@ -107,7 +107,7 @@
!echo "" > log/emqx.log.1 !echo "" > log/emqx.log.1
?SH-PROMPT ?SH-PROMPT
!cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/
!./bin/emqx install $VSN !./bin/emqx install $VSN
?Made release permanent: "$VSN" ?Made release permanent: "$VSN"

28
.github/workflows/build_packages.yaml vendored
View File

@ -85,11 +85,11 @@ jobs:
$version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
$regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
$pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip" $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.tar.gz"
} }
else { else {
$pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.tar.gz"
} }
## We do not build/release bcrypt and quic for windows package ## We do not build/release bcrypt and quic for windows package
Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
Remove-Item -Recurse -Force -Path _build/default/lib/quicer/ Remove-Item -Recurse -Force -Path _build/default/lib/quicer/
@ -192,12 +192,12 @@ jobs:
make ensure-rebar3 make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3 sudo cp rebar3 /usr/local/bin/rebar3
rm -rf _build/${{ matrix.profile }}/lib rm -rf _build/${{ matrix.profile }}/lib
make ${{ matrix.profile }}-zip make ${{ matrix.profile }}-tgz
- name: test - name: test
working-directory: source working-directory: source
run: | run: |
pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz)
unzip -q $pkg_name tar -zxf $pkg_name
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no' ready='no'
@ -297,7 +297,7 @@ jobs:
with: with:
path: source/_build/default/lib/quicer/ path: source/_build/default/lib/quicer/
key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
- name: downloads old emqx zip packages - name: download old emqx tgz packages
env: env:
OTP_VSN: ${{ matrix.otp }} OTP_VSN: ${{ matrix.otp }}
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
@ -320,10 +320,10 @@ jobs:
old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for tag in ${old_vsns[@]}; do for tag in ${old_vsns[@]}; do
package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz) | grep -oE "^[23]+")" ]; then
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256 wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz.sha256
echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1 echo "$(cat $package_name.tar.gz.sha256) $package_name.tar.gz" | sha256sum -c || exit 1
fi fi
done done
- name: build emqx packages - name: build emqx packages
@ -336,10 +336,10 @@ jobs:
run: | run: |
./scripts/buildx.sh \ ./scripts/buildx.sh \
--profile "${PROFILE}" \ --profile "${PROFILE}" \
--pkgtype "zip" \ --pkgtype "tgz" \
--arch "${ARCH}" \ --arch "${ARCH}" \
--builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}" --builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}"
## the pkg build is incremental on the zip build ## the pkg build is incremental on the tgz build
./scripts/buildx.sh \ ./scripts/buildx.sh \
--profile "${PROFILE}" \ --profile "${PROFILE}" \
--pkgtype "pkg" \ --pkgtype "pkg" \

16
.github/workflows/build_slim_packages.yaml vendored
View File

@ -63,10 +63,10 @@ jobs:
with: with:
path: _build/default/lib/quicer/ path: _build/default/lib/quicer/
key: ${{ matrix.os }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} key: ${{ matrix.os }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
- name: build and test zip package - name: build and test tgz package
run: | run: |
make ${EMQX_NAME}-zip make ${EMQX_NAME}-tgz
.ci/build_packages/tests.sh "$EMQX_PKG_NAME" zip .ci/build_packages/tests.sh "$EMQX_PKG_NAME" tgz
- name: run static checks - name: run static checks
if: contains(matrix.os, 'ubuntu') if: contains(matrix.os, 'ubuntu')
run: | run: |
@ -78,7 +78,7 @@ jobs:
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }}
path: _packages/**/*.zip path: _packages/${{ matrix.profile}}/*.tar.gz
mac: mac:
strategy: strategy:
fail-fast: false fail-fast: false
@ -140,11 +140,11 @@ jobs:
. $HOME/.kerl/${{ matrix.otp }}/activate . $HOME/.kerl/${{ matrix.otp }}/activate
make ensure-rebar3 make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3 sudo cp rebar3 /usr/local/bin/rebar3
make ${{ matrix.profile }}-zip make ${{ matrix.profile }}-tgz
- name: test - name: test
run: | run: |
pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz)
unzip -q $pkg_name tar -zxf $pkg_name
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no' ready='no'
@ -166,4 +166,4 @@ jobs:
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: macos name: macos
path: _packages/**/*.zip path: _packages/**/*.tar.gz

35
.github/workflows/run_api_tests.yaml vendored
View File

@ -18,21 +18,18 @@ jobs:
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- name: zip emqx-broker - name: build EMQ X CE and EE tgz package
if: endsWith(github.repository, 'emqx')
run: | run: |
make emqx-zip make emqx-tgz
- name: zip emqx-broker make emqx-enterprise-tgz
if: endsWith(github.repository, 'enterprise')
run: |
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
make emqx-enterprise-zip
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: emqx-broker name: emqx-ce
path: _packages/**/*.zip path: _packages/emqx/*.tar.gz
- uses: actions/upload-artifact@v2
with:
name: emqx-ee
path: _packages/emqx-enterprise/*.tar.gz
api-test: api-test:
needs: build needs: build
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -57,6 +54,9 @@ jobs:
- api_topic_rewrite - api_topic_rewrite
- api_event_message - api_event_message
- api_stats - api_stats
edition:
- emqx-ce
#TODO: add emqx-ee
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
with: with:
@ -70,13 +70,13 @@ jobs:
architecture: x64 # (x64 or x86) - defaults to x64 architecture: x64 # (x64 or x86) - defaults to x64
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: emqx-broker name: ${{ matrix.edition }}
path: . path: .
- name: start emqx-broker - name: start EMQ X service
env: env:
EMQX_LISTENERS__WSS__DEFAULT__BIND: "0.0.0.0:8085" EMQX_LISTENERS__WSS__DEFAULT__BIND: "0.0.0.0:8085"
run: | run: |
unzip ./emqx/*.zip tar -zxf ./*.tar.gz
./emqx/bin/emqx start ./emqx/bin/emqx start
- name: install jmeter - name: install jmeter
timeout-minutes: 10 timeout-minutes: 10
@ -117,4 +117,7 @@ jobs:
steps: steps:
- uses: geekyeggo/delete-artifact@v1 - uses: geekyeggo/delete-artifact@v1
with: with:
name: emqx-broker name: emqx-ce
- uses: geekyeggo/delete-artifact@v1
with:
name: emqx-ee

8
.github/workflows/run_relup_tests.yaml vendored
View File

@ -85,10 +85,10 @@ jobs:
cd emqx/_upgrade_base cd emqx/_upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for old_vsn in ${old_vsns[@]}; do for old_vsn in ${old_vsns[@]}; do
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.zip wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.tar.gz
done done
- name: build emqx - name: build emqx
run: make -C emqx ${PROFILE}-zip run: make -C emqx ${PROFILE}-tgz
- name: build emqtt-bench - name: build emqtt-bench
run: make -C emqtt-bench run: make -C emqtt-bench
- name: build lux - name: build lux
@ -105,8 +105,8 @@ jobs:
set -e -x -u set -e -x -u
if [ -n "$OLD_VSNS" ]; then if [ -n "$OLD_VSNS" ]; then
mkdir -p packages mkdir -p packages
cp emqx/_packages/${PROFILE}/*.zip packages cp emqx/_packages/${PROFILE}/*.tar.gz packages
cp emqx/_upgrade_base/*.zip packages cp emqx/_upgrade_base/*.tar.gz packages
lux \ lux \
--case_timeout infinity \ --case_timeout infinity \
--var PROFILE=$PROFILE \ --var PROFILE=$PROFILE \