diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 5fead0139..73622495d 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -4,12 +4,12 @@ set -euo pipefail set -x if [ -z "${1:-}" ]; then - echo "Usage $0 zip|pkg" + echo "Usage $0 tgz|pkg" exit 1 fi -if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then - echo "Usage $0 zip|pkg" +if [ "${2:-}" != 'tgz' ] && [ "${2:-}" != 'pkg' ]; then + echo "Usage $0 tgz|pkg" exit 1 fi @@ -22,8 +22,8 @@ export EMQX_NAME=${EMQX_NAME:-"emqx"} export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" -if [ "$PACKAGE_TYPE" = 'zip' ]; then - PKG_SUFFIX="zip" +if [ "$PACKAGE_TYPE" = 'tgz' ]; then + PKG_SUFFIX="tar.gz" else SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)" case "${SYSTEM:-}" in @@ -37,9 +37,10 @@ else fi PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" -PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}.${PKG_SUFFIX}" +PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}" if ! [ -f "$PACKAGE_FILE" ]; then echo "$PACKAGE_FILE is not a file" + exit 1 fi case "$(uname -m)" in @@ -68,8 +69,8 @@ emqx_test(){ cd "${PACKAGE_PATH}" local packagename="${PACKAGE_FILE_NAME}" case "$PKG_SUFFIX" in - "zip") - unzip -q "${PACKAGE_PATH}/${packagename}" + "tar.gz") + tar -zxf "${PACKAGE_PATH}/${packagename}" export EMQX_ZONES__DEFAULT__MQTT__SERVER_KEEPALIVE=60 export EMQX_MQTT__MAX_TOPIC_ALIAS=10 export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug @@ -205,10 +206,10 @@ relup_test(){ if [ -d "${RELUP_PACKAGE_PATH}" ];then cd "${RELUP_PACKAGE_PATH}" - find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" | + find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.tar.gz" | while read -r pkg; do packagename=$(basename "${pkg}") - unzip "$packagename" + tar -zxf "$packagename" if ! ./emqx/bin/emqx start; then cat emqx/log/erlang.log.1 || true cat emqx/log/emqx.log.1 || true @@ -216,7 +217,7 @@ relup_test(){ fi ./emqx/bin/emqx_ctl status ./emqx/bin/emqx versions - cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases + cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".tar.gz ./emqx/releases ./emqx/bin/emqx install "${TARGET_VERSION}" [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1 ./emqx/bin/emqx_ctl status diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index f5b36c648..4a8894476 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -22,7 +22,7 @@ [shell emqx] !cd $PACKAGE_PATH - !unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed -r 's/[v|e]//g')-amd64.zip + !tar -zxf ${PROFILE}-$(echo $old_vsn | sed -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz ?SH-PROMPT !cd emqx @@ -82,7 +82,7 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/ !./bin/emqx install $VSN ?Made release permanent: "$VSN" @@ -107,7 +107,7 @@ !echo "" > log/emqx.log.1 ?SH-PROMPT - !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/ !./bin/emqx install $VSN ?Made release permanent: "$VSN" diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 6076b49a5..956cfbc0e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -85,11 +85,11 @@ jobs: $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" - $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip" - } + $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.tar.gz" + } else { - $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" - } + $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.tar.gz" + } ## We do not build/release bcrypt and quic for windows package Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ Remove-Item -Recurse -Force -Path _build/default/lib/quicer/ @@ -192,12 +192,12 @@ jobs: make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 rm -rf _build/${{ matrix.profile }}/lib - make ${{ matrix.profile }}-zip + make ${{ matrix.profile }}-tgz - name: test working-directory: source run: | - pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) - unzip -q $pkg_name + pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz) + tar -zxf $pkg_name # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' @@ -297,7 +297,7 @@ jobs: with: path: source/_build/default/lib/quicer/ key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} - - name: downloads old emqx zip packages + - name: download old emqx tgz packages env: OTP_VSN: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} @@ -320,10 +320,10 @@ jobs: old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" - if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256 - echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1 + if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz) | grep -oE "^[23]+")" ]; then + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz.sha256 + echo "$(cat $package_name.tar.gz.sha256) $package_name.tar.gz" | sha256sum -c || exit 1 fi done - name: build emqx packages @@ -336,10 +336,10 @@ jobs: run: | ./scripts/buildx.sh \ --profile "${PROFILE}" \ - --pkgtype "zip" \ + --pkgtype "tgz" \ --arch "${ARCH}" \ --builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}" - ## the pkg build is incremental on the zip build + ## the pkg build is incremental on the tgz build ./scripts/buildx.sh \ --profile "${PROFILE}" \ --pkgtype "pkg" \ diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index dd8d2568d..1694c177a 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -63,10 +63,10 @@ jobs: with: path: _build/default/lib/quicer/ key: ${{ matrix.os }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }} - - name: build and test zip package + - name: build and test tgz package run: | - make ${EMQX_NAME}-zip - .ci/build_packages/tests.sh "$EMQX_PKG_NAME" zip + make ${EMQX_NAME}-tgz + .ci/build_packages/tests.sh "$EMQX_PKG_NAME" tgz - name: run static checks if: contains(matrix.os, 'ubuntu') run: | @@ -78,7 +78,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} - path: _packages/**/*.zip + path: _packages/${{ matrix.profile}}/*.tar.gz mac: strategy: fail-fast: false @@ -140,11 +140,11 @@ jobs: . $HOME/.kerl/${{ matrix.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 - make ${{ matrix.profile }}-zip + make ${{ matrix.profile }}-tgz - name: test run: | - pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) - unzip -q $pkg_name + pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz) + tar -zxf $pkg_name # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' @@ -166,4 +166,4 @@ jobs: - uses: actions/upload-artifact@v2 with: name: macos - path: _packages/**/*.zip + path: _packages/**/*.tar.gz diff --git a/.github/workflows/run_api_tests.yaml b/.github/workflows/run_api_tests.yaml index 084704537..d3dd485b1 100644 --- a/.github/workflows/run_api_tests.yaml +++ b/.github/workflows/run_api_tests.yaml @@ -18,21 +18,18 @@ jobs: steps: - uses: actions/checkout@v2 - - name: zip emqx-broker - if: endsWith(github.repository, 'emqx') + - name: build EMQ X CE and EE tgz package run: | - make emqx-zip - - name: zip emqx-broker - if: endsWith(github.repository, 'enterprise') - run: | - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - make emqx-enterprise-zip + make emqx-tgz + make emqx-enterprise-tgz - uses: actions/upload-artifact@v2 with: - name: emqx-broker - path: _packages/**/*.zip - + name: emqx-ce + path: _packages/emqx/*.tar.gz + - uses: actions/upload-artifact@v2 + with: + name: emqx-ee + path: _packages/emqx-enterprise/*.tar.gz api-test: needs: build runs-on: ubuntu-latest @@ -57,6 +54,9 @@ jobs: - api_topic_rewrite - api_event_message - api_stats + edition: + - emqx-ce + #TODO: add emqx-ee steps: - uses: actions/checkout@v2 with: @@ -70,13 +70,13 @@ jobs: architecture: x64 # (x64 or x86) - defaults to x64 - uses: actions/download-artifact@v2 with: - name: emqx-broker + name: ${{ matrix.edition }} path: . - - name: start emqx-broker + - name: start EMQ X service env: EMQX_LISTENERS__WSS__DEFAULT__BIND: "0.0.0.0:8085" run: | - unzip ./emqx/*.zip + tar -zxf ./*.tar.gz ./emqx/bin/emqx start - name: install jmeter timeout-minutes: 10 @@ -117,4 +117,7 @@ jobs: steps: - uses: geekyeggo/delete-artifact@v1 with: - name: emqx-broker + name: emqx-ce + - uses: geekyeggo/delete-artifact@v1 + with: + name: emqx-ee diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index 3b796d476..766484e36 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -85,10 +85,10 @@ jobs: cd emqx/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for old_vsn in ${old_vsns[@]}; do - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.tar.gz done - name: build emqx - run: make -C emqx ${PROFILE}-zip + run: make -C emqx ${PROFILE}-tgz - name: build emqtt-bench run: make -C emqtt-bench - name: build lux @@ -105,8 +105,8 @@ jobs: set -e -x -u if [ -n "$OLD_VSNS" ]; then mkdir -p packages - cp emqx/_packages/${PROFILE}/*.zip packages - cp emqx/_upgrade_base/*.zip packages + cp emqx/_packages/${PROFILE}/*.tar.gz packages + cp emqx/_upgrade_base/*.tar.gz packages lux \ --case_timeout infinity \ --var PROFILE=$PROFILE \