From 6ebf548eb2a02d8d2d5e5260efb5ddae7d8ac7b6 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Wed, 25 Jan 2023 17:19:01 +0100 Subject: [PATCH] ci: test packagecloud --- .github/workflows/build_packages.yaml | 232 ++++++-------------------- 1 file changed, 47 insertions(+), 185 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 558c4c666..09d82dd9e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -23,7 +23,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-ubuntu20.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -86,109 +86,6 @@ jobs: name: source path: source.zip - windows: - runs-on: windows-2019 - if: startsWith(github.ref_name, 'v') - needs: prepare - strategy: - fail-fast: false - matrix: - profile: # for now only CE for windows - - emqx - steps: - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: Expand-Archive -Path source.zip -DestinationPath ./ - - uses: ilammy/msvc-dev-cmd@v1.12.0 - - uses: erlef/setup-beam@v1.15.2 - with: - otp-version: 24.3.4.6 - - name: build - env: - PYTHON: python - DIAGNOSTIC: 1 - working-directory: source - run: | - # ensure crypto app (openssl) - erl -eval "erlang:display(crypto:info_lib())" -s init stop - make ${{ matrix.profile }}-tgz - - name: run emqx - timeout-minutes: 5 - working-directory: source - run: | - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start - Start-Sleep -s 5 - echo "EMQX started" - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop - echo "EMQX stopped" - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install - echo "EMQX installed" - ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall - echo "EMQX uninstalled" - - uses: actions/upload-artifact@v3 - if: success() - with: - name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/ - - name: Send notification to Slack - uses: slackapi/slack-github-action@v1.23.0 - if: failure() && github.event_name == 'schedule' - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - with: - payload: | - {"text": "Scheduled run of ${{ github.workflow }}@Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} - - mac: - needs: prepare - strategy: - fail-fast: false - matrix: - profile: - - ${{ needs.prepare.outputs.BUILD_PROFILE }} - otp: - - 24.3.4.2-1 - os: - - macos-11 - - macos-12-arm64 - runs-on: ${{ matrix.os }} - steps: - - uses: emqx/self-hosted-cleanup-action@v1.0.3 - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: | - ln -s . source - unzip -o -q source.zip - rm source source.zip - - uses: ./.github/actions/package-macos - with: - profile: ${{ matrix.profile }} - otp: ${{ matrix.otp }} - os: ${{ matrix.os }} - apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} - apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} - apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} - apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - - uses: actions/upload-artifact@v3 - if: success() - with: - name: ${{ matrix.profile }} - path: _packages/${{ matrix.profile }}/ - - name: Send notification to Slack - uses: slackapi/slack-github-action@v1.23.0 - if: failure() && github.event_name == 'schedule' - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - with: - payload: | - {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} - linux: needs: prepare runs-on: ${{ matrix.build_machine }} @@ -206,18 +103,18 @@ jobs: - amd64 - arm64 os: - - ["ubuntu20.04", "ubuntu/focal", "deb"] - - ["ubuntu18.04", "ubuntu/bionic", "deb"] - - ["debian11", "debian/bullseye", "deb"] - - ["debian10", "debian/buster", "deb"] - - ["el8", "el/8", "rpm"] - - ["el7", "el/7", "rpm"] - - ["amzn2", "el/6", "rpm"] + - ubuntu20.04 + - ubuntu18.04 + - debian11 + - debian10 + - el8 + - el7 + - amzn2 build_machine: - aws-arm64 - ubuntu-20.04 builder: - - 5.0-26 + - 5.0-27 elixir: - 1.13.4 exclude: @@ -229,17 +126,17 @@ jobs: - profile: emqx otp: 25.1.2-2 arch: amd64 - os: ["ubuntu20.04", "ubuntu/focal", "deb"] + os: ubuntu20.04 build_machine: ubuntu-20.04 - builder: 5.0-26 + builder: 5.0-27 elixir: 1.13.4 release_with: elixir - profile: emqx otp: 25.1.2-2 arch: amd64 - os: ["amzn2", "el/6", "rpm"] + os: amzn2 build_machine: ubuntu-20.04 - builder: 5.0-26 + builder: 5.0-27 elixir: 1.13.4 release_with: elixir @@ -259,62 +156,31 @@ jobs: - name: build emqx packages working-directory: source env: - BUILDER: ${{ matrix.builder }} ELIXIR: ${{ matrix.elixir }} - OTP: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} - ARCH: ${{ matrix.arch }} - SYSTEM: ${{ matrix.os[0] }} run: | set -eu git config --global --add safe.directory "/__w/emqx/emqx" + export ACLOCAL_PATH="/usr/share/aclocal:/usr/local/share/aclocal" # Align path for CMake caches if [ ! "$PWD" = "/emqx" ]; then ln -s $PWD /emqx cd /emqx fi echo "pwd is $PWD" - PKGTYPES="tgz pkg" - IS_ELIXIR="no" if [ ${{ matrix.release_with }} == 'elixir' ]; then - PKGTYPES="tgz" - # set Elixir build flag - IS_ELIXIR="yes" + make ${PROFILE}-elixir-tgz + ./scripts/pkg-tests.sh ${PROFILE}-elixir-tgz + else + make ${PROFILE}-pkg + ./scripts/pkg-tests.sh ${PROFILE}-pkg fi - for PKGTYPE in ${PKGTYPES}; - do - ./scripts/buildx.sh \ - --profile "${PROFILE}" \ - --pkgtype "${PKGTYPE}" \ - --arch "${ARCH}" \ - --elixir "${IS_ELIXIR}" \ - --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}" - done - uses: actions/upload-artifact@v3 if: success() with: name: ${{ matrix.profile }} path: source/_packages/${{ matrix.profile }}/ - - name: Push to packagecloud.io - if: success() && ${{ needs.prepare.outputs.IS_EXACT_TAG }} - working-directory: source - env: - PROFILE: ${{ matrix.profile }} - ARCH: ${{ matrix.arch }} - OS: ${{ matrix.os[0] }} - OS_CODENAME: ${{ matrix.os[1] }} - PKG_EXT: ${{ matrix.os[2] }} - VERSION: ${{ needs.prepare.outputs.VERSION }} - PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} - run: | - set -eu - REPO=$PROFILE - if [ $PROFILE = 'emqx-enterprise' ]; then - REPO='emqx-enterprise5' - docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN \ - ghcr.io/emqx/package_cloud push emqx/$REPO/$OS_CODENAME \ - _packages/$PROFILE/$PROFILE-$VERSION-$OS-$ARCH.$PKG_EXT - name: Send notification to Slack uses: slackapi/slack-github-action@v1.23.0 if: failure() && github.event_name == 'schedule' @@ -324,10 +190,9 @@ jobs: payload: | {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} - publish_artifacts: + packagecloud: runs-on: ubuntu-20.04 - needs: [prepare, mac, linux] - if: ${{ needs.prepare.outputs.IS_EXACT_TAG }} + needs: [prepare, linux] strategy: fail-fast: false matrix: @@ -338,35 +203,32 @@ jobs: with: name: ${{ matrix.profile }} path: packages/${{ matrix.profile }} - - name: install dos2unix - run: sudo apt-get update && sudo apt install -y dos2unix - - name: get packages - run: | - set -e -u - cd packages/${{ matrix.profile }} - # fix the .sha256 file format - for var in $(ls | grep emqx | grep -v sha256); do - dos2unix $var.sha256 - echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1 - done - cd - - - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: upload to aws s3 + - name: Push to packagecloud.io env: - PROFILE: ${{ matrix.profile }} + PROFILE: ${{ needs.prepare.outputs.BUILD_PROFILE }} + VERSION: ${{ needs.prepare.outputs.VERSION }} + PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} run: | - set -e -u - if [ $PROFILE = 'emqx' ]; then - s3dir='emqx-ce' - elif [ $PROFILE = 'emqx-enterprise' ]; then - s3dir='emqx-ee' - else - echo "unknown profile $PROFILE" - exit 1 + set -eu + REPO=$PROFILE + if [ $PROFILE = 'emqx-enterprise' ]; then + REPO='emqx-enterprise5' fi - aws s3 cp --recursive packages/$PROFILE s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} - aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*" + function push() { + docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN -v $(pwd)/$2:/w/$2 -w /w ghcr.io/emqx/package_cloud push id/$REPO/$1 $2 + } + push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-amd64.deb" + push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-arm64.deb" + push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-amd64.deb" + push "debian/bullseye" packages/$PROFILE/$PROFILE-$VERSION-debian11-arm64.deb + push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-amd64.deb" + push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-arm64.deb" + push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-amd64.deb" + push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-arm64.deb" + push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-amd64.rpm" + push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-arm64.rpm" + push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-amd64.rpm" + push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-arm64.rpm" + push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-amd64.rpm" + push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-arm64.rpm" +