From 1269a250067dc1ace43abe80f4788262e2fbb6ca Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 6 May 2023 10:07:21 +0200 Subject: [PATCH] ci: port 10619 to release-50 --- .github/workflows/build_packages.yaml | 174 ++++++-------------------- scripts/buildx.sh | 2 +- 2 files changed, 42 insertions(+), 134 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 2e3e135f1..a6bcdfaf6 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -5,8 +5,6 @@ concurrency: cancel-in-progress: true on: - schedule: - - cron: '0 */6 * * *' push: branches: - 'ci/**' @@ -23,25 +21,20 @@ on: jobs: prepare: runs-on: ubuntu-22.04 - if: (github.repository_owner == 'emqx' && github.event_name == 'schedule') || github.event_name != 'schedule' container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} VERSION: ${{ steps.get_profile.outputs.VERSION }} - steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.branch_or_tag }} # when input is not given, the event tag is used - path: source fetch-depth: 0 - name: Get profile to build id: get_profile run: | - cd source - git config --global --add safe.directory "$(pwd)" tag=${{ github.ref }} if git describe --tags --match "[v|e]*" --exact; then echo "WARN: This is an exact git tag, will publish release" @@ -78,31 +71,21 @@ jobs: esac echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT echo "VERSION=$(./pkg-vsn.sh $PROFILE)" >> $GITHUB_OUTPUT - - name: get_all_deps - run: | - make -C source deps-all - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source - path: source.zip windows: runs-on: windows-2019 if: startsWith(github.ref_name, 'v') - needs: prepare strategy: fail-fast: false matrix: profile: # for now only CE for windows - emqx steps: - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v3 with: - name: source - path: . - - name: unzip source code - run: Expand-Archive -Path source.zip -DestinationPath ./ + ref: ${{ github.event.inputs.branch_or_tag }} + fetch-depth: 0 + - uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: erlef/setup-beam@v1.15.2 with: @@ -111,14 +94,12 @@ jobs: env: PYTHON: python DIAGNOSTIC: 1 - working-directory: source run: | # ensure crypto app (openssl) erl -eval "erlang:display(crypto:info_lib())" -s init stop make ${{ matrix.profile }}-tgz - name: run emqx timeout-minutes: 5 - working-directory: source run: | ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start Start-Sleep -s 5 @@ -133,15 +114,7 @@ jobs: if: success() with: name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/ - - name: Send notification to Slack - uses: slackapi/slack-github-action@v1.23.0 - if: failure() && github.event_name == 'schedule' - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - with: - payload: | - {"text": "Scheduled run of ${{ github.workflow }}@Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + path: _packages/${{ matrix.profile }}/ mac: needs: prepare @@ -159,15 +132,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: emqx/self-hosted-cleanup-action@v1.0.3 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v3 with: - name: source - path: . - - name: unzip source code - run: | - ln -s . source - unzip -o -q source.zip - rm source source.zip + ref: ${{ github.event.inputs.branch_or_tag }} + fetch-depth: 0 - uses: ./.github/actions/package-macos with: profile: ${{ matrix.profile }} @@ -182,19 +150,14 @@ jobs: with: name: ${{ matrix.profile }} path: _packages/${{ matrix.profile }}/ - - name: Send notification to Slack - uses: slackapi/slack-github-action@v1.23.0 - if: failure() && github.event_name == 'schedule' - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - with: - payload: | - {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} linux: needs: prepare - if: github.event_name != 'schedule' - runs-on: aws-${{ matrix.arch }} + runs-on: ${{ matrix.build_machine }} + # always run in builder container because the host might have the wrong OTP version etc. + # otherwise buildx.sh does not run docker if arch and os matches the target arch and os. + container: + image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" strategy: fail-fast: false @@ -216,15 +179,24 @@ jobs: - el8 - el7 - amzn2 + build_machine: + - aws-arm64 + - ubuntu-22.04 builder: - 5.0-34 elixir: - 1.13.4 + exclude: + - arch: arm64 + build_machine: ubuntu-22.04 + - arch: amd64 + build_machine: aws-arm64 include: - profile: emqx otp: 25.1.2-3 arch: amd64 os: ubuntu22.04 + build_machine: ubuntu-22.04 builder: 5.0-34 elixir: 1.13.4 release_with: elixir @@ -232,6 +204,7 @@ jobs: otp: 25.1.2-3 arch: amd64 os: amzn2 + build_machine: ubuntu-22.04 builder: 5.0-34 elixir: 1.13.4 release_with: elixir @@ -242,26 +215,31 @@ jobs: steps: - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v3 + if: matrix.build_machine == 'aws-arm64' + + - uses: actions/checkout@v3 with: - name: source - path: . - - name: unzip source code - run: unzip -q source.zip + ref: ${{ github.event.inputs.branch_or_tag }} + fetch-depth: 0 + - name: build emqx packages - working-directory: source env: - BUILDER: ${{ matrix.builder }} ELIXIR: ${{ matrix.elixir }} - OTP: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} - SYSTEM: ${{ matrix.os }} run: | set -eu + git config --global --add safe.directory "/__w/emqx/emqx" + # Align path for CMake caches + if [ ! "$PWD" = "/emqx" ]; then + ln -s $PWD /emqx + cd /emqx + fi + echo "pwd is $PWD" PKGTYPES="tgz pkg" IS_ELIXIR="no" - if [ ${{ matrix.release_with }} == 'elixir' ]; then + WITH_ELIXIR=${{ matrix.release_with }} + if [ "${WITH_ELIXIR:-}" == 'elixir' ]; then PKGTYPES="tgz" # set Elixir build flag IS_ELIXIR="yes" @@ -273,88 +251,18 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --elixir "${IS_ELIXIR}" \ - --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}" + --builder "force_host" done - uses: actions/upload-artifact@v3 if: success() with: name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/ - - linux-schedule: - needs: prepare - if: github.event_name == 'schedule' - runs-on: aws-${{ matrix.arch }} - - strategy: - fail-fast: false - matrix: - profile: - - emqx - - emqx-enterprise - otp: - - 24.3.4.2-3 - arch: - - amd64 - os: - - debian10 - builder: - - 5.0-34 - elixir: - - 1.13.4 - - defaults: - run: - shell: bash - - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v3 - with: - name: source - path: . - - name: unzip source code - run: unzip -q source.zip - - name: build emqx packages - working-directory: source - env: - BUILDER: ${{ matrix.builder }} - ELIXIR: ${{ matrix.elixir }} - OTP: ${{ matrix.otp }} - PROFILE: ${{ matrix.profile }} - ARCH: ${{ matrix.arch }} - SYSTEM: ${{ matrix.os }} - run: | - set -eu - PKGTYPES="tgz pkg" - IS_ELIXIR="no" - for PKGTYPE in ${PKGTYPES}; - do - ./scripts/buildx.sh \ - --profile "${PROFILE}" \ - --pkgtype "${PKGTYPE}" \ - --arch "${ARCH}" \ - --elixir "${IS_ELIXIR}" \ - --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}" - done - - uses: actions/upload-artifact@v3 - if: success() - with: - name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/ - - name: Send notification to Slack - uses: slackapi/slack-github-action@v1.23.0 - if: failure() - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - with: - payload: | - {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"} + path: _packages/${{ matrix.profile }}/ publish_artifacts: runs-on: ubuntu-22.04 needs: [prepare, mac, linux] - if: needs.prepare.outputs.IS_EXACT_TAG && github.event_name != 'schedule' + if: needs.prepare.outputs.IS_EXACT_TAG == 'true' strategy: fail-fast: false matrix: diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 5c3a65369..9376647c0 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -140,7 +140,7 @@ BUILDER_SYSTEM="$(echo "$BUILDER" | awk -F'-' '{print $NF}')" CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}" IS_NATIVE_SYSTEM='no' -if [[ "$HOST_SYSTEM" = "$BUILDER_SYSTEM" ]]; then +if [[ "$BUILDER_SYSTEM" == "force_host" ]] || [[ "$BUILDER_SYSTEM" == "$HOST_SYSTEM" ]]; then IS_NATIVE_SYSTEM='yes' fi