From 1765c429c07b6cd968deb6837d1d300aa8d9c3b4 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 5 May 2023 21:38:47 +0200 Subject: [PATCH 1/5] ci: use cd instead of pushd --- .github/workflows/build_packages.yaml | 4 ++-- .github/workflows/build_packages_cron.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index a3a5f3855..64c48f1cb 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -77,12 +77,12 @@ jobs: echo "VERSION=$(./pkg-vsn.sh $PROFILE)" >> $GITHUB_OUTPUT - name: get_all_deps run: | - pushd source + cd source # make sure all previous left-overs are cleaned make clean-all # enterprise is a super-set, should be enough for all profiles make deps-emqx-enterprise - popd + cd .. zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 5d8943c6a..2f8316825 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -28,12 +28,12 @@ jobs: fetch-depth: 0 - name: get_all_deps run: | - pushd source + cd source # make sure all previous left-overs are cleaned make clean-all # enterprise is a super-set, should be enough for all profiles make deps-emqx-enterprise - popd + cd .. zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v3 with: From e739e3cadb78386115c46322de60a4f81c740db2 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 5 May 2023 21:49:15 +0200 Subject: [PATCH 2/5] ci: ensure == compare is not empty on the left --- .github/workflows/build_packages.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 64c48f1cb..99c930430 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -270,7 +270,8 @@ jobs: echo "pwd is $PWD" PKGTYPES="tgz pkg" IS_ELIXIR="no" - if [ ${{ matrix.release_with }} == 'elixir' ]; then + WITH_ELIXIR=${{ matrix.release_with }} + if [ "${WITH_ELIXIR:-}" == 'elixir' ]; then PKGTYPES="tgz" # set Elixir build flag IS_ELIXIR="yes" From 22815d4cec150cb2741faa3425a24a6fcd818a8a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 5 May 2023 22:15:27 +0200 Subject: [PATCH 3/5] ci(buildx): use a special builder value to indicate host build --- .github/workflows/build_packages.yaml | 2 +- .github/workflows/build_packages_cron.yaml | 2 +- scripts/buildx.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 99c930430..03294dce6 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -283,7 +283,7 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --elixir "${IS_ELIXIR}" \ - --builder "builder_to_be_ignored" + --builder "force_host" done - uses: actions/upload-artifact@v3 if: success() diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 2f8316825..86d44d512 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -95,7 +95,7 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --elixir "${IS_ELIXIR}" \ - --builder "builder_to_be_ignored" + --builder "force_host" done - uses: actions/upload-artifact@v3 if: success() diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 5c3a65369..9376647c0 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -140,7 +140,7 @@ BUILDER_SYSTEM="$(echo "$BUILDER" | awk -F'-' '{print $NF}')" CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}" IS_NATIVE_SYSTEM='no' -if [[ "$HOST_SYSTEM" = "$BUILDER_SYSTEM" ]]; then +if [[ "$BUILDER_SYSTEM" == "force_host" ]] || [[ "$BUILDER_SYSTEM" == "$HOST_SYSTEM" ]]; then IS_NATIVE_SYSTEM='yes' fi From a7f8e333219d8a69c2624d3ae9265205f683e94d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 5 May 2023 22:43:26 +0200 Subject: [PATCH 4/5] ci: no need to prepare source zip source.zip was need in v4 enterprise because the rebar3 deps fetch had to be done in the github runner host, but not inside the runner docker containers. now that all dependencies are opensource, there is no need for this step any more. --- .github/workflows/build_packages.yaml | 60 +++++---------------- .github/workflows/build_packages_cron.yaml | 62 +++++----------------- 2 files changed, 28 insertions(+), 94 deletions(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 03294dce6..f03af9672 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -26,19 +26,15 @@ jobs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} VERSION: ${{ steps.get_profile.outputs.VERSION }} - steps: - uses: actions/checkout@v3 with: ref: ${{ github.event.inputs.branch_or_tag }} # when input is not given, the event tag is used - path: source fetch-depth: 0 - name: Get profile to build id: get_profile run: | - cd source - git config --global --add safe.directory "$(pwd)" tag=${{ github.ref }} if git describe --tags --match "[v|e]*" --exact; then echo "WARN: This is an exact git tag, will publish release" @@ -75,36 +71,21 @@ jobs: esac echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT echo "VERSION=$(./pkg-vsn.sh $PROFILE)" >> $GITHUB_OUTPUT - - name: get_all_deps - run: | - cd source - # make sure all previous left-overs are cleaned - make clean-all - # enterprise is a super-set, should be enough for all profiles - make deps-emqx-enterprise - cd .. - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source - path: source.zip windows: runs-on: windows-2019 if: startsWith(github.ref_name, 'v') - needs: prepare strategy: fail-fast: false matrix: profile: # for now only CE for windows - emqx steps: - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v3 with: - name: source - path: . - - name: unzip source code - run: Expand-Archive -Path source.zip -DestinationPath ./ + ref: ${{ github.event.inputs.branch_or_tag }} + fetch-depth: 0 + - uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: erlef/setup-beam@v1.15.2 with: @@ -113,14 +94,12 @@ jobs: env: PYTHON: python DIAGNOSTIC: 1 - working-directory: source run: | # ensure crypto app (openssl) erl -eval "erlang:display(crypto:info_lib())" -s init stop make ${{ matrix.profile }}-tgz - name: run emqx timeout-minutes: 5 - working-directory: source run: | ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start Start-Sleep -s 5 @@ -135,7 +114,7 @@ jobs: if: success() with: name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/ + path: _packages/${{ matrix.profile }}/ mac: needs: prepare @@ -153,15 +132,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: emqx/self-hosted-cleanup-action@v1.0.3 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v3 with: - name: source - path: . - - name: unzip source code - run: | - ln -s . source - unzip -o -q source.zip - rm source source.zip + ref: ${{ github.event.inputs.branch_or_tag }} + fetch-depth: 0 - uses: ./.github/actions/package-macos with: profile: ${{ matrix.profile }} @@ -242,19 +216,13 @@ jobs: steps: - uses: AutoModality/action-clean@v1 if: matrix.build_machine == 'aws-arm64' - - uses: actions/download-artifact@v3 + + - uses: actions/checkout@v3 with: - name: source - path: . - - name: unzip source code - run: unzip -q source.zip - - name: tmp fix for el9 - if: matrix.os == 'el9' - run: | - set -eu - dnf install -y krb5-devel + ref: ${{ github.event.inputs.branch_or_tag }} + fetch-depth: 0 + - name: build emqx packages - working-directory: source env: ELIXIR: ${{ matrix.elixir }} PROFILE: ${{ matrix.profile }} @@ -289,7 +257,7 @@ jobs: if: success() with: name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/ + path: _packages/${{ matrix.profile }}/ publish_artifacts: runs-on: ubuntu-22.04 diff --git a/.github/workflows/build_packages_cron.yaml b/.github/workflows/build_packages_cron.yaml index 86d44d512..aace0d241 100644 --- a/.github/workflows/build_packages_cron.yaml +++ b/.github/workflows/build_packages_cron.yaml @@ -7,39 +7,9 @@ concurrency: on: schedule: - cron: '0 */6 * * *' + workflow_dispatch: jobs: - prepare: - runs-on: aws-amd64 - if: github.repository_owner == 'emqx' - container: ghcr.io/emqx/emqx-builder/5.0-34:1.13.4-24.3.4.2-3-ubuntu22.04 - strategy: - fail-fast: false - matrix: - profile: - - ['emqx', 'master'] - - ['emqx-enterprise', 'release-50'] - - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ matrix.profile[1] }} - path: source - fetch-depth: 0 - - name: get_all_deps - run: | - cd source - # make sure all previous left-overs are cleaned - make clean-all - # enterprise is a super-set, should be enough for all profiles - make deps-emqx-enterprise - cd .. - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source-${{ matrix.profile[0] }} - path: source.zip - linux: needs: prepare runs-on: aws-${{ matrix.arch }} @@ -54,6 +24,9 @@ jobs: profile: - emqx - emqx-enterprise + branch: + - master + - release-50 otp: - 24.3.4.2-3 arch: @@ -71,18 +44,16 @@ jobs: shell: bash steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/download-artifact@v3 + - uses: emqx/self-hosted-cleanup-action@v1.0.3 + - uses: actions/checkout@v3 with: - name: source-${{ matrix.profile }} - path: . - - name: unzip source code - run: unzip -q source.zip + ref: ${{ matrix.branch }} + fetch-depth: 0 + - name: build emqx packages - working-directory: source env: ELIXIR: ${{ matrix.elixir }} - PROFILE: ${{ matrix.profile[0] }} + PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} run: | set -eu @@ -101,7 +72,7 @@ jobs: if: success() with: name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/ + path: _packages/${{ matrix.profile }}/ - name: Send notification to Slack uses: slackapi/slack-github-action@v1.23.0 if: failure() @@ -126,15 +97,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: emqx/self-hosted-cleanup-action@v1.0.3 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v3 with: - name: source-${{ matrix.profile }} - path: . - - name: unzip source code - run: | - ln -s . source - unzip -o -q source.zip - rm source source.zip + ref: ${{ matrix.branch }} + fetch-depth: 0 - uses: ./.github/actions/package-macos with: profile: ${{ matrix.profile }} From 429d193af747457fba5ba7824b92719c224e672f Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Fri, 5 May 2023 23:58:24 +0200 Subject: [PATCH 5/5] ci: github action boolean in env check 'true' string --- .github/workflows/build_packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f03af9672..a6bcdfaf6 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -262,7 +262,7 @@ jobs: publish_artifacts: runs-on: ubuntu-22.04 needs: [prepare, mac, linux] - if: needs.prepare.outputs.IS_EXACT_TAG + if: needs.prepare.outputs.IS_EXACT_TAG == 'true' strategy: fail-fast: false matrix: