Merge pull request #7115 from Rory-Z/ci/update-build

ci(cross build): splitting cross builds and upload assets

.github/workflows/build_packages.yaml

@@ -8,11 +8,11 @@ on:
   push:
     branches:
       - 'main-v4.**'
+    tags:
+      - v*
+      - e*
   schedule:
     - cron:  '0 */6 * * *'
-  release:
-    types:
-      - published
   workflow_dispatch:
 
 jobs:
@@ -401,14 +401,6 @@ jobs:
         file: source/deploy/docker/Dockerfile.enterprise
         context: source
 
-  delete-artifact:
-    runs-on: ubuntu-20.04
-    needs: [prepare, mac, linux, docker]
-    steps:
-    - uses: geekyeggo/delete-artifact@v1
-      with:
-        name: source
-
   upload:
     runs-on: ubuntu-20.04
 
@@ -420,95 +412,34 @@ jobs:
       fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-
     steps:
-    - uses: actions/checkout@v2
-    - name: get_version
-      run: |
-        echo 'version<<EOF' >> $GITHUB_ENV
-        echo ${{ github.ref }} | sed -r  "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV
-        echo 'EOF' >> $GITHUB_ENV
     - uses: actions/download-artifact@v2
       with:
         name: ${{ matrix.profile }}
-        path: ./_packages/${{ matrix.profile }}
+        path: packages/${{ matrix.profile }}
     - name: install dos2unix
       run: sudo apt-get update && sudo apt install -y dos2unix
     - name: get packages
       run: |
         set -e -u
-        cd _packages/${{ matrix.profile }}
+        cd packages/${{ matrix.profile }}
         for var in $( ls |grep emqx |grep -v sha256); do
           dos2unix $var.sha256
           echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
         done
         cd -
+    - uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
     - name: upload aws s3
       run: |
         set -e -u
         if [ "${{ matrix.profile }}"  == "emqx" ];then
-            broker="emqx-ce"
+            s3dir="emqx-ce"
         else
-            broker=${{ matrix.profile }}
+            s3dir=${{ matrix.profile }}
         fi
-        aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws s3 cp --recursive packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/${s3dir}/${{ github.ref_name }}
-        aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/${s3dir}/${{ github.ref_name }}/*"
-        aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }}
-
-        aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }}
-        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*"
-    - uses: Rory-Z/upload-release-asset@v1
-      if: github.event_name == 'release' && matrix.profile != 'emqx-ee'
-      with:
-        repo: emqx
-        path: "_packages/${{ matrix.profile }}/emqx-*"
-        token: ${{ github.token }}
-    - uses: Rory-Z/upload-release-asset@v1
-      if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
-      with:
-        repo: emqx-enterprise
-        path: "_packages/${{ matrix.profile }}/emqx-*"
-        token: ${{ github.token }}
-    - name: update to emqx.io
-      if: github.event_name == 'release'
-      run: |
-        set -e -x -u
-        curl -w %{http_code} \
-             --insecure \
-             -H "Content-Type: application/json" \
-             -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
-             -X POST \
-             -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
-             ${{ secrets.EMQX_IO_RELEASE_API }}
-    - name: update repo.emqx.io
-      if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
-      run: |
-        curl --silent --show-error \
-          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-          -H "Accept: application/vnd.github.v3+json" \
-          -X POST \
-          -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \
-          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
-    - name: update repo.emqx.io
-      if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
-      run: |
-        curl --silent --show-error \
-          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-          -H "Accept: application/vnd.github.v3+json" \
-          -X POST \
-          -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \
-          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
-    - name: update homebrew packages
-      if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
-      run: |
-        if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
-            curl --silent --show-error \
-              -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-              -H "Accept: application/vnd.github.v3+json" \
-              -X POST \
-              -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
-              "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
-        fi
-    - uses: geekyeggo/delete-artifact@v1
-      with:
-        name: ${{ matrix.profile }}

.github/workflows/build_slim_packages.yaml

@@ -1,10 +1,6 @@
 name: Build slim packages
 
 on:
-  push:
-    tags:
-      - v*
-      - e*
   pull_request:
   workflow_dispatch:
 

.github/workflows/release.yaml

@@ -0,0 +1,97 @@
+name: Upload release assets
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  prepare:
+    runs-on: ubuntu-20.04
+    container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+
+    outputs:
+      profiles: ${{ steps.set_profile.outputs.profiles}}
+      s3dir: ${{ steps.set_profile.outputs.s3dir}}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: source
+          fetch-depth: 0
+      - name: set profile
+        id: set_profile
+        shell: bash
+        run: |
+          cd source
+          if make emqx-ee --dry-run > /dev/null 2>&1; then
+            echo "::set-output name=profiles::[\"emqx-ee\"]"
+          else
+            echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
+          fi
+
+  upload:
+    runs-on: ubuntu-20.04
+    needs: prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Get packages
+        run: |
+          if [ "${{ matrix.profile }}"  == "emqx" ];then
+              s3dir="emqx-ce"
+          else
+              s3dir=${{ matrix.profile }}
+          fi
+          aws s3 cp --recursive  s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} packages
+      - uses: alexellis/upload-assets@0.2.2
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          asset_paths: '["packages/*"]'
+      - name: update to emqx.io
+        run: |
+          set -e -x -u
+          curl -w %{http_code} \
+               --insecure \
+               -H "Content-Type: application/json" \
+               -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
+               -X POST \
+               -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
+               ${{ secrets.EMQX_IO_RELEASE_API }}
+      - name: update repo.emqx.io
+        if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
+        run: |
+          curl --silent --show-error \
+            -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            -X POST \
+            -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\", \"emqx_ee\": \"true\"}}" \
+            "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
+      - name: update repo.emqx.io
+        if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
+        run: |
+          curl --silent --show-error \
+            -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            -X POST \
+            -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\", \"emqx_ce\": \"true\"}}" \
+            "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
+      - name: update homebrew packages
+        if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
+        run: |
+          if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
+              curl --silent --show-error \
+                -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
+                -H "Accept: application/vnd.github.v3+json" \
+                -X POST \
+                -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \
+                "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
+          fi