Merge pull request #9037 from emqx/build-refactor-reusable-steps-into-actions

build: refactor reusable steps into actions

.github/actions/detect-profiles/action.yaml

@@ -0,0 +1,25 @@
+name: 'Detect profiles'
+inputs:
+  ci_git_token:
+    required: true
+    type: string
+outputs:
+  profiles:
+    description: 'Detected profiles'
+    value: ${{ steps.detect-profiles.outputs.profiles}}
+
+runs:
+  using: composite
+  steps:
+    - id: detect-profiles
+      shell: bash
+      run: |
+        if make emqx-ee --dry-run > /dev/null 2>&1; then
+          echo "::set-output name=profiles::[\"emqx-ee\"]"
+          echo "https://ci%40emqx.io:${{ inputs.ci_git_token }}@github.com" > $HOME/.git-credentials
+          git config --global credential.helper store
+          echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
+        else
+          echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
+          echo "EMQX_NAME=emqx" >> $GITHUB_ENV
+        fi

.github/actions/package-macos/action.yaml

@@ -0,0 +1,95 @@
+name: 'Create MacOS package'
+inputs:
+  profile: # emqx, emqx-enterprise
+    required: true
+    type: string
+  otp: # 24.2.1-1, 23.3.4.9-3
+    required: true
+    type: string
+  os:
+    required: false
+    type: string
+    default: macos-11
+  apple_id_password:
+    required: true
+    type: string
+  apple_developer_identity:
+    required: true
+    type: string
+  apple_developer_id_bundle:
+    required: true
+    type: string
+  apple_developer_id_bundle_password:
+    required: true
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - name: prepare
+      shell: bash
+      run: |
+        brew update
+        brew install curl zip unzip gnu-sed kerl coreutils unixodbc freetds openssl@1.1
+        echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
+        echo "/usr/local/bin" >> $GITHUB_PATH
+    - uses: actions/cache@v2
+      id: cache
+      with:
+        path: ~/.kerl/${{ inputs.otp }}
+        key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit
+    - name: build erlang
+      if: steps.cache.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        KERL_BUILD_BACKEND: git
+        OTP_GITHUB_URL: https://github.com/emqx/otp
+        KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
+      run: |
+        kerl update releases
+        kerl build ${{ inputs.otp }}
+        kerl install ${{ inputs.otp }} $HOME/.kerl/${{ inputs.otp }}
+    - name: build ${{ inputs.profile }}
+      env:
+        AUTO_INSTALL_BUILD_DEPS: 1
+        APPLE_SIGN_BINARIES: 1
+        APPLE_ID: developers@emqx.io
+        APPLE_TEAM_ID: 26N6HYJLZA
+        APPLE_ID_PASSWORD: ${{ inputs.apple_id_password }}
+        APPLE_DEVELOPER_IDENTITY: ${{ inputs.apple_developer_identity }}
+        APPLE_DEVELOPER_ID_BUNDLE: ${{ inputs.apple_developer_id_bundle }}
+        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }}
+      shell: bash
+      run: |
+        . $HOME/.kerl/${{ inputs.otp }}/activate
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        make ${{ inputs.profile }}-zip
+    - name: test ${{ inputs.profile }}
+      shell: bash
+      run: |
+        pkg_name=$(basename _packages/${{ inputs.profile }}/${{ inputs.profile }}-*.zip)
+        unzip -q _packages/${{ inputs.profile }}/$pkg_name
+        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
+        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
+        ready='no'
+        for i in {1..10}; do
+          if curl -fs 127.0.0.1:18083 > /dev/null; then
+            ready='yes'
+            break
+          fi
+          sleep 1
+        done
+        if [ "$ready" != "yes" ]; then
+          echo "Timed out waiting for emqx to be ready"
+          cat emqx/log/erlang.log.1
+          exit 1
+        fi
+        ./emqx/bin/emqx_ctl status
+        if ! ./emqx/bin/emqx stop; then
+          cat emqx/log/erlang.log.1 || true
+          cat emqx/log/emqx.log.1 || true
+          echo "failed to stop emqx"
+          exit 1
+        fi
+        rm -rf emqx

.github/workflows/build_packages.yaml

@@ -23,23 +23,18 @@ jobs:
     container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 
     outputs:
-      profiles: ${{ steps.set_profile.outputs.profiles}}
+      profiles: ${{ steps.detect-profiles.outputs.profiles}}
 
     steps:
       - uses: actions/checkout@v2
         with:
           path: source
           fetch-depth: 0
-      - name: set profile
+      - id: detect-profiles
-        id: set_profile
+        working-directory: source
-        shell: bash
+        uses: ./.github/actions/detect-profiles
-        run: |
+        with:
-          cd source
+          ci_git_token: ${{ secrets.CI_GIT_TOKEN }}
-          if make emqx-ee --dry-run > /dev/null 2>&1; then
-            echo "::set-output name=profiles::[\"emqx-ee\"]"
-          else
-            echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
-          fi
       - name: get_all_deps
         if: endsWith(github.repository, 'emqx')
         run: |
@@ -126,13 +121,13 @@ jobs:
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-        erl_otp:
+        otp:
           - 23.3.4.9-3
         exclude:
           - profile: emqx-edge
-        macos:
+        os:
-          - macos-10.15
+          - macos-11
-    runs-on: ${{ matrix.macos  }}
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/download-artifact@v2
@@ -140,70 +135,23 @@ jobs:
         name: source
         path: .
     - name: unzip source code
-      run: unzip -q source.zip
-    - name: prepare
       run: |
-        brew update
+        ln -s . source
-        brew install curl zip unzip gnu-sed kerl unixodbc freetds
+        unzip -q source.zip
-        echo "/usr/local/bin" >> $GITHUB_PATH
+        rm source source.zip
-        git config --global credential.helper store
+    - uses: ./.github/actions/package-macos
-    - uses: actions/cache@v2
-      id: cache
       with:
-        path: ~/.kerl/${{ matrix.erl_otp }}
+        profile: ${{ matrix.profile }}
-        key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}
+        otp: ${{ matrix.otp }}
-    - name: build erlang
+        os: ${{ matrix.os }}
-      if: steps.cache.outputs.cache-hit != 'true'
+        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
-      timeout-minutes: 60
+        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-      env:
+        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-        KERL_BUILD_BACKEND: git
+        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-        OTP_GITHUB_URL: https://github.com/emqx/otp
-      run: |
-        kerl update releases
-        kerl build ${{ matrix.erl_otp }}
-        kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
-    - name: build
-      run: |
-        . $HOME/.kerl/${{ matrix.erl_otp }}/activate
-        cd source
-        make ensure-rebar3
-        sudo cp rebar3 /usr/local/bin/rebar3
-        rm -rf _build/${{ matrix.profile }}/lib
-        make ${{ matrix.profile }}-zip
-    - name: test
-      run: |
-        cd source
-        pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip)
-        unzip -q _packages/${{ matrix.profile }}/$pkg_name
-        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
-        ready='no'
-        for i in {1..10}; do
-          if curl -fs 127.0.0.1:18083 > /dev/null; then
-            ready='yes'
-            break
-          fi
-          sleep 1
-        done
-        if [ "$ready" != "yes" ]; then
-          echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
-          exit 1
-        fi
-        ./emqx/bin/emqx_ctl status
-        if ! ./emqx/bin/emqx stop; then
-          cat emqx/log/erlang.log.1 || true
-          cat emqx/log/emqx.log.1 || true
-          echo "failed to stop emqx"
-          exit 1
-        fi
-        rm -rf emqx
-        #sha256sum ./_packages/${{ matrix.profile }}/$pkg_name | head -c64  > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
-        openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}'  > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
     - uses: actions/upload-artifact@v1
       with:
-        name: ${{ matrix.profile }}
+        name: ${{ matrix.profile }}-${{ matrix.otp }}
-        path: source/_packages/${{ matrix.profile }}/.
+        path: _packages/${{ matrix.profile }}/.
 
   linux:
     runs-on: ubuntu-20.04

.github/workflows/build_slim_packages.yaml

@@ -10,7 +10,7 @@ jobs:
 
     strategy:
       matrix:
-        erl_otp:
+        otp:
         - erl23.3.4.9-3
         os:
         - ubuntu20.04
@@ -26,26 +26,20 @@ jobs:
           - runs-on: aws-amd64
             use-self-hosted: false
 
-    container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: emqx/build-env:${{ matrix.otp }}-${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v1
-    - name: prepare
+    - uses: ./.github/actions/detect-profiles
-      run: |
+      with:
-        if make emqx-ee --dry-run > /dev/null 2>&1; then
+        ci_git_token: ${{ secrets.CI_GIT_TOKEN }}
-          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-          git config --global credential.helper store
-          echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
-        else
-          echo "EMQX_NAME=emqx" >> $GITHUB_ENV
-        fi
     - name: fix-git-unsafe-repository
       run: git config --global --add safe.directory /__w/emqx/emqx
     - uses: actions/cache@v2
       with:
         # dialyzer PLTs
         path: ~/.cache/rebar3/
-        key: dialyer-${{ matrix.erl_otp }}
+        key: dialyer-${{ matrix.otp }}
     - name: make xref
       run: make xref
     - name: make dialyzer
@@ -71,85 +65,32 @@ jobs:
   mac:
     strategy:
       matrix:
-        erl_otp:
+        profile:
-        - 23.3.4.9-3
+          - emqx
-        macos:
+        otp:
-        - macos-11
+          - 23.3.4.9-3
-    runs-on: ${{ matrix.macos }}
+        os:
+          - macos-11
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v1
-    - name: prepare
+    - uses: ./.github/actions/detect-profiles
-      run: |
-        if make emqx-ee --dry-run > /dev/null 2>&1; then
-          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-          git config --global credential.helper store
-          echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
-        else
-          echo "EMQX_NAME=emqx" >> $GITHUB_ENV
-        fi
-    - name: prepare
-      run: |
-        brew update
-        brew install curl zip unzip gnu-sed kerl unixodbc freetds
-        echo "/usr/local/bin" >> $GITHUB_PATH
-        git config --global credential.helper store
-    - uses: actions/cache@v2
-      id: cache
       with:
-        path: ~/.kerl/${{ matrix.erl_otp }}
+        ci_git_token: ${{ secrets.CI_GIT_TOKEN }}
-        key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}-static-ssl-disable-hipe-disable-jit
+    - uses: ./.github/actions/package-macos
-    - name: build erlang
+      with:
-      if: steps.cache.outputs.cache-hit != 'true'
+        profile: ${{ matrix.profile }}
-      timeout-minutes: 60
+        otp: ${{ matrix.otp }}
-      env:
+        os: ${{ matrix.os }}
-        KERL_BUILD_BACKEND: git
+        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
-        OTP_GITHUB_URL: https://github.com/emqx/otp
+        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-        KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
+        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-      run: |
+        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-        kerl update releases
-        kerl build ${{ matrix.erl_otp }}
-        kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
-    - name: build
-      env:
-        APPLE_SIGN_BINARIES: 1
-        APPLE_ID: developers@emqx.io
-        APPLE_TEAM_ID: 26N6HYJLZA
-        APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-        APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-        APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-      run: |
-        . $HOME/.kerl/${{ matrix.erl_otp }}/activate
-        make ensure-rebar3
-        sudo cp rebar3 /usr/local/bin/rebar3
-        make ${EMQX_NAME}-zip
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
         name: rebar3.crashdump
         path: ./rebar3.crashdump
-    - name: test
-      run: |
-        pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip)
-        unzip -q _packages/${EMQX_NAME}/$pkg_name
-        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
-        ready='no'
-        for i in {1..10}; do
-          if curl -fs 127.0.0.1:18083 > /dev/null; then
-            ready='yes'
-            break
-          fi
-          sleep 1
-        done
-        if [ "$ready" != "yes" ]; then
-          echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
-          exit 1
-        fi
-        ./emqx/bin/emqx_ctl status
-        ./emqx/bin/emqx stop
-        rm -rf emqx
     - uses: actions/upload-artifact@v2
       with:
         name: macos

.github/workflows/release.yaml

@@ -10,24 +10,18 @@ jobs:
     container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 
     outputs:
-      profiles: ${{ steps.set_profile.outputs.profiles}}
+      profiles: ${{ steps.detect-profiles.outputs.profiles}}
-      s3dir: ${{ steps.set_profile.outputs.s3dir}}
 
     steps:
       - uses: actions/checkout@v2
         with:
           path: source
           fetch-depth: 0
-      - name: set profile
+      - id: detect-profiles
-        id: set_profile
+        working-directory: source
-        shell: bash
+        uses: ./.github/actions/detect-profiles
-        run: |
+        with:
-          cd source
+          ci_git_token: ${{ secrets.CI_GIT_TOKEN }}
-          if make emqx-ee --dry-run > /dev/null 2>&1; then
-            echo "::set-output name=profiles::[\"emqx-ee\"]"
-          else
-            echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
-          fi
 
   upload:
     runs-on: ubuntu-20.04