Merge pull request #9037 from emqx/build-refactor-reusable-steps-into-actions

build: refactor reusable steps into actions
2022-09-26 08:40:57 +02:00 · 2022-09-26 08:40:57 +02:00 · 63817418d1
parent dbf3a3cee7 b74632b5ba
commit 63817418d1
5 changed files with 173 additions and 170 deletions
--- a/.github/actions/detect-profiles/action.yaml
+++ b/.github/actions/detect-profiles/action.yaml
@ -0,0 +1,25 @@
+name: 'Detect profiles'
+inputs:
+  ci_git_token:
+    required: true
+    type: string
+outputs:
+  profiles:
+    description: 'Detected profiles'
+    value: ${{ steps.detect-profiles.outputs.profiles}}
+
+runs:
+  using: composite
+  steps:
+    - id: detect-profiles
+      shell: bash
+      run: |
+        if make emqx-ee --dry-run > /dev/null 2>&1; then
+          echo "::set-output name=profiles::[\"emqx-ee\"]"
+          echo "https://ci%40emqx.io:${{ inputs.ci_git_token }}@github.com" > $HOME/.git-credentials
+          git config --global credential.helper store
+          echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
+        else
+          echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
+          echo "EMQX_NAME=emqx" >> $GITHUB_ENV
+        fi
--- a/.github/actions/package-macos/action.yaml
+++ b/.github/actions/package-macos/action.yaml
@ -0,0 +1,95 @@
+name: 'Create MacOS package'
+inputs:
+  profile: # emqx, emqx-enterprise
+    required: true
+    type: string
+  otp: # 24.2.1-1, 23.3.4.9-3
+    required: true
+    type: string
+  os:
+    required: false
+    type: string
+    default: macos-11
+  apple_id_password:
+    required: true
+    type: string
+  apple_developer_identity:
+    required: true
+    type: string
+  apple_developer_id_bundle:
+    required: true
+    type: string
+  apple_developer_id_bundle_password:
+    required: true
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - name: prepare
+      shell: bash
+      run: |
+        brew update
+        brew install curl zip unzip gnu-sed kerl coreutils unixodbc freetds openssl@1.1
+        echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
+        echo "/usr/local/bin" >> $GITHUB_PATH
+    - uses: actions/cache@v2
+      id: cache
+      with:
+        path: ~/.kerl/${{ inputs.otp }}
+        key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit
+    - name: build erlang
+      if: steps.cache.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        KERL_BUILD_BACKEND: git
+        OTP_GITHUB_URL: https://github.com/emqx/otp
+        KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
+      run: |
+        kerl update releases
+        kerl build ${{ inputs.otp }}
+        kerl install ${{ inputs.otp }} $HOME/.kerl/${{ inputs.otp }}
+    - name: build ${{ inputs.profile }}
+      env:
+        AUTO_INSTALL_BUILD_DEPS: 1
+        APPLE_SIGN_BINARIES: 1
+        APPLE_ID: developers@emqx.io
+        APPLE_TEAM_ID: 26N6HYJLZA
+        APPLE_ID_PASSWORD: ${{ inputs.apple_id_password }}
+        APPLE_DEVELOPER_IDENTITY: ${{ inputs.apple_developer_identity }}
+        APPLE_DEVELOPER_ID_BUNDLE: ${{ inputs.apple_developer_id_bundle }}
+        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }}
+      shell: bash
+      run: |
+        . $HOME/.kerl/${{ inputs.otp }}/activate
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        make ${{ inputs.profile }}-zip
+    - name: test ${{ inputs.profile }}
+      shell: bash
+      run: |
+        pkg_name=$(basename _packages/${{ inputs.profile }}/${{ inputs.profile }}-*.zip)
+        unzip -q _packages/${{ inputs.profile }}/$pkg_name
+        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
+        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
+        ready='no'
+        for i in {1..10}; do
+          if curl -fs 127.0.0.1:18083 > /dev/null; then
+            ready='yes'
+            break
+          fi
+          sleep 1
+        done
+        if [ "$ready" != "yes" ]; then
+          echo "Timed out waiting for emqx to be ready"
+          cat emqx/log/erlang.log.1
+          exit 1
+        fi
+        ./emqx/bin/emqx_ctl status
+        if ! ./emqx/bin/emqx stop; then
+          cat emqx/log/erlang.log.1 || true
+          cat emqx/log/emqx.log.1 || true
+          echo "failed to stop emqx"
+          exit 1
+        fi
+        rm -rf emqx
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@ -23,23 +23,18 @@ jobs:
    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04

    outputs:
-      profiles: ${{ steps.set_profile.outputs.profiles}}
+      profiles: ${{ steps.detect-profiles.outputs.profiles}}

    steps:
      - uses: actions/checkout@v2
        with:
          path: source
          fetch-depth: 0
-      - name: set profile
-        id: set_profile
-        shell: bash
-        run: |
-          cd source
-          if make emqx-ee --dry-run > /dev/null 2>&1; then
-            echo "::set-output name=profiles::[\"emqx-ee\"]"
-          else
-            echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
-          fi
+      - id: detect-profiles
+        working-directory: source
+        uses: ./.github/actions/detect-profiles
+        with:
+          ci_git_token: ${{ secrets.CI_GIT_TOKEN }}
      - name: get_all_deps
        if: endsWith(github.repository, 'emqx')
        run: |
@ -126,13 +121,13 @@ jobs:
    strategy:
      matrix:
        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-        erl_otp:
+        otp:
          - 23.3.4.9-3
        exclude:
          - profile: emqx-edge
-        macos:
-          - macos-10.15
-    runs-on: ${{ matrix.macos  }}
+        os:
+          - macos-11
+    runs-on: ${{ matrix.os }}

    steps:
    - uses: actions/download-artifact@v2
@ -140,70 +135,23 @@ jobs:
        name: source
        path: .
    - name: unzip source code
-      run: unzip -q source.zip
-    - name: prepare
      run: |
-        brew update
-        brew install curl zip unzip gnu-sed kerl unixodbc freetds
-        echo "/usr/local/bin" >> $GITHUB_PATH
-        git config --global credential.helper store
-    - uses: actions/cache@v2
-      id: cache
+        ln -s . source
+        unzip -q source.zip
+        rm source source.zip
+    - uses: ./.github/actions/package-macos
      with:
-        path: ~/.kerl/${{ matrix.erl_otp }}
-        key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}
-    - name: build erlang
-      if: steps.cache.outputs.cache-hit != 'true'
-      timeout-minutes: 60
-      env:
-        KERL_BUILD_BACKEND: git
-        OTP_GITHUB_URL: https://github.com/emqx/otp
-      run: |
-        kerl update releases
-        kerl build ${{ matrix.erl_otp }}
-        kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
-    - name: build
-      run: |
-        . $HOME/.kerl/${{ matrix.erl_otp }}/activate
-        cd source
-        make ensure-rebar3
-        sudo cp rebar3 /usr/local/bin/rebar3
-        rm -rf _build/${{ matrix.profile }}/lib
-        make ${{ matrix.profile }}-zip
-    - name: test
-      run: |
-        cd source
-        pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip)
-        unzip -q _packages/${{ matrix.profile }}/$pkg_name
-        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
-        ready='no'
-        for i in {1..10}; do
-          if curl -fs 127.0.0.1:18083 > /dev/null; then
-            ready='yes'
-            break
-          fi
-          sleep 1
-        done
-        if [ "$ready" != "yes" ]; then
-          echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
-          exit 1
-        fi
-        ./emqx/bin/emqx_ctl status
-        if ! ./emqx/bin/emqx stop; then
-          cat emqx/log/erlang.log.1 || true
-          cat emqx/log/emqx.log.1 || true
-          echo "failed to stop emqx"
-          exit 1
-        fi
-        rm -rf emqx
-        #sha256sum ./_packages/${{ matrix.profile }}/$pkg_name | head -c64  > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
-        openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}'  > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
+        profile: ${{ matrix.profile }}
+        otp: ${{ matrix.otp }}
+        os: ${{ matrix.os }}
+        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
+        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
+        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
+        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
    - uses: actions/upload-artifact@v1
      with:
-        name: ${{ matrix.profile }}
-        path: source/_packages/${{ matrix.profile }}/.
+        name: ${{ matrix.profile }}-${{ matrix.otp }}
+        path: _packages/${{ matrix.profile }}/.

  linux:
    runs-on: ubuntu-20.04
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@ -10,7 +10,7 @@ jobs:

    strategy:
      matrix:
-        erl_otp:
+        otp:
        - erl23.3.4.9-3
        os:
        - ubuntu20.04
@ -26,26 +26,20 @@ jobs:
          - runs-on: aws-amd64
            use-self-hosted: false

-    container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: emqx/build-env:${{ matrix.otp }}-${{ matrix.os }}

    steps:
    - uses: actions/checkout@v1
-    - name: prepare
-      run: |
-        if make emqx-ee --dry-run > /dev/null 2>&1; then
-          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-          git config --global credential.helper store
-          echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
-        else
-          echo "EMQX_NAME=emqx" >> $GITHUB_ENV
-        fi
+    - uses: ./.github/actions/detect-profiles
+      with:
+        ci_git_token: ${{ secrets.CI_GIT_TOKEN }}
    - name: fix-git-unsafe-repository
      run: git config --global --add safe.directory /__w/emqx/emqx
    - uses: actions/cache@v2
      with:
        # dialyzer PLTs
        path: ~/.cache/rebar3/
-        key: dialyer-${{ matrix.erl_otp }}
+        key: dialyer-${{ matrix.otp }}
    - name: make xref
      run: make xref
    - name: make dialyzer
@ -71,85 +65,32 @@ jobs:
  mac:
    strategy:
      matrix:
-        erl_otp:
-        - 23.3.4.9-3
-        macos:
-        - macos-11
-    runs-on: ${{ matrix.macos }}
+        profile:
+          - emqx
+        otp:
+          - 23.3.4.9-3
+        os:
+          - macos-11
+    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/checkout@v1
-    - name: prepare
-      run: |
-        if make emqx-ee --dry-run > /dev/null 2>&1; then
-          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-          git config --global credential.helper store
-          echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
-        else
-          echo "EMQX_NAME=emqx" >> $GITHUB_ENV
-        fi
-    - name: prepare
-      run: |
-        brew update
-        brew install curl zip unzip gnu-sed kerl unixodbc freetds
-        echo "/usr/local/bin" >> $GITHUB_PATH
-        git config --global credential.helper store
-    - uses: actions/cache@v2
-      id: cache
+    - uses: ./.github/actions/detect-profiles
      with:
-        path: ~/.kerl/${{ matrix.erl_otp }}
-        key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}-static-ssl-disable-hipe-disable-jit
-    - name: build erlang
-      if: steps.cache.outputs.cache-hit != 'true'
-      timeout-minutes: 60
-      env:
-        KERL_BUILD_BACKEND: git
-        OTP_GITHUB_URL: https://github.com/emqx/otp
-        KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
-      run: |
-        kerl update releases
-        kerl build ${{ matrix.erl_otp }}
-        kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
-    - name: build
-      env:
-        APPLE_SIGN_BINARIES: 1
-        APPLE_ID: developers@emqx.io
-        APPLE_TEAM_ID: 26N6HYJLZA
-        APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-        APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-        APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-      run: |
-        . $HOME/.kerl/${{ matrix.erl_otp }}/activate
-        make ensure-rebar3
-        sudo cp rebar3 /usr/local/bin/rebar3
-        make ${EMQX_NAME}-zip
+        ci_git_token: ${{ secrets.CI_GIT_TOKEN }}
+    - uses: ./.github/actions/package-macos
+      with:
+        profile: ${{ matrix.profile }}
+        otp: ${{ matrix.otp }}
+        os: ${{ matrix.os }}
+        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
+        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
+        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
+        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
    - uses: actions/upload-artifact@v1
      if: failure()
      with:
        name: rebar3.crashdump
        path: ./rebar3.crashdump
-    - name: test
-      run: |
-        pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip)
-        unzip -q _packages/${EMQX_NAME}/$pkg_name
-        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
-        ready='no'
-        for i in {1..10}; do
-          if curl -fs 127.0.0.1:18083 > /dev/null; then
-            ready='yes'
-            break
-          fi
-          sleep 1
-        done
-        if [ "$ready" != "yes" ]; then
-          echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
-          exit 1
-        fi
-        ./emqx/bin/emqx_ctl status
-        ./emqx/bin/emqx stop
-        rm -rf emqx
    - uses: actions/upload-artifact@v2
      with:
        name: macos
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@ -10,24 +10,18 @@ jobs:
    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04

    outputs:
-      profiles: ${{ steps.set_profile.outputs.profiles}}
-      s3dir: ${{ steps.set_profile.outputs.s3dir}}
+      profiles: ${{ steps.detect-profiles.outputs.profiles}}

    steps:
      - uses: actions/checkout@v2
        with:
          path: source
          fetch-depth: 0
-      - name: set profile
-        id: set_profile
-        shell: bash
-        run: |
-          cd source
-          if make emqx-ee --dry-run > /dev/null 2>&1; then
-            echo "::set-output name=profiles::[\"emqx-ee\"]"
-          else
-            echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
-          fi
+      - id: detect-profiles
+        working-directory: source
+        uses: ./.github/actions/detect-profiles
+        with:
+          ci_git_token: ${{ secrets.CI_GIT_TOKEN }}

  upload:
    runs-on: ubuntu-20.04