From b74632b5bace9100d7112872c38be46932ca389c Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Fri, 23 Sep 2022 14:34:17 +0200 Subject: [PATCH] build: refactor reusable steps into actions --- .github/actions/detect-profiles/action.yaml | 25 +++++ .github/actions/package-macos/action.yaml | 95 +++++++++++++++++ .github/workflows/build_packages.yaml | 98 +++++------------- .github/workflows/build_slim_packages.yaml | 107 +++++--------------- .github/workflows/release.yaml | 18 ++-- 5 files changed, 173 insertions(+), 170 deletions(-) create mode 100644 .github/actions/detect-profiles/action.yaml create mode 100644 .github/actions/package-macos/action.yaml diff --git a/.github/actions/detect-profiles/action.yaml b/.github/actions/detect-profiles/action.yaml new file mode 100644 index 000000000..f28191f88 --- /dev/null +++ b/.github/actions/detect-profiles/action.yaml @@ -0,0 +1,25 @@ +name: 'Detect profiles' +inputs: + ci_git_token: + required: true + type: string +outputs: + profiles: + description: 'Detected profiles' + value: ${{ steps.detect-profiles.outputs.profiles}} + +runs: + using: composite + steps: + - id: detect-profiles + shell: bash + run: | + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "::set-output name=profiles::[\"emqx-ee\"]" + echo "https://ci%40emqx.io:${{ inputs.ci_git_token }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV + else + echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]" + echo "EMQX_NAME=emqx" >> $GITHUB_ENV + fi diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml new file mode 100644 index 000000000..5c0649b17 --- /dev/null +++ b/.github/actions/package-macos/action.yaml @@ -0,0 +1,95 @@ +name: 'Create MacOS package' +inputs: + profile: # emqx, emqx-enterprise + required: true + type: string + otp: # 24.2.1-1, 23.3.4.9-3 + required: true + type: string + os: + required: false + type: string + default: macos-11 + apple_id_password: + required: true + type: string + apple_developer_identity: + required: true + type: string + apple_developer_id_bundle: + required: true + type: string + apple_developer_id_bundle_password: + required: true + type: string + +runs: + using: composite + steps: + - name: prepare + shell: bash + run: | + brew update + brew install curl zip unzip gnu-sed kerl coreutils unixodbc freetds openssl@1.1 + echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH + echo "/usr/local/bin" >> $GITHUB_PATH + - uses: actions/cache@v2 + id: cache + with: + path: ~/.kerl/${{ inputs.otp }} + key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit + - name: build erlang + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + env: + KERL_BUILD_BACKEND: git + OTP_GITHUB_URL: https://github.com/emqx/otp + KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit + run: | + kerl update releases + kerl build ${{ inputs.otp }} + kerl install ${{ inputs.otp }} $HOME/.kerl/${{ inputs.otp }} + - name: build ${{ inputs.profile }} + env: + AUTO_INSTALL_BUILD_DEPS: 1 + APPLE_SIGN_BINARIES: 1 + APPLE_ID: developers@emqx.io + APPLE_TEAM_ID: 26N6HYJLZA + APPLE_ID_PASSWORD: ${{ inputs.apple_id_password }} + APPLE_DEVELOPER_IDENTITY: ${{ inputs.apple_developer_identity }} + APPLE_DEVELOPER_ID_BUNDLE: ${{ inputs.apple_developer_id_bundle }} + APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }} + shell: bash + run: | + . $HOME/.kerl/${{ inputs.otp }}/activate + make ensure-rebar3 + sudo cp rebar3 /usr/local/bin/rebar3 + make ${{ inputs.profile }}-zip + - name: test ${{ inputs.profile }} + shell: bash + run: | + pkg_name=$(basename _packages/${{ inputs.profile }}/${{ inputs.profile }}-*.zip) + unzip -q _packages/${{ inputs.profile }}/$pkg_name + gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins + ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 + ready='no' + for i in {1..10}; do + if curl -fs 127.0.0.1:18083 > /dev/null; then + ready='yes' + break + fi + sleep 1 + done + if [ "$ready" != "yes" ]; then + echo "Timed out waiting for emqx to be ready" + cat emqx/log/erlang.log.1 + exit 1 + fi + ./emqx/bin/emqx_ctl status + if ! ./emqx/bin/emqx stop; then + cat emqx/log/erlang.log.1 || true + cat emqx/log/emqx.log.1 || true + echo "failed to stop emqx" + exit 1 + fi + rm -rf emqx diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index f1590768d..f2024fb5b 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -23,23 +23,18 @@ jobs: container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 outputs: - profiles: ${{ steps.set_profile.outputs.profiles}} + profiles: ${{ steps.detect-profiles.outputs.profiles}} steps: - uses: actions/checkout@v2 with: path: source fetch-depth: 0 - - name: set profile - id: set_profile - shell: bash - run: | - cd source - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "::set-output name=profiles::[\"emqx-ee\"]" - else - echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]" - fi + - id: detect-profiles + working-directory: source + uses: ./.github/actions/detect-profiles + with: + ci_git_token: ${{ secrets.CI_GIT_TOKEN }} - name: get_all_deps if: endsWith(github.repository, 'emqx') run: | @@ -126,13 +121,13 @@ jobs: strategy: matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} - erl_otp: + otp: - 23.3.4.9-3 exclude: - profile: emqx-edge - macos: - - macos-10.15 - runs-on: ${{ matrix.macos }} + os: + - macos-11 + runs-on: ${{ matrix.os }} steps: - uses: actions/download-artifact@v2 @@ -140,70 +135,23 @@ jobs: name: source path: . - name: unzip source code - run: unzip -q source.zip - - name: prepare run: | - brew update - brew install curl zip unzip gnu-sed kerl unixodbc freetds - echo "/usr/local/bin" >> $GITHUB_PATH - git config --global credential.helper store - - uses: actions/cache@v2 - id: cache + ln -s . source + unzip -q source.zip + rm source source.zip + - uses: ./.github/actions/package-macos with: - path: ~/.kerl/${{ matrix.erl_otp }} - key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }} - - name: build erlang - if: steps.cache.outputs.cache-hit != 'true' - timeout-minutes: 60 - env: - KERL_BUILD_BACKEND: git - OTP_GITHUB_URL: https://github.com/emqx/otp - run: | - kerl update releases - kerl build ${{ matrix.erl_otp }} - kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} - - name: build - run: | - . $HOME/.kerl/${{ matrix.erl_otp }}/activate - cd source - make ensure-rebar3 - sudo cp rebar3 /usr/local/bin/rebar3 - rm -rf _build/${{ matrix.profile }}/lib - make ${{ matrix.profile }}-zip - - name: test - run: | - cd source - pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip) - unzip -q _packages/${{ matrix.profile }}/$pkg_name - gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins - ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 - ready='no' - for i in {1..10}; do - if curl -fs 127.0.0.1:18083 > /dev/null; then - ready='yes' - break - fi - sleep 1 - done - if [ "$ready" != "yes" ]; then - echo "Timed out waiting for emqx to be ready" - cat emqx/log/erlang.log.1 - exit 1 - fi - ./emqx/bin/emqx_ctl status - if ! ./emqx/bin/emqx stop; then - cat emqx/log/erlang.log.1 || true - cat emqx/log/emqx.log.1 || true - echo "failed to stop emqx" - exit 1 - fi - rm -rf emqx - #sha256sum ./_packages/${{ matrix.profile }}/$pkg_name | head -c64 > ./_packages/${{ matrix.profile }}/$pkg_name.sha256 - openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}' > ./_packages/${{ matrix.profile }}/$pkg_name.sha256 + profile: ${{ matrix.profile }} + otp: ${{ matrix.otp }} + os: ${{ matrix.os }} + apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} + apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} + apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} + apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - uses: actions/upload-artifact@v1 with: - name: ${{ matrix.profile }} - path: source/_packages/${{ matrix.profile }}/. + name: ${{ matrix.profile }}-${{ matrix.otp }} + path: _packages/${{ matrix.profile }}/. linux: runs-on: ubuntu-20.04 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 12be1a46c..c0bba8c96 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -10,7 +10,7 @@ jobs: strategy: matrix: - erl_otp: + otp: - erl23.3.4.9-3 os: - ubuntu20.04 @@ -26,26 +26,20 @@ jobs: - runs-on: aws-amd64 use-self-hosted: false - container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }} + container: emqx/build-env:${{ matrix.otp }}-${{ matrix.os }} steps: - uses: actions/checkout@v1 - - name: prepare - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV - else - echo "EMQX_NAME=emqx" >> $GITHUB_ENV - fi + - uses: ./.github/actions/detect-profiles + with: + ci_git_token: ${{ secrets.CI_GIT_TOKEN }} - name: fix-git-unsafe-repository run: git config --global --add safe.directory /__w/emqx/emqx - uses: actions/cache@v2 with: # dialyzer PLTs path: ~/.cache/rebar3/ - key: dialyer-${{ matrix.erl_otp }} + key: dialyer-${{ matrix.otp }} - name: make xref run: make xref - name: make dialyzer @@ -71,85 +65,32 @@ jobs: mac: strategy: matrix: - erl_otp: - - 23.3.4.9-3 - macos: - - macos-11 - runs-on: ${{ matrix.macos }} + profile: + - emqx + otp: + - 23.3.4.9-3 + os: + - macos-11 + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v1 - - name: prepare - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV - else - echo "EMQX_NAME=emqx" >> $GITHUB_ENV - fi - - name: prepare - run: | - brew update - brew install curl zip unzip gnu-sed kerl unixodbc freetds - echo "/usr/local/bin" >> $GITHUB_PATH - git config --global credential.helper store - - uses: actions/cache@v2 - id: cache + - uses: ./.github/actions/detect-profiles with: - path: ~/.kerl/${{ matrix.erl_otp }} - key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}-static-ssl-disable-hipe-disable-jit - - name: build erlang - if: steps.cache.outputs.cache-hit != 'true' - timeout-minutes: 60 - env: - KERL_BUILD_BACKEND: git - OTP_GITHUB_URL: https://github.com/emqx/otp - KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit - run: | - kerl update releases - kerl build ${{ matrix.erl_otp }} - kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} - - name: build - env: - APPLE_SIGN_BINARIES: 1 - APPLE_ID: developers@emqx.io - APPLE_TEAM_ID: 26N6HYJLZA - APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }} - APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} - APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} - APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - run: | - . $HOME/.kerl/${{ matrix.erl_otp }}/activate - make ensure-rebar3 - sudo cp rebar3 /usr/local/bin/rebar3 - make ${EMQX_NAME}-zip + ci_git_token: ${{ secrets.CI_GIT_TOKEN }} + - uses: ./.github/actions/package-macos + with: + profile: ${{ matrix.profile }} + otp: ${{ matrix.otp }} + os: ${{ matrix.os }} + apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }} + apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }} + apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }} + apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }} - uses: actions/upload-artifact@v1 if: failure() with: name: rebar3.crashdump path: ./rebar3.crashdump - - name: test - run: | - pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) - unzip -q _packages/${EMQX_NAME}/$pkg_name - gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins - ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 - ready='no' - for i in {1..10}; do - if curl -fs 127.0.0.1:18083 > /dev/null; then - ready='yes' - break - fi - sleep 1 - done - if [ "$ready" != "yes" ]; then - echo "Timed out waiting for emqx to be ready" - cat emqx/log/erlang.log.1 - exit 1 - fi - ./emqx/bin/emqx_ctl status - ./emqx/bin/emqx stop - rm -rf emqx - uses: actions/upload-artifact@v2 with: name: macos diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index bdbd7e982..71e5fe48a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,24 +10,18 @@ jobs: container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 outputs: - profiles: ${{ steps.set_profile.outputs.profiles}} - s3dir: ${{ steps.set_profile.outputs.s3dir}} + profiles: ${{ steps.detect-profiles.outputs.profiles}} steps: - uses: actions/checkout@v2 with: path: source fetch-depth: 0 - - name: set profile - id: set_profile - shell: bash - run: | - cd source - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "::set-output name=profiles::[\"emqx-ee\"]" - else - echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]" - fi + - id: detect-profiles + working-directory: source + uses: ./.github/actions/detect-profiles + with: + ci_git_token: ${{ secrets.CI_GIT_TOKEN }} upload: runs-on: ubuntu-20.04