yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ci: port changes made in master branch

This commit is contained in:
Zaiming Shi 2021-11-19 21:54:04 +01:00
parent daeac6edf4
commit 099e2a8752
2 changed files with 54 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
.github/workflows/build_packages.yaml vendored
View File

@ -1,5 +1,9 @@
name: Cross build packages name: Cross build packages
concurrency:
group: build-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on: on:
schedule: schedule:
- cron: '0 */6 * * *' - cron: '0 */6 * * *'
@ -11,11 +15,12 @@ on:
jobs: jobs:
prepare: prepare:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
# prepare source with any OTP version, no need for a matrix
container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04 container: ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-ubuntu20.04
outputs: outputs:
profiles: ${{ steps.set_profile.outputs.profiles}} profiles: ${{ steps.set_profile.outputs.profiles }}
old_vsns: ${{ steps.set_profile.outputs.old_vsns}} old_vsns: ${{ steps.set_profile.outputs.old_vsns }}
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -25,8 +30,8 @@ jobs:
- name: set profile - name: set profile
id: set_profile id: set_profile
shell: bash shell: bash
working-directory: source
run: | run: |
cd source
vsn="$(./pkg-vsn.sh)" vsn="$(./pkg-vsn.sh)"
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
if make emqx-ee --dry-run > /dev/null 2>&1; then if make emqx-ee --dry-run > /dev/null 2>&1; then
@ -43,7 +48,7 @@ jobs:
run: | run: |
make -C source deps-all make -C source deps-all
zip -ryq source.zip source/* source/.[^.]* zip -ryq source.zip source/* source/.[^.]*
- name: get_all_deps - name: get_all_deps_ee
if: endsWith(github.repository, 'enterprise') if: endsWith(github.repository, 'enterprise')
run: | run: |
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
@ -63,6 +68,7 @@ jobs:
if: endsWith(github.repository, 'emqx') if: endsWith(github.repository, 'emqx')
strategy: strategy:
fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp: otp:
@ -78,14 +84,16 @@ jobs:
- name: unzip source code - name: unzip source code
run: Expand-Archive -Path source.zip -DestinationPath ./ run: Expand-Archive -Path source.zip -DestinationPath ./
- uses: ilammy/msvc-dev-cmd@v1 - uses: ilammy/msvc-dev-cmd@v1
- uses: gleam-lang/setup-erlang@v1.1.0 - uses: gleam-lang/setup-erlang@v1.1.2
id: install_erlang id: install_erlang
## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows
with: with:
otp-version: ${{ matrix.otp }} otp-version: ${{ matrix.otp }}
- name: build - name: build
env: env:
PYTHON: python PYTHON: python
DIAGNOSTIC: 1 DIAGNOSTIC: 1
working-directory: source
run: | run: |
$env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH"
@ -97,7 +105,6 @@ jobs:
else { else {
$pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip"
} }
cd source
## We do not build/release bcrypt for windows package ## We do not build/release bcrypt for windows package
Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
if (Test-Path rebar.lock) { if (Test-Path rebar.lock) {
@ -114,8 +121,8 @@ jobs:
Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256 Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256
- name: run emqx - name: run emqx
timeout-minutes: 1 timeout-minutes: 1
working-directory: source
run: | run: |
cd source
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
Start-Sleep -s 5 Start-Sleep -s 5
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
@ -128,18 +135,19 @@ jobs:
path: source/_packages/${{ matrix.profile }}/. path: source/_packages/${{ matrix.profile }}/.
mac: mac:
runs-on: macos-10.15
needs: prepare needs: prepare
strategy: strategy:
fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
erl_otp: erl_otp:
- 23.3.4.9-3 - 23.3.4.9-3
macos:
- macos-11
- macos-10.15
exclude: exclude:
- profile: emqx-edge - profile: emqx-edge
runs-on: ${{ matrix.macos }}
steps: steps:
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
@ -157,7 +165,7 @@ jobs:
id: cache id: cache
with: with:
path: ~/.kerl path: ~/.kerl
key: erl${{ matrix.erl_otp }}-macos10.15 key: erl${{ matrix.erl_otp }}-${{ matrix.macos }}
- name: build erlang - name: build erlang
if: steps.cache.outputs.cache-hit != 'true' if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60 timeout-minutes: 60
@ -169,18 +177,19 @@ jobs:
kerl build ${{ matrix.erl_otp }} kerl build ${{ matrix.erl_otp }}
kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }} kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
- name: build - name: build
working-directory: source
run: | run: |
. $HOME/.kerl/${{ matrix.erl_otp }}/activate . $HOME/.kerl/${{ matrix.erl_otp }}/activate
cd source
make ensure-rebar3 make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3 sudo cp rebar3 /usr/local/bin/rebar3
rm -rf _build/${{ matrix.profile }}/lib rm -rf _build/${{ matrix.profile }}/lib
make ${{ matrix.profile }}-zip make ${{ matrix.profile }}-zip
- name: test - name: test
working-directory: source
run: | run: |
cd source set -x
pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip) pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
unzip -q _packages/${{ matrix.profile }}/$pkg_name unzip -q $pkg_name
gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no' ready='no'
@ -199,11 +208,11 @@ jobs:
./emqx/bin/emqx_ctl status ./emqx/bin/emqx_ctl status
./emqx/bin/emqx stop ./emqx/bin/emqx stop
rm -rf emqx rm -rf emqx
openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}' > ./_packages/${{ matrix.profile }}/$pkg_name.sha256 openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/') if: startsWith(github.ref, 'refs/tags/')
with: with:
name: ${{ matrix.profile }} name: ${{ matrix.profile }}-${{ matrix.otp }}
path: source/_packages/${{ matrix.profile }}/. path: source/_packages/${{ matrix.profile }}/.
linux: linux:
@ -212,6 +221,7 @@ jobs:
needs: prepare needs: prepare
strategy: strategy:
fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp: otp:
@ -270,6 +280,7 @@ jobs:
ARCH: ${{ matrix.arch }} ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }} SYSTEM: ${{ matrix.os }}
OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }} OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
working-directory: source
run: | run: |
set -e -x -u set -e -x -u
broker=$PROFILE broker=$PROFILE
@ -280,8 +291,8 @@ jobs:
export ARCH="arm" export ARCH="arm"
fi fi
mkdir -p source/_upgrade_base mkdir -p _upgrade_base
cd source/_upgrade_base cd _upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for tag in ${old_vsns[@]}; do for tag in ${old_vsns[@]}; do
package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
@ -322,15 +333,15 @@ jobs:
- uses: actions/upload-artifact@v1 - uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/') if: startsWith(github.ref, 'refs/tags/')
with: with:
name: ${{ matrix.profile }} name: ${{ matrix.profile }}-${{ matrix.otp }}
path: source/_packages/${{ matrix.profile }}/. path: source/_packages/${{ matrix.profile }}/.
docker: docker:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
needs: prepare needs: prepare
strategy: strategy:
fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp: otp:
@ -399,6 +410,8 @@ jobs:
strategy: strategy:
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp:
- 23.3.4.9-3
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
@ -409,7 +422,7 @@ jobs:
echo 'EOF' >> $GITHUB_ENV echo 'EOF' >> $GITHUB_ENV
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: ${{ matrix.profile }} name: ${{ matrix.profile }}-${{ matrix.otp }}
path: ./_packages/${{ matrix.profile }} path: ./_packages/${{ matrix.profile }}
- name: install dos2unix - name: install dos2unix
run: sudo apt-get update && sudo apt install -y dos2unix run: sudo apt-get update && sudo apt install -y dos2unix
@ -460,7 +473,7 @@ jobs:
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
${{ secrets.EMQX_IO_RELEASE_API }} ${{ secrets.EMQX_IO_RELEASE_API }}
- name: update repo.emqx.io - name: update repo.emqx.io
if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
run: | run: |
curl --silent --show-error \ curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
@ -469,7 +482,7 @@ jobs:
-d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
- name: update repo.emqx.io - name: update repo.emqx.io
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' if: github.event_name == 'release' && matrix.profile == 'emqx'
run: | run: |
curl --silent --show-error \ curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \

21
.github/workflows/build_slim_packages.yaml vendored
View File

@ -1,5 +1,10 @@
name: Build slim packages name: Build slim packages
concurrency:
group: slim-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on: on:
push: push:
tags: tags:
@ -13,6 +18,7 @@ jobs:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
strategy: strategy:
fail-fast: false
matrix: matrix:
erl_otp: erl_otp:
- 23.3.4.9-3 - 23.3.4.9-3
@ -43,7 +49,7 @@ jobs:
with: with:
name: rebar3.crashdump name: rebar3.crashdump
path: ./rebar3.crashdump path: ./rebar3.crashdump
- name: pakcages test - name: packages test
run: | run: |
export CODE_PATH=$GITHUB_WORKSPACE export CODE_PATH=$GITHUB_WORKSPACE
.ci/build_packages/tests.sh .ci/build_packages/tests.sh
@ -53,12 +59,17 @@ jobs:
path: _packages/**/*.zip path: _packages/**/*.zip
mac: mac:
runs-on: macos-10.15
strategy: strategy:
fail-fast: false
matrix: matrix:
erl_otp: erl_otp:
- 23.3.4.9-3 - 23.3.4.9-3
macos:
- macos-11
- macos-10.15
runs-on: ${{ matrix.macos }}
steps: steps:
- uses: actions/checkout@v1 - uses: actions/checkout@v1
@ -82,7 +93,7 @@ jobs:
id: cache id: cache
with: with:
path: ~/.kerl path: ~/.kerl
key: erl${{ matrix.erl_otp }}-macos10.15 key: otp-${{ matrix.erl_otp }}-${{ matrix.macos }}
- name: build erlang - name: build erlang
if: steps.cache.outputs.cache-hit != 'true' if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60 timeout-minutes: 60
@ -106,8 +117,8 @@ jobs:
path: ./rebar3.crashdump path: ./rebar3.crashdump
- name: test - name: test
run: | run: |
pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) pkg_name=$(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip)
unzip -q _packages/${EMQX_NAME}/$pkg_name unzip -q $pkg_name
gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no' ready='no'