emqx/.github/workflows/build_packages.yaml

531 lines
18 KiB
YAML

name: Cross build packages
on:
schedule:
- cron: '0 */6 * * *'
release:
types:
- published
workflow_dispatch:
jobs:
prepare:
strategy:
matrix:
otp:
- "23.2.7.2-emqx-2"
- "24.0.5-emqx-1"
runs-on: ubuntu-20.04
container: "ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-ubuntu20.04"
outputs:
profiles: ${{ steps.set_profile.outputs.profiles }}
old_vsns: ${{ steps.set_profile.outputs.old_vsns }}
steps:
- uses: actions/checkout@v2
with:
path: source
fetch-depth: 0
- name: set profile
id: set_profile
shell: bash
working-directory: source
run: |
vsn="$(./pkg-vsn.sh)"
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
if make emqx-ee --dry-run > /dev/null 2>&1; then
old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
echo "::set-output name=old_vsns::$old_vsns"
echo "::set-output name=profiles::[\"emqx-ee\"]"
else
old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
echo "::set-output name=old_vsns::$old_vsns"
echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
fi
- name: get otp version
id: get_otp_version
run: |
otp="$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)"
echo "::set-output name=otp::$otp"
- name: set get token
if: endsWith(github.repository, 'enterprise')
run: |
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
- name: get deps
working-directory: source
run: |
make ensure-rebar3
./rebar3 as default get-deps
rm -rf rebar.lock
- name: gen zip file
run: zip -ryq source-${{ steps.get_otp_version.outputs.otp }}.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v2
with:
name: source-${{ steps.get_otp_version.outputs.otp }}
path: source-${{ steps.get_otp_version.outputs.otp }}.zip
windows:
runs-on: windows-2019
needs: prepare
if: endsWith(github.repository, 'emqx')
strategy:
fail-fast: false
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
exclude:
- profile: emqx-edge
steps:
- uses: actions/download-artifact@v2
with:
name: source-23.2.7.2-emqx-2
path: .
- name: unzip source code
run: Expand-Archive -Path source-23.2.7.2-emqx-2.zip -DestinationPath ./
- uses: ilammy/msvc-dev-cmd@v1
- uses: gleam-lang/setup-erlang@v1.1.2
id: install_erlang
## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows
with:
otp-version: 23.2
- name: build
env:
PYTHON: python
DIAGNOSTIC: 1
working-directory: source
run: |
$env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH"
$version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
$regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
$pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip"
}
else {
$pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip"
}
## We do not build/release bcrypt and quic for windows package
Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
Remove-Item -Recurse -Force -Path _build/default/lib/quicer/
if (Test-Path rebar.lock) {
Remove-Item -Force -Path rebar.lock
}
make ensure-rebar3
copy rebar3 "${{ steps.install_erlang.outputs.erlpath }}\bin"
ls "${{ steps.install_erlang.outputs.erlpath }}\bin"
rebar3 --help
make ${{ matrix.profile }}
mkdir -p _packages/${{ matrix.profile }}
Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name
mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }}
Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256
- name: run emqx
timeout-minutes: 1
working-directory: source
run: |
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
Start-Sleep -s 5
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
- uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/')
with:
name: ${{ matrix.profile }}-23.2.7.2-emqx-2
path: source/_packages/${{ matrix.profile }}/.
mac:
needs: prepare
strategy:
fail-fast: false
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
macos:
- macos-11
- macos-10.15
otp:
- 24.0.5-emqx-1
exclude:
- profile: emqx-edge
runs-on: ${{ matrix.macos }}
steps:
- uses: actions/download-artifact@v2
with:
name: source-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip
- name: prepare
run: |
brew update
brew install curl zip unzip gnu-sed kerl unixodbc freetds
echo "/usr/local/bin" >> $GITHUB_PATH
git config --global credential.helper store
- uses: actions/cache@v2
id: cache
with:
path: ~/.kerl
key: otp-${{ matrix.otp }}-${{ matrix.macos }}
- name: build erlang
if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60
run: |
kerl build git https://github.com/emqx/otp.git OTP-${{ matrix.otp }} ${{ matrix.otp }}
kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
- name: build
working-directory: source
run: |
. $HOME/.kerl/${{ matrix.otp }}/activate
make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3
make ${{ matrix.profile }}-zip
- name: test
working-directory: source
run: |
pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip | head)
unzip -q $pkg_name
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no'
for i in {1..10}; do
if curl -fs 127.0.0.1:18083/api/v5/status > /dev/null; then
ready='yes'
break
fi
sleep 1
done
if [ "$ready" != "yes" ]; then
echo "Timed out waiting for emqx to be ready"
cat emqx/log/erlang.log.1
exit 1
fi
./emqx/bin/emqx_ctl status
./emqx/bin/emqx stop
rm -rf emqx
openssl dgst -sha256 $pkg_name | awk '{print $2}' > $pkg_name.sha256
- uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/')
with:
name: ${{ matrix.profile }}-${{ matrix.otp }}
path: source/_packages/${{ matrix.profile }}/.
linux:
runs-on: ubuntu-20.04
needs: prepare
strategy:
fail-fast: false
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
os:
- ubuntu20.04
- ubuntu18.04
- ubuntu16.04
- debian10
- debian9
# - opensuse
- centos8
- centos7
- centos6
- raspbian10
# - raspbian9
arch:
- amd64
- arm64
otp:
- 23.2.7.2-emqx-2
- 24.0.5-emqx-1
exclude:
- os: centos6
arch: arm64
- os: raspbian9
arch: amd64
- os: raspbian10
arch: amd64
- os: raspbian9
profile: emqx
- os: raspbian10
profile: emqx
- os: raspbian9
profile: emqx-ee
- os: raspbian10
profile: emqx-ee
defaults:
run:
shell: bash
steps:
- uses: docker/setup-buildx-action@v1
- uses: docker/setup-qemu-action@v1
with:
image: tonistiigi/binfmt:latest
platforms: all
- uses: actions/download-artifact@v2
with:
name: source-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip
- name: downloads old emqx zip packages
env:
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }}
OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
working-directory: source
run: |
set -e -x -u
broker=$PROFILE
if [ $PROFILE = "emqx" ];then
broker="emqx-ce"
fi
if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then
export ARCH="arm"
fi
mkdir -p _upgrade_base
cd _upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for tag in ${old_vsns[@]}; do
if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
fi
done
- name: build emqx packages
env:
OTP: ${{ matrix.otp }}
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }}
working-directory: source
run: |
docker run -i --rm \
-v $(pwd):/emqx \
--workdir /emqx \
--platform linux/$ARCH \
ghcr.io/emqx/emqx-builder-helper/5.0:$OTP-$SYSTEM \
bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
make $PROFILE-pkg || cat rebar3.crashdump; \
EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
- name: create sha256
env:
PROFILE: ${{ matrix.profile}}
working-directory: source
run: |
if [ -d _packages/$PROFILE ]; then
cd _packages/$PROFILE
for var in $(ls emqx-* ); do
sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
done
cd -
fi
- uses: actions/upload-artifact@v1
if: startsWith(github.ref, 'refs/tags/')
with:
name: ${{ matrix.profile }}-${{ matrix.otp }}
path: source/_packages/${{ matrix.profile }}/.
docker:
runs-on: ubuntu-20.04
needs: prepare
strategy:
fail-fast: false
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp:
- 24.0.5-emqx-1
steps:
- uses: actions/download-artifact@v2
with:
name: source-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip
- name: get version
id: version
working-directory: source
run: echo "::set-output name=version::$(./pkg-vsn.sh)"
- uses: docker/setup-buildx-action@v1
- uses: docker/setup-qemu-action@v1
with:
image: tonistiigi/binfmt:latest
platforms: all
- uses: docker/build-push-action@v2
if: github.event_name != 'release'
with:
push: false
pull: true
no-cache: true
platforms: linux/amd64,linux/arm64
tags: emqx/${{ matrix.profile }}:${{ steps.version.outputs.version }}
build-args: |
PKG_VSN=${{ steps.version.outputs.version }}
BUILD_FROM=ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-alpine3.14
RUN_FROM=alpine:3.14
EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile
context: source
- uses: docker/login-action@v1
if: github.event_name == 'release'
with:
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- uses: docker/build-push-action@v2
if: github.event_name == 'release'
with:
push: true
pull: true
no-cache: true
platforms: linux/amd64,linux/arm64
tags: emqx/${{ matrix.profile }}:${{ steps.version.outputs.version }}
build-args: |
PKG_VSN=${{ steps.version.outputs.version }}
BUILD_FROM=ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-alpine3.14
RUN_FROM=alpine:3.14
EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile
context: source
delete-artifact:
runs-on: ubuntu-20.04
strategy:
matrix:
otp:
- 23.2.7.2-emqx-2
- 24.0.5-emqx-1
needs: [prepare, mac, linux, docker]
steps:
- uses: geekyeggo/delete-artifact@v1
with:
name: source-${{ matrix.otp }}
upload:
runs-on: ubuntu-20.04
if: startsWith(github.ref, 'refs/tags/')
needs: [prepare, mac, linux, docker]
strategy:
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp:
- 24.0.5-emqx-1
steps:
- uses: actions/checkout@v2
- name: get_version
run: |
echo 'version<<EOF' >> $GITHUB_ENV
echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV
echo 'EOF' >> $GITHUB_ENV
- uses: actions/download-artifact@v2
with:
name: ${{ matrix.profile }}-${{ matrix.otp }}
path: ./_packages/${{ matrix.profile }}
- name: install dos2unix
run: sudo apt-get update && sudo apt install -y dos2unix
- name: get packages
run: |
set -e -u
cd _packages/${{ matrix.profile }}
for var in $( ls |grep emqx |grep -v sha256); do
dos2unix $var.sha256
echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
done
cd -
- name: upload aws s3
run: |
set -e -u
if [ "${{ matrix.profile }}" == "emqx" ];then
broker="emqx-ce"
else
broker=${{ matrix.profile }}
fi
aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }}
aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }}
aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*"
- uses: Rory-Z/upload-release-asset@v1
if: github.event_name == 'release' && matrix.profile != 'emqx-ee'
with:
repo: emqx
path: "_packages/${{ matrix.profile }}/emqx-*"
token: ${{ github.token }}
- uses: Rory-Z/upload-release-asset@v1
if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
with:
repo: emqx-enterprise
path: "_packages/${{ matrix.profile }}/emqx-*"
token: ${{ github.token }}
- name: update to emqx.io
if: github.event_name == 'release'
run: |
set -e -x -u
curl -w %{http_code} \
--insecure \
-H "Content-Type: application/json" \
-H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
-X POST \
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
${{ secrets.EMQX_IO_RELEASE_API }}
- name: push docker image to docker hub
if: github.event_name == 'release'
run: |
set -e -x -u
sudo make docker-prepare
cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
sudo TARGET=emqx/${{ matrix.profile }} make docker-push
sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
- name: update repo.emqx.io
if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
run: |
curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
-X POST \
-d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
- name: update repo.emqx.io
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
run: |
curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
-X POST \
-d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
- name: update homebrew packages
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
run: |
if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
curl --silent --show-error \
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
-X POST \
-d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
fi
- uses: geekyeggo/delete-artifact@v1
with:
name: ${{ matrix.profile }}