emqx/.github/workflows/build_packages.yaml

397 lines
13 KiB
YAML

name: Cross build packages
concurrency:
group: build-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
on:
push:
branches:
- 'main-v4.**'
tags:
- v*
- e*
schedule:
- cron: '0 */6 * * *'
workflow_dispatch:
jobs:
prepare:
# avoid building when syncing to ee repo
if: endsWith(github.repository, 'emqx')
runs-on: ubuntu-20.04
container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
outputs:
profiles: ${{ steps.detect-profiles.outputs.profiles}}
steps:
- uses: actions/checkout@v3
with:
path: source
fetch-depth: 0 # clone full git history
- name: detect-profiles
id: detect-profiles
uses: ./source/.github/actions/detect-profiles
- name: get_all_deps
if: endsWith(github.repository, 'emqx')
run: |
make -C source deps-all
zip -ryq source.zip source/* source/.[^.]*
- name: get_all_deps
if: endsWith(github.repository, 'enterprise')
run: |
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
make -C source deps-all
zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3
with:
name: source
path: source.zip
windows:
runs-on: windows-2019
needs: prepare
if: endsWith(github.repository, 'emqx')
strategy:
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
otp:
- 23.3.4.17
exclude:
- profile: emqx-edge
steps:
- uses: actions/download-artifact@v3
with:
name: source
path: .
- name: unzip source code
run: Expand-Archive -Path source.zip -DestinationPath ./
- uses: ilammy/msvc-dev-cmd@v1.12.0
- uses: erlef/setup-beam@v1
with:
otp-version: ${{ matrix.otp }}
- name: build
env:
PYTHON: python
DIAGNOSTIC: 1
run: |
erl -eval "erlang:display(crypto:info_lib())" -s init stop
$version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
$regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
$pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip"
}
else {
$pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip"
}
cd source
## We do not build/release bcrypt for windows package
Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
if (Test-Path rebar.lock) {
Remove-Item -Force -Path rebar.lock
}
make ensure-rebar3
make ${{ matrix.profile }}
mkdir -p _packages/${{ matrix.profile }}
Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name
mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }}
sha256sum "_packages/${{ matrix.profile }}/$pkg_name" | head -c 64 > "_packages/${{ matrix.profile }}/${pkg_name}.sha256"
- name: run emqx
timeout-minutes: 1
run: |
cd source
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
Start-Sleep -s 5
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.profile }}
path: source/_packages/${{ matrix.profile }}/
mac:
needs: prepare
strategy:
matrix:
otp:
- 23.3.4.18-1
os:
- macos-11
runs-on: ${{ matrix.os }}
steps:
- uses: actions/download-artifact@v3
with:
name: source
path: .
- name: unzip source code
run: |
ln -s . source
unzip -q source.zip
rm source source.zip
- uses: ./.github/actions/detect-profiles
- uses: ./.github/actions/package-macos
with:
otp: ${{ matrix.otp }}
os: ${{ matrix.os }}
apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- uses: actions/upload-artifact@v3
with:
name: ${{ env.EMQX_NAME }}
path: _packages/${{ env.EMQX_NAME }}/
linux:
runs-on: ubuntu-20.04
needs: prepare
strategy:
fail-fast: false
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
arch:
- amd64
- arm64
os:
- ubuntu20.04
- ubuntu18.04
- ubuntu16.04
- debian10
- debian9
# - opensuse
- centos8
- centos7
- centos6
# - raspbian10 #armv6l is too slow to emulate
# - raspbian9
otp_version:
#- 23.2.7.2-emqx-3
- 23.3.4.18-1
exclude:
- os: centos6
arch: arm64
- os: raspbian9
arch: amd64
- os: raspbian10
arch: amd64
- os: raspbian9
profile: emqx
- os: raspbian10
profile: emqx
- os: raspbian9
profile: emqx-ee
- os: raspbian10
profile: emqx-ee
defaults:
run:
shell: bash
steps:
- name: prepare docker
run: |
mkdir -p $HOME/.docker
echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json
sudo systemctl restart docker
docker info
docker buildx create --use --name mybuild
docker run --rm --privileged tonistiigi/binfmt --install all
- uses: actions/download-artifact@v3
with:
name: source
path: .
- name: unzip source code
run: unzip -q source.zip
- name: build emqx packages
env:
ERL_OTP: erl${{ matrix.otp_version }}
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }}
run: |
set -e -u
cd source
docker buildx build --no-cache \
--platform=linux/$ARCH \
-t cross_build_emqx_for_$SYSTEM \
-f .ci/build_packages/Dockerfile \
--build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \
--build-arg EMQX_NAME=$PROFILE \
--output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar .
mkdir -p /tmp/packages/$PROFILE
tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/*
mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/
rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar
docker rm -f $(docker ps -a -q)
docker volume prune -f
- name: create sha256
env:
PROFILE: ${{ matrix.profile }}
ERL_OTP: erl${{ matrix.otp_version }}
ARCH: ${{ matrix.arch }}
run: |
if [ -d /tmp/packages/$PROFILE ]; then
cd /tmp/packages/$PROFILE
for var in $(ls emqx-* ); do
if [[ $ERL_OTP == erl23.2* ]]; then
# Keep package with new OTP as default
# But move package with old otp to track 2
echo "rename track 2 package"
oldfile="$var"
var="${var/${ARCH}/otp23.2-${ARCH}}"
mv "$oldfile" "$var"
fi
bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
done
cd -
fi
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.profile }}
path: /tmp/packages/${{ matrix.profile }}/
docker:
runs-on: ubuntu-20.04
needs: prepare
strategy:
fail-fast: false
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
registry:
- 'docker.io'
- 'public.ecr.aws'
exclude:
# we don't have an aws ecr repo for enterprise and edge yet
- profile: emqx-edge
registry: 'public.ecr.aws'
- profile: emqx-ee
registry: 'public.ecr.aws'
steps:
- uses: actions/download-artifact@v3
with:
name: source
path: .
- name: unzip source code
run: unzip -q source.zip
- uses: docker/setup-buildx-action@v2
- uses: docker/setup-qemu-action@v2
with:
image: tonistiigi/binfmt:latest
platforms: all
- uses: aws-actions/configure-aws-credentials@v1-node16
if: matrix.registry == 'public.ecr.aws'
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
- name: Docker login to aws ecr
if: matrix.registry == 'public.ecr.aws'
run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
- uses: docker/login-action@v2
if: matrix.registry == 'docker.io'
with:
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- uses: docker/metadata-action@v4
id: meta
with:
images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }}
## only 5.0 is latest
flavor: |
latest=false
tags: |
type=ref,event=branch
type=ref,event=pr
type=match,pattern=[v|e](.*),group=1
labels:
org.opencontainers.image.otp.version=${{ matrix.otp }}
- uses: docker/build-push-action@v3
if: matrix.profile != 'emqx-ee'
with:
## only push when stable tag and rc tag
push: ${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }}
pull: true
no-cache: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_FROM=emqx/build-env:erl23.3.4.18-1-alpine
RUN_FROM=alpine:3.12
EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile
context: source
- uses: docker/build-push-action@v3
if: matrix.profile == 'emqx-ee'
with:
## only push when stable tag and rc tag
push: ${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }}
pull: true
no-cache: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_FROM=emqx/build-env:erl23.3.4.18-1-alpine
RUN_FROM=alpine:3.12
EMQX_NAME=${{ matrix.profile }}
file: source/deploy/docker/Dockerfile.enterprise
context: source
publish_artifacts:
runs-on: ubuntu-20.04
if: startsWith(github.ref, 'refs/tags/')
needs: [prepare, mac, linux, docker]
strategy:
fail-fast: false
matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
steps:
- uses: actions/download-artifact@v3
with:
name: ${{ matrix.profile }}
path: packages/${{ matrix.profile }}
- name: install dos2unix
run: sudo apt-get update && sudo apt install -y dos2unix
- name: get packages
run: |
set -e -u
cd packages/${{ matrix.profile }}
for var in $( ls |grep emqx |grep -v sha256); do
dos2unix $var.sha256
echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
done
cd -
- uses: aws-actions/configure-aws-credentials@v1-node16
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
- name: upload to aws s3
run: |
set -e -u
if [ "${{ matrix.profile }}" == "emqx" ];then
s3dir="emqx-ce"
else
s3dir=${{ matrix.profile }}
fi
aws s3 cp --recursive packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/${s3dir}/${{ github.ref_name }}
aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/${s3dir}/${{ github.ref_name }}/*"