496 lines
17 KiB
YAML
496 lines
17 KiB
YAML
name: Cross build packages
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 */6 * * *'
|
|
release:
|
|
types:
|
|
- published
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
prepare:
|
|
runs-on: ubuntu-20.04
|
|
container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
|
|
|
|
outputs:
|
|
profiles: ${{ steps.set_profile.outputs.profiles}}
|
|
old_vsns: ${{ steps.set_profile.outputs.old_vsns}}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
path: source
|
|
fetch-depth: 0
|
|
- name: set profile
|
|
id: set_profile
|
|
shell: bash
|
|
run: |
|
|
cd source
|
|
vsn="$(./pkg-vsn.sh)"
|
|
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
|
|
if make emqx-ee --dry-run > /dev/null 2>&1; then
|
|
old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
|
|
echo "::set-output name=old_vsns::$old_vsns"
|
|
echo "::set-output name=profiles::[\"emqx-ee\"]"
|
|
else
|
|
old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
|
|
echo "::set-output name=old_vsns::$old_vsns"
|
|
echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
|
|
fi
|
|
- name: get_all_deps
|
|
if: endsWith(github.repository, 'emqx')
|
|
run: |
|
|
make -C source deps-all
|
|
zip -ryq source.zip source/* source/.[^.]*
|
|
- name: get_all_deps
|
|
if: endsWith(github.repository, 'enterprise')
|
|
run: |
|
|
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
|
|
git config --global credential.helper store
|
|
echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token
|
|
make -C source deps-all
|
|
zip -ryq source.zip source/* source/.[^.]*
|
|
- uses: actions/upload-artifact@v2
|
|
with:
|
|
name: source
|
|
path: source.zip
|
|
|
|
windows:
|
|
runs-on: windows-2019
|
|
|
|
needs: prepare
|
|
if: endsWith(github.repository, 'emqx')
|
|
|
|
strategy:
|
|
matrix:
|
|
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
|
|
exclude:
|
|
- profile: emqx-edge
|
|
|
|
steps:
|
|
- uses: actions/download-artifact@v2
|
|
with:
|
|
name: source
|
|
path: .
|
|
- name: unzip source code
|
|
run: Expand-Archive -Path source.zip -DestinationPath ./
|
|
- uses: ilammy/msvc-dev-cmd@v1
|
|
- uses: gleam-lang/setup-erlang@v1.1.0
|
|
id: install_erlang
|
|
with:
|
|
otp-version: 23.2
|
|
- name: build
|
|
env:
|
|
PYTHON: python
|
|
DIAGNOSTIC: 1
|
|
run: |
|
|
$env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH"
|
|
|
|
$version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
|
|
if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
|
|
$regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
|
|
$pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip"
|
|
}
|
|
else {
|
|
$pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip"
|
|
}
|
|
cd source
|
|
## We do not build/release bcrypt for windows package
|
|
Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
|
|
if (Test-Path rebar.lock) {
|
|
Remove-Item -Force -Path rebar.lock
|
|
}
|
|
make ensure-rebar3
|
|
copy rebar3 "${{ steps.install_erlang.outputs.erlpath }}\bin"
|
|
ls "${{ steps.install_erlang.outputs.erlpath }}\bin"
|
|
rebar3 --help
|
|
make ${{ matrix.profile }}
|
|
mkdir -p _packages/${{ matrix.profile }}
|
|
Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name
|
|
mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }}
|
|
Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}' > _packages/${{ matrix.profile }}/$pkg_name.sha256
|
|
- name: run emqx
|
|
timeout-minutes: 1
|
|
run: |
|
|
cd source
|
|
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
|
|
Start-Sleep -s 5
|
|
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
|
|
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
|
|
./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
|
|
- uses: actions/upload-artifact@v1
|
|
if: startsWith(github.ref, 'refs/tags/')
|
|
with:
|
|
name: ${{ matrix.profile }}
|
|
path: source/_packages/${{ matrix.profile }}/.
|
|
|
|
mac:
|
|
runs-on: macos-10.15
|
|
|
|
needs: prepare
|
|
|
|
strategy:
|
|
matrix:
|
|
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
|
|
erl_otp:
|
|
- 23.2.7.2-emqx-2
|
|
exclude:
|
|
- profile: emqx-edge
|
|
|
|
steps:
|
|
- uses: actions/download-artifact@v2
|
|
with:
|
|
name: source
|
|
path: .
|
|
- name: unzip source code
|
|
run: unzip -q source.zip
|
|
- name: prepare
|
|
run: |
|
|
brew update
|
|
brew install curl zip unzip gnu-sed kerl unixodbc freetds
|
|
echo "/usr/local/bin" >> $GITHUB_PATH
|
|
git config --global credential.helper store
|
|
- uses: actions/cache@v2
|
|
id: cache
|
|
with:
|
|
path: ~/.kerl
|
|
key: erl${{ matrix.erl_otp }}-macos10.15
|
|
- name: build erlang
|
|
if: steps.cache.outputs.cache-hit != 'true'
|
|
timeout-minutes: 60
|
|
env:
|
|
KERL_BUILD_BACKEND: git
|
|
OTP_GITHUB_URL: https://github.com/emqx/otp
|
|
run: |
|
|
kerl update releases
|
|
kerl build ${{ matrix.erl_otp }}
|
|
kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
|
|
- name: build
|
|
run: |
|
|
. $HOME/.kerl/${{ matrix.erl_otp }}/activate
|
|
cd source
|
|
make ensure-rebar3
|
|
sudo cp rebar3 /usr/local/bin/rebar3
|
|
rm -rf _build/${{ matrix.profile }}/lib
|
|
make ${{ matrix.profile }}-zip
|
|
- name: test
|
|
run: |
|
|
cd source
|
|
pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip)
|
|
unzip -q _packages/${{ matrix.profile }}/$pkg_name
|
|
gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
|
|
./emqx/bin/emqx start || cat emqx/log/erlang.log.1
|
|
ready='no'
|
|
for i in {1..10}; do
|
|
if curl -fs 127.0.0.1:18083 > /dev/null; then
|
|
ready='yes'
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
if [ "$ready" != "yes" ]; then
|
|
echo "Timed out waiting for emqx to be ready"
|
|
cat emqx/log/erlang.log.1
|
|
exit 1
|
|
fi
|
|
./emqx/bin/emqx_ctl status
|
|
./emqx/bin/emqx stop
|
|
rm -rf emqx
|
|
openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}' > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
|
|
- uses: actions/upload-artifact@v1
|
|
if: startsWith(github.ref, 'refs/tags/')
|
|
with:
|
|
name: ${{ matrix.profile }}
|
|
path: source/_packages/${{ matrix.profile }}/.
|
|
|
|
linux:
|
|
runs-on: ubuntu-20.04
|
|
|
|
needs: prepare
|
|
|
|
strategy:
|
|
matrix:
|
|
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
|
|
arch:
|
|
- amd64
|
|
- arm64
|
|
os:
|
|
- ubuntu20.04
|
|
- ubuntu18.04
|
|
- ubuntu16.04
|
|
- debian10
|
|
- debian9
|
|
# - opensuse
|
|
- centos8
|
|
- centos7
|
|
- centos6
|
|
- raspbian10
|
|
# - raspbian9
|
|
exclude:
|
|
- os: centos6
|
|
arch: arm64
|
|
- os: raspbian9
|
|
arch: amd64
|
|
- os: raspbian10
|
|
arch: amd64
|
|
- os: raspbian9
|
|
profile: emqx
|
|
- os: raspbian10
|
|
profile: emqx
|
|
- os: raspbian9
|
|
profile: emqx-ee
|
|
- os: raspbian10
|
|
profile: emqx-ee
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
steps:
|
|
- name: prepare docker
|
|
run: |
|
|
mkdir -p $HOME/.docker
|
|
echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
|
|
echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json
|
|
sudo systemctl restart docker
|
|
docker info
|
|
docker buildx create --use --name mybuild
|
|
docker run --rm --privileged tonistiigi/binfmt --install all
|
|
- uses: actions/download-artifact@v2
|
|
with:
|
|
name: source
|
|
path: .
|
|
- name: unzip source code
|
|
run: unzip -q source.zip
|
|
- name: downloads old emqx zip packages
|
|
env:
|
|
PROFILE: ${{ matrix.profile }}
|
|
ARCH: ${{ matrix.arch }}
|
|
SYSTEM: ${{ matrix.os }}
|
|
OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
|
|
run: |
|
|
set -e -x -u
|
|
broker=$PROFILE
|
|
if [ $PROFILE = "emqx" ];then
|
|
broker="emqx-ce"
|
|
fi
|
|
if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then
|
|
export ARCH="arm"
|
|
fi
|
|
|
|
mkdir -p source/_upgrade_base
|
|
cd source/_upgrade_base
|
|
old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
|
|
for tag in ${old_vsns[@]}; do
|
|
if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
|
|
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
|
|
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
|
|
echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
|
|
fi
|
|
done
|
|
- name: build emqx packages
|
|
env:
|
|
ERL_OTP: erl23.2.7.2-emqx-2
|
|
PROFILE: ${{ matrix.profile }}
|
|
ARCH: ${{ matrix.arch }}
|
|
SYSTEM: ${{ matrix.os }}
|
|
run: |
|
|
set -e -u
|
|
cd source
|
|
docker buildx build --no-cache \
|
|
--platform=linux/$ARCH \
|
|
-t cross_build_emqx_for_$SYSTEM \
|
|
-f .ci/build_packages/Dockerfile \
|
|
--build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \
|
|
--build-arg EMQX_NAME=$PROFILE \
|
|
--output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar .
|
|
|
|
mkdir -p /tmp/packages/$PROFILE
|
|
tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/*
|
|
mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/
|
|
rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar
|
|
|
|
docker rm -f $(docker ps -a -q)
|
|
docker volume prune -f
|
|
- name: create sha256
|
|
env:
|
|
PROFILE: ${{ matrix.profile}}
|
|
run: |
|
|
if [ -d /tmp/packages/$PROFILE ]; then
|
|
cd /tmp/packages/$PROFILE
|
|
for var in $(ls emqx-* ); do
|
|
bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
|
|
done
|
|
cd -
|
|
fi
|
|
- uses: actions/upload-artifact@v1
|
|
if: startsWith(github.ref, 'refs/tags/')
|
|
with:
|
|
name: ${{ matrix.profile }}
|
|
path: /tmp/packages/${{ matrix.profile }}/.
|
|
|
|
docker:
|
|
runs-on: ubuntu-20.04
|
|
|
|
needs: prepare
|
|
|
|
strategy:
|
|
matrix:
|
|
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
|
|
arch:
|
|
- [amd64, x86_64]
|
|
- [arm64v8, aarch64]
|
|
- [arm32v7, arm]
|
|
- [i386, i386]
|
|
- [s390x, s390x]
|
|
exclude:
|
|
- profile: emqx-ee
|
|
arch: [i386, i386]
|
|
- profile: emqx-ee
|
|
arch: [s390x, s390x]
|
|
|
|
steps:
|
|
- uses: actions/download-artifact@v2
|
|
with:
|
|
name: source
|
|
path: .
|
|
- name: unzip source code
|
|
run: unzip -q source.zip
|
|
- name: build emqx docker image
|
|
env:
|
|
PROFILE: ${{ matrix.profile }}
|
|
ARCH: ${{ matrix.arch[0] }}
|
|
QEMU_ARCH: ${{ matrix.arch[1] }}
|
|
run: |
|
|
sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
|
|
|
cd source
|
|
sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH make docker
|
|
cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
|
|
- uses: actions/upload-artifact@v1
|
|
if: startsWith(github.ref, 'refs/tags/')
|
|
with:
|
|
name: ${{ matrix.profile }}
|
|
path: source/_packages/${{ matrix.profile }}/.
|
|
|
|
delete-artifact:
|
|
runs-on: ubuntu-20.04
|
|
needs: [prepare, mac, linux, docker]
|
|
steps:
|
|
- uses: geekyeggo/delete-artifact@v1
|
|
with:
|
|
name: source
|
|
|
|
upload:
|
|
runs-on: ubuntu-20.04
|
|
|
|
if: startsWith(github.ref, 'refs/tags/')
|
|
|
|
needs: [prepare, mac, linux, docker]
|
|
|
|
strategy:
|
|
matrix:
|
|
profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
- name: get_version
|
|
run: |
|
|
echo 'version<<EOF' >> $GITHUB_ENV
|
|
echo ${{ github.ref }} | sed -r "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV
|
|
echo 'EOF' >> $GITHUB_ENV
|
|
- uses: actions/download-artifact@v2
|
|
with:
|
|
name: ${{ matrix.profile }}
|
|
path: ./_packages/${{ matrix.profile }}
|
|
- name: install dos2unix
|
|
run: sudo apt-get update && sudo apt install -y dos2unix
|
|
- name: get packages
|
|
run: |
|
|
set -e -u
|
|
cd _packages/${{ matrix.profile }}
|
|
for var in $( ls |grep emqx |grep -v sha256); do
|
|
dos2unix $var.sha256
|
|
echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
|
|
done
|
|
cd -
|
|
- name: upload aws s3
|
|
run: |
|
|
set -e -u
|
|
if [ "${{ matrix.profile }}" == "emqx" ];then
|
|
broker="emqx-ce"
|
|
else
|
|
broker=${{ matrix.profile }}
|
|
fi
|
|
aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }}
|
|
|
|
aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }}
|
|
aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*"
|
|
- uses: Rory-Z/upload-release-asset@v1
|
|
if: github.event_name == 'release' && matrix.profile != 'emqx-ee'
|
|
with:
|
|
repo: emqx
|
|
path: "_packages/${{ matrix.profile }}/emqx-*"
|
|
token: ${{ github.token }}
|
|
- uses: Rory-Z/upload-release-asset@v1
|
|
if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
|
|
with:
|
|
repo: emqx-enterprise
|
|
path: "_packages/${{ matrix.profile }}/emqx-*"
|
|
token: ${{ github.token }}
|
|
- name: update to emqx.io
|
|
if: github.event_name == 'release'
|
|
run: |
|
|
set -e -x -u
|
|
curl -w %{http_code} \
|
|
--insecure \
|
|
-H "Content-Type: application/json" \
|
|
-H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
|
|
-X POST \
|
|
-d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
|
|
${{ secrets.EMQX_IO_RELEASE_API }}
|
|
- name: push docker image to docker hub
|
|
if: github.event_name == 'release'
|
|
run: |
|
|
set -e -x -u
|
|
sudo make docker-prepare
|
|
cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
|
|
echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
|
|
sudo TARGET=emqx/${{ matrix.profile }} make docker-push
|
|
sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
|
|
- name: update repo.emqx.io
|
|
if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
|
|
run: |
|
|
curl --silent --show-error \
|
|
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
-X POST \
|
|
-d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \
|
|
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
|
|
- name: update repo.emqx.io
|
|
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
|
|
run: |
|
|
curl --silent --show-error \
|
|
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
-X POST \
|
|
-d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \
|
|
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
|
|
- name: update homebrew packages
|
|
if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
|
|
run: |
|
|
if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
|
|
curl --silent --show-error \
|
|
-H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
-X POST \
|
|
-d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
|
|
"https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
|
|
fi
|
|
- uses: geekyeggo/delete-artifact@v1
|
|
with:
|
|
name: ${{ matrix.profile }}
|