yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore(CI): easier build packages 

Signed-off-by: zhanghongtong <rory-z@outlook.com>
This commit is contained in:
zhanghongtong 2021-08-04 09:45:36 +08:00 committed by Rory Z
parent e6573fe32b
commit cf20fc6db7
2 changed files with 35 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.ci/build_packages/Dockerfile
View File

@ -1,16 +0,0 @@
ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
FROM ${BUILD_FROM}
ARG EMQX_NAME=emqx
COPY . /emqx
WORKDIR /emqx
RUN rm -rf _build/${EMQX_NAME}/lib _build/${EMQX_NAME}-pkg/lib
RUN make ${EMQX_NAME}-zip || cat rebar3.crashdump
RUN make ${EMQX_NAME}-pkg || cat rebar3.crashdump
RUN /emqx/.ci/build_packages/tests.sh

83
.github/workflows/build_packages.yaml vendored
View File

@ -25,8 +25,8 @@ jobs:
- name: set profile - name: set profile
id: set_profile id: set_profile
shell: bash shell: bash
working-directory: source
run: | run: |
cd source
vsn="$(./pkg-vsn.sh)" vsn="$(./pkg-vsn.sh)"
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
if make emqx-ee --dry-run > /dev/null 2>&1; then if make emqx-ee --dry-run > /dev/null 2>&1; then
@ -38,20 +38,18 @@ jobs:
echo "::set-output name=old_vsns::$old_vsns" echo "::set-output name=old_vsns::$old_vsns"
echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]" echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
fi fi
- name: get_all_deps - name: set get token
if: endsWith(github.repository, 'emqx')
run: |
make -C source deps-all
rm source/rebar.lock
zip -ryq source.zip source/* source/.[^.]*
- name: get_all_deps
if: endsWith(github.repository, 'enterprise') if: endsWith(github.repository, 'enterprise')
run: | run: |
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token - name: get deps
make -C source deps-all working-directory: source
zip -ryq source.zip source/* source/.[^.]* run: |
make ensure-rebar3
./rebar3 as default get-deps
- name: gen zip file
run: zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v2 - uses: actions/upload-artifact@v2
with: with:
name: source name: source
@ -175,7 +173,6 @@ jobs:
cd source cd source
make ensure-rebar3 make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3 sudo cp rebar3 /usr/local/bin/rebar3
rm -rf _build/${{ matrix.profile }}/lib
make ${{ matrix.profile }}-zip make ${{ matrix.profile }}-zip
- name: test - name: test
run: | run: |
@ -216,8 +213,6 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
profile: ${{fromJSON(needs.prepare.outputs.profiles)}} profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
erl_otp:
- 23.2.7.2-emqx-2
arch: arch:
- amd64 - amd64
- arm64 - arm64
@ -233,6 +228,8 @@ jobs:
- centos6 - centos6
- raspbian10 - raspbian10
# - raspbian9 # - raspbian9
erl_otp:
- 23.2.7.2-emqx-2
exclude: exclude:
- os: centos6 - os: centos6
arch: arm64 arch: arm64
@ -254,15 +251,11 @@ jobs:
shell: bash shell: bash
steps: steps:
- name: prepare docker - uses: docker/setup-buildx-action@v1
run: | - uses: docker/setup-qemu-action@v1
mkdir -p $HOME/.docker with:
echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json image: tonistiigi/binfmt:latest
echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json platforms: all
sudo systemctl restart docker
docker info
docker buildx create --use --name mybuild
docker run --rm --privileged tonistiigi/binfmt --install all
- uses: actions/download-artifact@v2 - uses: actions/download-artifact@v2
with: with:
name: source name: source
@ -275,6 +268,7 @@ jobs:
ARCH: ${{ matrix.arch }} ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }} SYSTEM: ${{ matrix.os }}
OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }} OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
working-directory: source
run: | run: |
set -e -x -u set -e -x -u
broker=$PROFILE broker=$PROFILE
@ -285,8 +279,8 @@ jobs:
export ARCH="arm" export ARCH="arm"
fi fi
mkdir -p source/_upgrade_base mkdir -p _upgrade_base
cd source/_upgrade_base cd _upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for tag in ${old_vsns[@]}; do for tag in ${old_vsns[@]}; do
if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
@ -301,32 +295,25 @@ jobs:
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }} ARCH: ${{ matrix.arch }}
SYSTEM: ${{ matrix.os }} SYSTEM: ${{ matrix.os }}
working-directory: source
run: | run: |
set -e -u docker run -i --rm \
cd source -v $(pwd):/emqx \
docker buildx build --no-cache \ --workdir /emqx \
--platform=linux/$ARCH \ --platform linux/$ARCH \
-t cross_build_emqx_for_$SYSTEM \ emqx/build-env:$ERL_OTP-$SYSTEM \
-f .ci/build_packages/Dockerfile \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
--build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \ make $PROFILE-pkg || cat rebar3.crashdump; \
--build-arg EMQX_NAME=$PROFILE \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
--output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar .
mkdir -p /tmp/packages/$PROFILE
tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/*
mv emqx/_packages/$PROFILE/* /tmp/packages/$PROFILE/
rm -rf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar
docker rm -f $(docker ps -a -q)
docker volume prune -f
- name: create sha256 - name: create sha256
env: env:
PROFILE: ${{ matrix.profile}} PROFILE: ${{ matrix.profile}}
working-directory: source
run: | run: |
if [ -d /tmp/packages/$PROFILE ]; then if [ -d _packages/$PROFILE ]; then
cd /tmp/packages/$PROFILE cd _packages/$PROFILE
for var in $(ls emqx-* ); do for var in $(ls emqx-* ); do
bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256" sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
done done
cd - cd -
fi fi
@ -334,7 +321,7 @@ jobs:
if: startsWith(github.ref, 'refs/tags/') if: startsWith(github.ref, 'refs/tags/')
with: with:
name: ${{ matrix.profile }} name: ${{ matrix.profile }}
path: /tmp/packages/${{ matrix.profile }}/. path: source/_packages/${{ matrix.profile }}/.
docker: docker:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
@ -364,8 +351,8 @@ jobs:
env: env:
ERL_OTP: erl${{ matrix.erl_otp }} ERL_OTP: erl${{ matrix.erl_otp }}
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
working-directory: source
run: | run: |
cd source
PKG_VSN="$(./pkg-vsn.sh)" PKG_VSN="$(./pkg-vsn.sh)"
docker buildx build --no-cache \ docker buildx build --no-cache \
--platform=linux/amd64,linux/arm64 \ --platform=linux/amd64,linux/arm64 \
@ -385,8 +372,8 @@ jobs:
env: env:
ERL_OTP: erl${{ matrix.erl_otp }} ERL_OTP: erl${{ matrix.erl_otp }}
PROFILE: ${{ matrix.profile }} PROFILE: ${{ matrix.profile }}
working-directory: source
run: | run: |
cd source
PKG_VSN="$(./pkg-vsn.sh)" PKG_VSN="$(./pkg-vsn.sh)"
docker buildx build --no-cache \ docker buildx build --no-cache \
--platform=linux/amd64,linux/arm64 \ --platform=linux/amd64,linux/arm64 \