yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore(CI): update workflows 

Signed-off-by: zhanghongtong <rory-z@outlook.com>
This commit is contained in:
zhanghongtong 2021-08-23 09:41:37 +08:00 committed by Rory Z
parent 335b8fec63
commit 41f2b77ec3
3 changed files with 302 additions and 286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

300
.github/workflows/run_fvt_tests.yaml vendored
View File

@ -8,34 +8,84 @@ on:
pull_request:
jobs:
docker_test:
prepare:
strategy:
matrix:
container:
- "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04"
- "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04"
runs-on: ubuntu-20.04
container: ${{ matrix.container }}
outputs:
profile: ${{ steps.profile.outputs.profile }}
steps:
- uses: actions/checkout@v1
- uses: gleam-lang/setup-erlang@v1.1.2
id: install_erlang
with:
otp-version: 24.0.5
- name: prepare
- name: get otp version
id: get_otp_version
run: |
otp="$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)"
echo "::set-output name=otp::$otp"
- uses: actions/checkout@v2
with:
path: source
fetch-depth: 0
- name: set profile
id: profile
shell: bash
working-directory: source
run: |
vsn="$(./pkg-vsn.sh)"
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
make deps-emqx-ee
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
echo "::set-output name=profile::emqx-ee"
else
echo "PROFILE=emqx" >> $GITHUB_ENV
echo "TARGET=emqx/emqx" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
echo "::set-output name=profile::emqx"
fi
- name: make emqx image
run: make $PROFILE-docker
- name: get deps
working-directory: source
run: |
make ensure-rebar3
./rebar3 as default get-deps
rm -rf rebar.lock
- name: gen zip file
run: zip -ryq source-${{ steps.get_otp_version.outputs.otp }}.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v2
with:
name: source-${{ steps.get_otp_version.outputs.otp }}
path: source-${{ steps.get_otp_version.outputs.otp }}.zip
docker_test:
runs-on: ubuntu-20.04
needs: prepare
strategy:
fail-fast: false
matrix:
otp:
- 23.2.7.2-emqx-2
- 24.0.5-emqx-1
steps:
- uses: actions/download-artifact@v2
with:
name: source-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip
- name: make docker image
working-directory: source
env:
OTP: ${{ matrix.otp }}
run: |
make ${{ needs.prepare.outputs.profile }}-docker
echo "TARGET=emqx/${{ needs.prepare.outputs.profile }}" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
- name: run emqx
timeout-minutes: 5
working-directory: source
run: |
set -e -u -x
echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env
@ -49,13 +99,6 @@ jobs:
echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx";
sleep 5;
done
# - name: verify EMQX_LOADED_PLUGINS override working
# run: |
# expected="{emqx_sn, true}."
# output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1)
# if [ "$expected" != "$output" ]; then
# exit 1
# fi
- name: make paho tests
run: |
if ! docker exec -i python /scripts/pytest.sh; then
@ -69,58 +112,41 @@ jobs:
helm_test:
runs-on: ubuntu-20.04
needs: prepare
strategy:
fail-fast: false
matrix:
otp:
- 23.2.7.2-emqx-2
- 24.0.5-emqx-1
steps:
- uses: actions/checkout@v1
- uses: gleam-lang/setup-erlang@v1.1.2
id: install_erlang
- uses: actions/download-artifact@v2
with:
otp-version: 24.0.5
- name: prepare
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
make deps-emqx-ee
echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
else
echo "TARGET=emqx/emqx" >> $GITHUB_ENV
echo "PROFILE=emqx" >> $GITHUB_ENV
fi
- name: make emqx image
run: make $PROFILE-docker
- name: install k3s
name: source-${{ matrix.otp }}
path: .
- name: unzip source code
run: unzip -q source-${{ matrix.otp }}.zip
- name: make docker image
working-directory: source
env:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
OTP: ${{ matrix.otp }}
run: |
sudo sh -c "echo \"127.0.0.1 $(hostname)\" >> /etc/hosts"
curl -sfL https://get.k3s.io | sh -
sudo chmod 644 /etc/rancher/k3s/k3s.yaml
kubectl cluster-info
- name: install helm
env:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
run: |
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
sudo chmod 700 get_helm.sh
sudo ./get_helm.sh
helm version
make ${{ needs.prepare.outputs.profile }}-docker
echo "TARGET=emqx/${{ needs.prepare.outputs.profile }}" >> $GITHUB_ENV
echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
- run: minikube start
- name: run emqx on chart
env:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
timeout-minutes: 5
working-directory: source
run: |
version=$(./pkg-vsn.sh)
sudo docker save ${TARGET}:$version -o emqx.tar.gz
sudo k3s ctr image import emqx.tar.gz
minikube image load $TARGET:$EMQX_TAG
sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml
sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml
sed -i -r "s/^appVersion: .*$/appVersion: \"$EMQX_TAG\"/g" deploy/charts/emqx/Chart.yaml
helm install emqx \
--set image.repository=${TARGET} \
--set image.repository=$TARGET \
--set image.pullPolicy=Never \
--set emqxAclConfig="" \
--set image.pullPolicy=Never \
@ -139,22 +165,16 @@ jobs:
done
- name: get emqx-0 pods log
if: failure()
env:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
run: |
kubectl describe pods emqx-0
kubectl logs emqx-0
- name: get emqx-1 pods log
if: failure()
env:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
run: |
kubectl describe pods emqx-1
kubectl logs emqx-1
- name: get emqx-2 pods log
if: failure()
env:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
run: |
kubectl describe pods emqx-2
kubectl logs emqx-2
@ -168,140 +188,6 @@ jobs:
pip install pytest
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: run paho test
env:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
run: |
emqx_svc=$(kubectl get svc --namespace default emqx -o jsonpath="{.spec.clusterIP}")
emqx1=$(kubectl get pods emqx-1 -o jsonpath='{.status.podIP}')
emqx2=$(kubectl get pods emqx-2 -o jsonpath='{.status.podIP}')
pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host $emqx_svc
RESULT=$?
pytest -v paho.mqtt.testing/interoperability/test_cluster --host1 $emqx1 --host2 $emqx2
RESULT=$((RESULT + $?))
if [ 0 -ne $RESULT ]; then
kubectl logs emqx-1
kubectl logs emqx-2
fi
exit $RESULT
relup_test:
strategy:
matrix:
container:
- "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04"
- "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04"
runs-on: ubuntu-20.04
container: ${{ matrix.container }}
defaults:
run:
shell: bash
steps:
- uses: actions/setup-python@v2
with:
python-version: '3.8'
architecture: 'x64'
- uses: actions/checkout@v2
with:
repository: emqx/paho.mqtt.testing
ref: develop-4.0
path: paho.mqtt.testing
- uses: actions/checkout@v2
with:
repository: terry-xiaoyu/one_more_emqx
ref: master
path: one_more_emqx
- uses: actions/checkout@v2
with:
repository: emqx/emqtt-bench
ref: master
path: emqtt-bench
- uses: actions/checkout@v2
with:
repository: hawk/lux
ref: lux-2.6
path: lux
- uses: actions/checkout@v2
with:
repository: ${{ github.repository }}
path: emqx
fetch-depth: 0
- name: prepare
run: |
if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
else
echo "PROFILE=emqx" >> $GITHUB_ENV
fi
- name: get version
run: |
set -e -x -u
cd emqx
if [ $PROFILE = "emqx" ];then
broker="emqx-ce"
edition='opensource'
else
broker="emqx-ee"
edition='enterprise'
fi
echo "BROKER=$broker" >> $GITHUB_ENV
vsn="$(./pkg-vsn.sh)"
echo "VSN=$vsn" >> $GITHUB_ENV
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
if [ $PROFILE = "emqx" ]; then
old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
else
old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
fi
echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
- name: download emqx
run: |
set -e -x -u
mkdir -p emqx/_upgrade_base
cd emqx/_upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for old_vsn in ${old_vsns[@]}; do
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip
done
- name: build emqx
run: make -C emqx ${PROFILE}-zip
- name: build emqtt-bench
run: make -C emqtt-bench
- name: build lux
run: |
set -e -u -x
cd lux
autoconf
./configure
make
make install
- name: run relup test
timeout-minutes: 20
run: |
set -e -x -u
if [ -n "$OLD_VSNS" ]; then
mkdir -p packages
cp emqx/_packages/${PROFILE}/*.zip packages
cp emqx/_upgrade_base/*.zip packages
lux \
--case_timeout infinity \
--var PROFILE=$PROFILE \
--var PACKAGE_PATH=$(pwd)/packages \
--var BENCH_PATH=$(pwd)/emqtt-bench \
--var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
--var VSN="$VSN" \
--var OLD_VSNS="$OLD_VSNS" \
emqx/.ci/fvt_tests/relup.lux
fi
- uses: actions/upload-artifact@v1
if: failure()
with:
name: lux_logs
path: lux_logs
kubectl port-forward service/emqx 1883:1883 > /dev/null &
pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1"

130
.github/workflows/run_relup_tests.yaml vendored Normal file
View File

@ -0,0 +1,130 @@
name: Release Upgrade Tests
on:
push:
tags:
- v*
- e*
pull_request:
jobs:
relup_test:
strategy:
matrix:
container:
- "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04"
- "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04"
runs-on: ubuntu-20.04
container: ${{ matrix.container }}
defaults:
run:
shell: bash
steps:
- uses: actions/setup-python@v2
with:
python-version: '3.8'
architecture: 'x64'
- uses: actions/checkout@v2
with:
repository: emqx/paho.mqtt.testing
ref: develop-4.0
path: paho.mqtt.testing
- uses: actions/checkout@v2
with:
repository: terry-xiaoyu/one_more_emqx
ref: master
path: one_more_emqx
- uses: actions/checkout@v2
with:
repository: emqx/emqtt-bench
ref: master
path: emqtt-bench
- uses: actions/checkout@v2
with:
repository: hawk/lux
ref: lux-2.6
path: lux
- uses: actions/checkout@v2
with:
repository: ${{ github.repository }}
path: emqx
fetch-depth: 0
- name: prepare
run: |
if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
echo "PROFILE=emqx-ee" >> $GITHUB_ENV
else
echo "PROFILE=emqx" >> $GITHUB_ENV
fi
- name: get version
run: |
set -e -x -u
cd emqx
if [ $PROFILE = "emqx" ];then
broker="emqx-ce"
edition='opensource'
else
broker="emqx-ee"
edition='enterprise'
fi
echo "BROKER=$broker" >> $GITHUB_ENV
vsn="$(./pkg-vsn.sh)"
echo "VSN=$vsn" >> $GITHUB_ENV
pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
if [ $PROFILE = "emqx" ]; then
old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
else
old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
fi
echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
- name: download emqx
run: |
set -e -x -u
mkdir -p emqx/_upgrade_base
cd emqx/_upgrade_base
old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
for old_vsn in ${old_vsns[@]}; do
wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip
done
- name: build emqx
run: make -C emqx ${PROFILE}-zip
- name: build emqtt-bench
run: make -C emqtt-bench
- name: build lux
run: |
set -e -u -x
cd lux
autoconf
./configure
make
make install
- name: run relup test
timeout-minutes: 20
run: |
set -e -x -u
if [ -n "$OLD_VSNS" ]; then
mkdir -p packages
cp emqx/_packages/${PROFILE}/*.zip packages
cp emqx/_upgrade_base/*.zip packages
lux \
--case_timeout infinity \
--var PROFILE=$PROFILE \
--var PACKAGE_PATH=$(pwd)/packages \
--var BENCH_PATH=$(pwd)/emqtt-bench \
--var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
--var VSN="$VSN" \
--var OLD_VSNS="$OLD_VSNS" \
emqx/.ci/fvt_tests/relup.lux
fi
- uses: actions/upload-artifact@v1
if: failure()
with:
name: lux_logs
path: lux_logs

2
build
View File

@ -128,7 +128,7 @@ make_docker() {
docker build --no-cache \
--build-arg PKG_VSN="$PKG_VSN" \
--build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine \
--build-arg BUILD_FROM="emqx/build-env:erl${OTP:-23.2.7.2-emqx-2}-alpine" \
--build-arg EMQX_NAME="$PROFILE" \
--tag "emqx/$PROFILE:$PKG_VSN" \
-f deploy/docker/Dockerfile .