From 41f2b77ec3d99075fa67500340feda51f3255cff Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Mon, 23 Aug 2021 09:41:37 +0800 Subject: [PATCH] chore(CI): update workflows Signed-off-by: zhanghongtong --- .github/workflows/run_fvt_tests.yaml | 456 ++++++++++--------------- .github/workflows/run_relup_tests.yaml | 130 +++++++ build | 2 +- 3 files changed, 302 insertions(+), 286 deletions(-) create mode 100644 .github/workflows/run_relup_tests.yaml diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 272d69ca7..a4b9df5c2 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -8,300 +8,186 @@ on: pull_request: jobs: - docker_test: - runs-on: ubuntu-20.04 + prepare: + strategy: + matrix: + container: + - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" + - "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04" - steps: - - uses: actions/checkout@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang - with: - otp-version: 24.0.5 - - name: prepare - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token - make deps-emqx-ee - echo "PROFILE=emqx-ee" >> $GITHUB_ENV - echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV - echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV - else - echo "PROFILE=emqx" >> $GITHUB_ENV - echo "TARGET=emqx/emqx" >> $GITHUB_ENV - echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV - fi - - name: make emqx image - run: make $PROFILE-docker - - name: run emqx - timeout-minutes: 5 - run: | - set -e -u -x - echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env - echo "EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env - echo "EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env - docker-compose \ - -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ - -f .ci/docker-compose-file/docker-compose-python.yaml \ - up -d - while ! docker exec -i node1.emqx.io bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1; do - echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; - sleep 5; - done - # - name: verify EMQX_LOADED_PLUGINS override working - # run: | - # expected="{emqx_sn, true}." - # output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1) - # if [ "$expected" != "$output" ]; then - # exit 1 - # fi - - name: make paho tests - run: | - if ! docker exec -i python /scripts/pytest.sh; then - echo "DUMP_CONTAINER_LOGS_BGN" - docker logs haproxy - docker logs node1.emqx.io - docker logs node2.emqx.io - echo "DUMP_CONTAINER_LOGS_END" - exit 1 - fi + runs-on: ubuntu-20.04 + container: ${{ matrix.container }} - helm_test: - runs-on: ubuntu-20.04 + outputs: + profile: ${{ steps.profile.outputs.profile }} - steps: - - uses: actions/checkout@v1 - - uses: gleam-lang/setup-erlang@v1.1.2 - id: install_erlang - with: - otp-version: 24.0.5 - - name: prepare - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token - make deps-emqx-ee - echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV - echo "PROFILE=emqx-ee" >> $GITHUB_ENV - else - echo "TARGET=emqx/emqx" >> $GITHUB_ENV - echo "PROFILE=emqx" >> $GITHUB_ENV - fi - - name: make emqx image - run: make $PROFILE-docker - - name: install k3s - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - run: | - sudo sh -c "echo \"127.0.0.1 $(hostname)\" >> /etc/hosts" - curl -sfL https://get.k3s.io | sh - - sudo chmod 644 /etc/rancher/k3s/k3s.yaml - kubectl cluster-info - - name: install helm - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - run: | - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - sudo chmod 700 get_helm.sh - sudo ./get_helm.sh - helm version - - name: run emqx on chart - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - timeout-minutes: 5 - run: | - version=$(./pkg-vsn.sh) - sudo docker save ${TARGET}:$version -o emqx.tar.gz - sudo k3s ctr image import emqx.tar.gz + steps: + - name: get otp version + id: get_otp_version + run: | + otp="$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)" + echo "::set-output name=otp::$otp" + - uses: actions/checkout@v2 + with: + path: source + fetch-depth: 0 + - name: set profile + id: profile + shell: bash + working-directory: source + run: | + vsn="$(./pkg-vsn.sh)" + if make emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + echo "::set-output name=profile::emqx-ee" + else + echo "::set-output name=profile::emqx" + fi + - name: get deps + working-directory: source + run: | + make ensure-rebar3 + ./rebar3 as default get-deps + rm -rf rebar.lock + - name: gen zip file + run: zip -ryq source-${{ steps.get_otp_version.outputs.otp }}.zip source/* source/.[^.]* + - uses: actions/upload-artifact@v2 + with: + name: source-${{ steps.get_otp_version.outputs.otp }} + path: source-${{ steps.get_otp_version.outputs.otp }}.zip - sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml - sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml + docker_test: + runs-on: ubuntu-20.04 + needs: prepare - helm install emqx \ - --set image.repository=${TARGET} \ - --set image.pullPolicy=Never \ - --set emqxAclConfig="" \ - --set image.pullPolicy=Never \ - --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \ - --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ - deploy/charts/emqx \ - --debug + strategy: + fail-fast: false + matrix: + otp: + - 23.2.7.2-emqx-2 + - 24.0.5-emqx-1 - while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \ - != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do - echo "=============================="; - kubectl get pods; - echo "=============================="; - echo "waiting emqx started"; - sleep 10; - done - - name: get emqx-0 pods log - if: failure() - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - run: | - kubectl describe pods emqx-0 - kubectl logs emqx-0 - - name: get emqx-1 pods log - if: failure() - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - run: | - kubectl describe pods emqx-1 - kubectl logs emqx-1 - - name: get emqx-2 pods log - if: failure() - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - run: | - kubectl describe pods emqx-2 - kubectl logs emqx-2 - - uses: actions/checkout@v2 - with: - repository: emqx/paho.mqtt.testing - ref: develop-4.0 - path: paho.mqtt.testing - - name: install pytest - run: | - pip install pytest - echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: run paho test - env: - KUBECONFIG: "/etc/rancher/k3s/k3s.yaml" - run: | - emqx_svc=$(kubectl get svc --namespace default emqx -o jsonpath="{.spec.clusterIP}") - emqx1=$(kubectl get pods emqx-1 -o jsonpath='{.status.podIP}') - emqx2=$(kubectl get pods emqx-2 -o jsonpath='{.status.podIP}') + steps: + - uses: actions/download-artifact@v2 + with: + name: source-${{ matrix.otp }} + path: . + - name: unzip source code + run: unzip -q source-${{ matrix.otp }}.zip + - name: make docker image + working-directory: source + env: + OTP: ${{ matrix.otp }} + run: | + make ${{ needs.prepare.outputs.profile }}-docker + echo "TARGET=emqx/${{ needs.prepare.outputs.profile }}" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV + - name: run emqx + timeout-minutes: 5 + working-directory: source + run: | + set -e -u -x + echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env + echo "EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env + echo "EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env + docker-compose \ + -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ + -f .ci/docker-compose-file/docker-compose-python.yaml \ + up -d + while ! docker exec -i node1.emqx.io bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1; do + echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx"; + sleep 5; + done + - name: make paho tests + run: | + if ! docker exec -i python /scripts/pytest.sh; then + echo "DUMP_CONTAINER_LOGS_BGN" + docker logs haproxy + docker logs node1.emqx.io + docker logs node2.emqx.io + echo "DUMP_CONTAINER_LOGS_END" + exit 1 + fi - pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host $emqx_svc - RESULT=$? - pytest -v paho.mqtt.testing/interoperability/test_cluster --host1 $emqx1 --host2 $emqx2 - RESULT=$((RESULT + $?)) - if [ 0 -ne $RESULT ]; then - kubectl logs emqx-1 - kubectl logs emqx-2 - fi - exit $RESULT + helm_test: + runs-on: ubuntu-20.04 + needs: prepare - relup_test: - strategy: - matrix: - container: - - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" - - "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04" + strategy: + fail-fast: false + matrix: + otp: + - 23.2.7.2-emqx-2 + - 24.0.5-emqx-1 - runs-on: ubuntu-20.04 - container: ${{ matrix.container }} + steps: + - uses: actions/download-artifact@v2 + with: + name: source-${{ matrix.otp }} + path: . + - name: unzip source code + run: unzip -q source-${{ matrix.otp }}.zip + - name: make docker image + working-directory: source + env: + OTP: ${{ matrix.otp }} + run: | + make ${{ needs.prepare.outputs.profile }}-docker + echo "TARGET=emqx/${{ needs.prepare.outputs.profile }}" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV + - run: minikube start + - name: run emqx on chart + timeout-minutes: 5 + working-directory: source + run: | + minikube image load $TARGET:$EMQX_TAG - defaults: - run: - shell: bash - steps: - - uses: actions/setup-python@v2 - with: - python-version: '3.8' - architecture: 'x64' - - uses: actions/checkout@v2 - with: - repository: emqx/paho.mqtt.testing - ref: develop-4.0 - path: paho.mqtt.testing - - uses: actions/checkout@v2 - with: - repository: terry-xiaoyu/one_more_emqx - ref: master - path: one_more_emqx - - uses: actions/checkout@v2 - with: - repository: emqx/emqtt-bench - ref: master - path: emqtt-bench - - uses: actions/checkout@v2 - with: - repository: hawk/lux - ref: lux-2.6 - path: lux - - uses: actions/checkout@v2 - with: - repository: ${{ github.repository }} - path: emqx - fetch-depth: 0 - - name: prepare - run: | - if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token - echo "PROFILE=emqx-ee" >> $GITHUB_ENV - else - echo "PROFILE=emqx" >> $GITHUB_ENV - fi - - name: get version - run: | - set -e -x -u - cd emqx - if [ $PROFILE = "emqx" ];then - broker="emqx-ce" - edition='opensource' - else - broker="emqx-ee" - edition='enterprise' - fi - echo "BROKER=$broker" >> $GITHUB_ENV + sed -i -r "s/^appVersion: .*$/appVersion: \"$EMQX_TAG\"/g" deploy/charts/emqx/Chart.yaml - vsn="$(./pkg-vsn.sh)" - echo "VSN=$vsn" >> $GITHUB_ENV + helm install emqx \ + --set image.repository=$TARGET \ + --set image.pullPolicy=Never \ + --set emqxAclConfig="" \ + --set image.pullPolicy=Never \ + --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \ + --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \ + deploy/charts/emqx \ + --debug - pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" - if [ $PROFILE = "emqx" ]; then - old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" - else - old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")" - fi - echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV - - name: download emqx - run: | - set -e -x -u - mkdir -p emqx/_upgrade_base - cd emqx/_upgrade_base - old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) - for old_vsn in ${old_vsns[@]}; do - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip - done - - name: build emqx - run: make -C emqx ${PROFILE}-zip - - name: build emqtt-bench - run: make -C emqtt-bench - - name: build lux - run: | - set -e -u -x - cd lux - autoconf - ./configure - make - make install - - name: run relup test - timeout-minutes: 20 - run: | - set -e -x -u - if [ -n "$OLD_VSNS" ]; then - mkdir -p packages - cp emqx/_packages/${PROFILE}/*.zip packages - cp emqx/_upgrade_base/*.zip packages - lux \ - --case_timeout infinity \ - --var PROFILE=$PROFILE \ - --var PACKAGE_PATH=$(pwd)/packages \ - --var BENCH_PATH=$(pwd)/emqtt-bench \ - --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ - --var VSN="$VSN" \ - --var OLD_VSNS="$OLD_VSNS" \ - emqx/.ci/fvt_tests/relup.lux - fi - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: lux_logs - path: lux_logs + while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \ + != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do + echo "=============================="; + kubectl get pods; + echo "=============================="; + echo "waiting emqx started"; + sleep 10; + done + - name: get emqx-0 pods log + if: failure() + run: | + kubectl describe pods emqx-0 + kubectl logs emqx-0 + - name: get emqx-1 pods log + if: failure() + run: | + kubectl describe pods emqx-1 + kubectl logs emqx-1 + - name: get emqx-2 pods log + if: failure() + run: | + kubectl describe pods emqx-2 + kubectl logs emqx-2 + - uses: actions/checkout@v2 + with: + repository: emqx/paho.mqtt.testing + ref: develop-4.0 + path: paho.mqtt.testing + - name: install pytest + run: | + pip install pytest + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: run paho test + run: | + kubectl port-forward service/emqx 1883:1883 > /dev/null & + pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1" diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml new file mode 100644 index 000000000..312ef1152 --- /dev/null +++ b/.github/workflows/run_relup_tests.yaml @@ -0,0 +1,130 @@ +name: Release Upgrade Tests + +on: + push: + tags: + - v* + - e* + pull_request: + +jobs: + relup_test: + strategy: + matrix: + container: + - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04" + - "emqx/build-env:erl24.0.5-emqx-1-ubuntu20.04" + + runs-on: ubuntu-20.04 + container: ${{ matrix.container }} + + defaults: + run: + shell: bash + steps: + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + - uses: actions/checkout@v2 + with: + repository: emqx/paho.mqtt.testing + ref: develop-4.0 + path: paho.mqtt.testing + - uses: actions/checkout@v2 + with: + repository: terry-xiaoyu/one_more_emqx + ref: master + path: one_more_emqx + - uses: actions/checkout@v2 + with: + repository: emqx/emqtt-bench + ref: master + path: emqtt-bench + - uses: actions/checkout@v2 + with: + repository: hawk/lux + ref: lux-2.6 + path: lux + - uses: actions/checkout@v2 + with: + repository: ${{ github.repository }} + path: emqx + fetch-depth: 0 + - name: prepare + run: | + if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store + echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token + echo "PROFILE=emqx-ee" >> $GITHUB_ENV + else + echo "PROFILE=emqx" >> $GITHUB_ENV + fi + - name: get version + run: | + set -e -x -u + cd emqx + if [ $PROFILE = "emqx" ];then + broker="emqx-ce" + edition='opensource' + else + broker="emqx-ee" + edition='enterprise' + fi + echo "BROKER=$broker" >> $GITHUB_ENV + + vsn="$(./pkg-vsn.sh)" + echo "VSN=$vsn" >> $GITHUB_ENV + + pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" + if [ $PROFILE = "emqx" ]; then + old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" + else + old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")" + fi + echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV + - name: download emqx + run: | + set -e -x -u + mkdir -p emqx/_upgrade_base + cd emqx/_upgrade_base + old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) + for old_vsn in ${old_vsns[@]}; do + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip + done + - name: build emqx + run: make -C emqx ${PROFILE}-zip + - name: build emqtt-bench + run: make -C emqtt-bench + - name: build lux + run: | + set -e -u -x + cd lux + autoconf + ./configure + make + make install + - name: run relup test + timeout-minutes: 20 + run: | + set -e -x -u + if [ -n "$OLD_VSNS" ]; then + mkdir -p packages + cp emqx/_packages/${PROFILE}/*.zip packages + cp emqx/_upgrade_base/*.zip packages + lux \ + --case_timeout infinity \ + --var PROFILE=$PROFILE \ + --var PACKAGE_PATH=$(pwd)/packages \ + --var BENCH_PATH=$(pwd)/emqtt-bench \ + --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \ + --var VSN="$VSN" \ + --var OLD_VSNS="$OLD_VSNS" \ + emqx/.ci/fvt_tests/relup.lux + fi + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: lux_logs + path: lux_logs diff --git a/build b/build index 2b030a1de..727c3a1a2 100755 --- a/build +++ b/build @@ -128,7 +128,7 @@ make_docker() { docker build --no-cache \ --build-arg PKG_VSN="$PKG_VSN" \ - --build-arg BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-alpine \ + --build-arg BUILD_FROM="emqx/build-env:erl${OTP:-23.2.7.2-emqx-2}-alpine" \ --build-arg EMQX_NAME="$PROFILE" \ --tag "emqx/$PROFILE:$PKG_VSN" \ -f deploy/docker/Dockerfile .