yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ci(test): test parallel CT runs on self-hosted runners

This commit is contained in:
Traphalet 2022-10-25 09:24:16 +03:00 committed by Zaiming (Stone) Shi
parent d1a5dcd222
commit 25b6a1c158
3 changed files with 275 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

362
.github/workflows/run_test_cases.yaml vendored
View File

@ -10,140 +10,234 @@ on:
pull_request: pull_request:
jobs: jobs:
run_proper_test: prepare:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
outputs:
ct_apps: ${{ steps.run_find_apps.outputs.ct_apps }}
steps:
- uses: actions/checkout@v3
with:
path: source
fetch-depth: 0
- name: git credentials
run: |
if make emqx-ee --dry-run > /dev/null 2>&1; then
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
git config --global credential.helper store
fi
- name: find_ct_apps
working-directory: source
id: run_find_apps
# emqx_plugin_libs doesn't have a test suite -> excluded from app list
run: |
ct_apps="$(./scripts/find-apps.sh --json | jq -c 'del (.[] | select (. == "apps/emqx_plugin_libs"))')"
echo "ct-apps: $ct_apps"
echo "ct_apps=$ct_apps" >> $GITHUB_OUTPUT
- name: get_all_deps
working-directory: source
run: |
make deps-all
./rebar3 as test compile
cd ..
zip -ryq source.zip source/* source/.[^.]*
- uses: actions/upload-artifact@v3
with:
name: source
path: source.zip
steps: eunit_and_proper:
- uses: actions/checkout@v2 needs: prepare
- name: set git credentials runs-on: ubuntu-20.04
run: | container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
if make emqx-ee --dry-run > /dev/null 2>&1; then strategy:
echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials fail-fast: false
git config --global credential.helper store matrix:
fi task:
- name: proper - eunit
run: make proper - proper
steps:
- uses: AutoModality/action-clean@v1
- uses: actions/download-artifact@v3
with:
name: source
path: .
- name: unzip source code
run: unzip -o -q source.zip
# produces eunit.coverdata and proper.coverdata
- name: eunit and proper
working-directory: source
run: make ${{ matrix.task }}
- uses: actions/upload-artifact@v3
with:
name: cover-${{ matrix.task }}
path: source/_build/test/cover
run_common_test: ct:
runs-on: ${{ matrix.runs-on }} needs: prepare
strategy: runs-on: ${{ matrix.runs-on }}
fail-fast: false strategy:
matrix: max-parallel: 12
runs-on: fail-fast: false
- aws-amd64 matrix:
- ubuntu-20.04 app_name: ${{ fromJson(needs.prepare.outputs.ct_apps) }}
use-self-hosted: runs-on:
- ${{ github.repository_owner == 'emqx' }} - aws-amd64
exclude: - ubuntu-20.04
- runs-on: ubuntu-20.04 use-self-hosted:
use-self-hosted: true - ${{ github.repository_owner == 'emqx' }}
- runs-on: aws-amd64 exclude:
use-self-hosted: false - runs-on: ubuntu-20.04
steps: use-self-hosted: true
- uses: actions/checkout@v2 - runs-on: aws-amd64
# to avoid dirty self-hosted runners use-self-hosted: false
- name: stop containers steps:
run: | - uses: AutoModality/action-clean@v1
docker rm -f $(docker ps -qa) || true - uses: actions/download-artifact@v3
docker network rm $(docker network ls -q) || true with:
- name: docker compose up name: source
if: endsWith(github.repository, 'emqx') path: .
env: - name: unzip source code
MYSQL_TAG: 8 run: unzip -q source.zip
REDIS_TAG: 6 # to avoid dirty self-hosted runners
MONGO_TAG: 4 - name: stop containers
PGSQL_TAG: 13 run: |
LDAP_TAG: 2.4.50 docker rm -f $(docker ps -qa) || true
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} docker network rm $(docker network ls -q) || true
run: | - name: docker compose up
docker-compose \ working-directory: source
-f .ci/docker-compose-file/docker-compose.yaml \ if: endsWith(github.repository, 'emqx')
-f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ env:
-f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ MYSQL_TAG: 8
-f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ REDIS_TAG: 6
-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ MONGO_TAG: 4
-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ PGSQL_TAG: 13
-f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ LDAP_TAG: 2.4.50
up -d --build GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: docker compose up run: |
if: endsWith(github.repository, 'emqx-enterprise') docker-compose \
env: -f .ci/docker-compose-file/docker-compose.yaml \
MYSQL_TAG: 8 -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
REDIS_TAG: 6 -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
MONGO_TAG: 4 -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
PGSQL_TAG: 13 -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
LDAP_TAG: 2.4.50 -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
OPENTSDB_TAG: latest -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
INFLUXDB_TAG: 1.7.6 up -d --build
DYNAMODB_TAG: 1.11.477 docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
TIMESCALE_TAG: latest-pg11 - name: docker compose up
CASSANDRA_TAG: 3.11.6 working-directory: source
RABBITMQ_TAG: 3.7 if: endsWith(github.repository, 'emqx-enterprise')
KAFKA_TAG: 2.5.0 env:
PULSAR_TAG: 2.3.2 MYSQL_TAG: 8
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDIS_TAG: 6
timeout-minutes: 20 MONGO_TAG: 4
run: | PGSQL_TAG: 13
docker-compose \ LDAP_TAG: 2.4.50
-f .ci/docker-compose-file/docker-compose.yaml \ OPENTSDB_TAG: latest
-f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ INFLUXDB_TAG: 1.7.6
-f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ DYNAMODB_TAG: 1.11.477
-f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ TIMESCALE_TAG: latest-pg11
-f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ CASSANDRA_TAG: 3.11.6
-f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ RABBITMQ_TAG: 3.7
-f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ KAFKA_TAG: 2.5.0
-f .ci/docker-compose-file/docker-compose-enterprise.yaml \ PULSAR_TAG: 2.3.2
-f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \ timeout-minutes: 20
-f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \ run: |
-f .ci/docker-compose-file/docker-compose-enterprise-kafka-tcp.yaml \ docker-compose \
-f .ci/docker-compose-file/docker-compose-enterprise-opentsdb-tcp.yaml \ -f .ci/docker-compose-file/docker-compose.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-pulsar-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
-f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
up -d --build -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" -f .ci/docker-compose-file/docker-compose-enterprise.yaml \
docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" -f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \
while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ -f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \
!= $(docker ps -a --filter name=client | wc -l) ]; do -f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \
sleep 5 -f .ci/docker-compose-file/docker-compose-enterprise-kafka-tcp.yaml \
done -f .ci/docker-compose-file/docker-compose-enterprise-opentsdb-tcp.yaml \
- name: run eunit -f .ci/docker-compose-file/docker-compose-enterprise-pulsar-tcp.yaml \
run: | -f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \
docker exec -i erlang bash -c "make eunit" -f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \
- name: run common test -f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \
run: | -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \
docker exec -i erlang bash -c "make ct" up -d --build
- name: run cover docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
run: | docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
printenv > .env while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" != $(docker ps -a --filter name=client | wc -l) ]; do
docker exec -i erlang bash -c "make cover" sleep 5
docker exec --env-file .env -i erlang bash -c "make coveralls" done
- name: cat rebar.crashdump - name: run common test
if: failure() run: docker exec -i erlang bash -c "make ${{ matrix.app_name }}-ct-pipeline"
run: if [ -f 'rebar3.crashdump' ];then cat 'rebar3.crashdump'; fi - name: run cover
- uses: actions/upload-artifact@v1 run: |
if: failure() printenv > .env
with: docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
name: logs docker exec -i erlang bash -c "make cover"
path: _build/test/logs docker exec --env-file .env -i erlang bash -c "make coveralls"
- uses: actions/upload-artifact@v1 - name: cat rebar.crashdump
with: if: failure()
name: cover working-directory: source
path: _build/test/cover run: if [ -f 'rebar3.crashdump' ];then cat 'rebar3.crashdump'; fi
- name: set log file name
if: failure()
run: echo "LOGFILENAME=logs-$(echo ${{ matrix.app_name }} | tr '/' '_')" >> $GITHUB_ENV
- uses: actions/upload-artifact@v3
if: failure()
with:
name: ${{ env.LOGFILENAME }}
path: source/_build/test/logs
if-no-files-found: warn
- uses: actions/upload-artifact@v3
with:
name: cover
path: source/_build/test/cover
if-no-files-found: warn
finish: make_cover:
needs: run_common_test needs:
runs-on: ubuntu-20.04 - eunit_and_proper
steps: - ct
- name: Coveralls Finished runs-on: ubuntu-20.04
env: container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps:
run: | - uses: AutoModality/action-clean@v1
curl -v -k https://coveralls.io/webhook \ - uses: actions/download-artifact@v3
--header "Content-Type: application/json" \ with:
--data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true name: source
path: .
- name: unzip source code
run: unzip -q source.zip
- uses: actions/download-artifact@v3
name: download cover data
with:
name: cover
path: source/_build/test/cover
- name: make cover
working-directory: source
run: make cover
- name: send to coveralls
working-directory: source
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: make coveralls
- name: get coveralls logs
working-directory: source
if: failure()
run: cat rebar3.crashdump
finish:
needs: make_cover
runs-on: ubuntu-20.04
steps:
- name: Coveralls Finished
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
curl -v -k https://coveralls.io/webhook \
--header "Content-Type: application/json" \
--data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true

8
Makefile
View File

@ -61,6 +61,14 @@ $1-ct: $(REBAR)
endef endef
$(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
## app/name-ct-pipeline targets are used in pipeline -> make cover data for each app
.PHONY: $(APPS:%=%-ct-pipeline)
define gen-app-ct-target-pipeline
$1-ct-pipeline: $(REBAR)
$(REBAR) ct --name 'test@127.0.0.1' -c -v --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
endef
$(foreach app,$(APPS),$(eval $(call gen-app-ct-target-pipeline,$(app))))
## apps/name-prop targets ## apps/name-prop targets
.PHONY: $(APPS:%=%-prop) .PHONY: $(APPS:%=%-prop)
define gen-app-prop-target define gen-app-prop-target

46
scripts/find-apps.sh
View File

@ -5,6 +5,29 @@ set -euo pipefail
# ensure dir # ensure dir
cd -P -- "$(dirname -- "$0")/.." cd -P -- "$(dirname -- "$0")/.."
help() {
echo
echo "-h|--help: To display this usage info"
echo "--json: Print apps in json"
}
WANT_JSON='no'
while [ "$#" -gt 0 ]; do
case $1 in
-h|--help)
help
exit 0
;;
--json)
WANT_JSON='yes'
shift 1
;;
*)
echo "unknown option $1"
exit 1
;;
esac
done
if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then
# Otherwise windows may resolve to find.exe # Otherwise windows may resolve to find.exe
FIND="/usr/bin/find" FIND="/usr/bin/find"
@ -17,17 +40,26 @@ find_app() {
"$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d "$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d
} }
# append emqx application first EM="emqx"
echo 'emqx' CE="$(find_app 'apps')"
find_app 'apps'
if [ -f 'EMQX_ENTERPRISE' ]; then if [ -f 'EMQX_ENTERPRISE' ]; then
find_app 'lib-ee' LIB="$(find_app 'lib-ee')"
else else
find_app 'lib-ce' LIB="$(find_app 'lib-ce')"
fi fi
## find directories in lib-extra ## find directories in lib-extra
find_app 'lib-extra' LIBE="$(find_app 'lib-extra')"
## find symlinks in lib-extra ## find symlinks in lib-extra
"$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print LIBES="$("$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print)"
APPS_ALL="$(echo -e "${EM}\n${CE}\n${LIB}\n${LIBE}\n${LIBES}")"
if [ "$WANT_JSON" = 'yes' ]; then
echo "${APPS_ALL}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")'
else
echo "${APPS_ALL}"
fi