From 8ff8a8b55d45ecabe18cd4b7012368884bec5a71 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sat, 5 Nov 2022 13:31:48 +0100 Subject: [PATCH] ci: split fast (regular) and docker-compose (docker) ct --- .github/workflows/run_test_cases.yaml | 121 +++++++++++++++----------- Makefile | 7 +- apps/emqx_auth_ldap/docker-ct | 0 apps/emqx_auth_mongo/docker-ct | 0 apps/emqx_auth_mysql/docker-ct | 0 apps/emqx_auth_pgsql/docker-ct | 0 apps/emqx_auth_redis/docker-ct | 0 scripts/find-apps.sh | 46 +++++++--- 8 files changed, 110 insertions(+), 64 deletions(-) create mode 100644 apps/emqx_auth_ldap/docker-ct create mode 100644 apps/emqx_auth_mongo/docker-ct create mode 100644 apps/emqx_auth_mysql/docker-ct create mode 100644 apps/emqx_auth_pgsql/docker-ct create mode 100644 apps/emqx_auth_redis/docker-ct diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index d67ed1395..afc122751 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -14,31 +14,37 @@ jobs: runs-on: ubuntu-20.04 container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 outputs: - ct_apps: ${{ steps.run_find_apps.outputs.ct_apps }} + fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }} + docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }} steps: - uses: actions/checkout@v3 with: path: source fetch-depth: 0 - name: git credentials + if: endsWith(github.repository, 'emqx-enterprise') run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - fi + echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials + git config --global credential.helper store - name: find_ct_apps working-directory: source id: run_find_apps # emqx_plugin_libs doesn't have a test suite -> excluded from app list - # emqx ct is run independently -> exclude it from the app list + # emqx ct is run independently -> exclude it from the app list (regular ct) + # lib-ee/emqx_node_rebalance -> exclude it from app list (regular ct) run: | - ct_apps="$(./scripts/find-apps.sh --json | jq -c 'del (.[] | select (. == "apps/emqx_plugin_libs" or . == "emqx"))')" - echo "ct-apps: $ct_apps" - echo "ct_apps=$ct_apps" >> $GITHUB_OUTPUT + fast_ct_apps="$(./scripts/find-apps.sh --ct fast --json)" + docker_ct_apps="$(./scripts/find-apps.sh --ct docker --json)" + echo "fast_ct_apps=$fast_ct_apps" | tee -a $GITHUB_OUTPUT + echo "docker_ct_apps=$docker_ct_apps" | tee -a $GITHUB_OUTPUT - name: get_all_deps working-directory: source run: | - make deps-all + # build the default profile for two purposes + # 1. download all dependencies (so the individual app runs do not depend on github credentials) + # 2. some of the files such as segmented config files are not created when compiling only the test profile + make + # compile test profile to speed up the individual runs ./rebar3 as test compile cd .. zip -ryq source.zip source/* source/.[^.]* @@ -75,12 +81,24 @@ jobs: path: source/_build/test/cover if-no-files-found: warn - emqx_ct: + fast_ct: needs: prepare runs-on: ubuntu-20.04 container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 strategy: fail-fast: false + matrix: + app_name: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }} + runs-on: + - aws-amd64 + - ubuntu-20.04 + use-self-hosted: + - ${{ github.repository_owner == 'emqx' }} + exclude: + - runs-on: ubuntu-20.04 + use-self-hosted: true + - runs-on: aws-amd64 + use-self-hosted: false steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 @@ -89,24 +107,26 @@ jobs: path: . - name: unzip source code run: unzip -o -q source.zip - # produces emqx-emqx.coverdata - - name: emqx-ct-pipeline + # produces emqx-.coverdata + - name: make-app-ct-pipeline working-directory: source - run: make emqx-ct-pipeline + run: | + export CT_READABLE=true + make ${{ matrix.app_name }}-ct-pipeline - uses: actions/upload-artifact@v3 with: name: cover path: source/_build/test/cover if-no-files-found: warn - ct: + docker_ct: needs: prepare runs-on: ${{ matrix.runs-on }} strategy: max-parallel: 12 fail-fast: false matrix: - app_name: ${{ fromJson(needs.prepare.outputs.ct_apps) }} + app_name: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }} runs-on: - aws-amd64 - ubuntu-20.04 @@ -132,28 +152,6 @@ jobs: docker network rm $(docker network ls -q) || true - name: docker compose up working-directory: source - if: endsWith(github.repository, 'emqx') - env: - MYSQL_TAG: 8 - REDIS_TAG: 6 - MONGO_TAG: 4 - PGSQL_TAG: 13 - LDAP_TAG: 2.4.50 - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - docker-compose \ - -f .ci/docker-compose-file/docker-compose.yaml \ - -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ - -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ - up -d --build - docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" - - name: docker compose up - working-directory: source - if: endsWith(github.repository, 'emqx-enterprise') env: MYSQL_TAG: 8 REDIS_TAG: 6 @@ -169,16 +167,19 @@ jobs: KAFKA_TAG: 2.5.0 PULSAR_TAG: 2.3.2 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - timeout-minutes: 20 + timeout-minutes: 40 run: | - docker-compose \ + ulimit -n + docker_compose_files="\ -f .ci/docker-compose-file/docker-compose.yaml \ -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ - -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ + -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml" + ee_docker_compose_files="\ -f .ci/docker-compose-file/docker-compose-enterprise.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \ @@ -189,16 +190,30 @@ jobs: -f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \ -f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \ - up -d --build - docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml" + if [ -f EMQX_ENTERPRISE ]; then + docker_compose_files="${docker_compose_files} ${ee_docker_compose_files}" + fi + # only build ldap docker image when necessary, it takes 9 minutes + if [[ ${{ matrix.app_name }} == *ldap ]]; then + docker_compose_files="${docker_compose_files} -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml" + fi + docker-compose $docker_compose_files up -d --build + if [ -f EMQX_ENTERPRISE ]; then + docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" + fi docker exec -i erlang bash -c "git config --global --add safe.directory /emqx" - while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \ - != $(docker ps -a --filter name=client | wc -l) ]; do - sleep 5 + clients="$(docker ps -q --filter name=client)" + for client in ${clients}; do + docker ps -a --filter name=client + echo "waiting for docker ${client} to exit" + if ! timeout 60 docker wait "${client}"; then + docker-compose $docker_compose_files logs | tee docker-compose.log + exit 1 + fi done - name: run common test - run: docker exec -i erlang bash -c "make ${{ matrix.app_name }}-ct-pipeline" + run: docker exec -i erlang bash -c "env CT_READABLE=true make ${{ matrix.app_name }}-ct-pipeline" - name: cat rebar.crashdump if: failure() working-directory: source @@ -210,7 +225,9 @@ jobs: if: failure() with: name: ${{ env.LOGFILENAME }} - path: source/_build/test/logs + path: | + docker-compose.log + source/_build/test/logs if-no-files-found: warn - uses: actions/upload-artifact@v3 with: @@ -221,8 +238,8 @@ jobs: make_cover: needs: - eunit_and_proper - - emqx_ct - - ct + - fast_ct + - docker_ct runs-on: ubuntu-20.04 container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04 steps: diff --git a/Makefile b/Makefile index dfe6ceaa0..64f278098 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,8 @@ REL_PROFILES := emqx emqx-edge PKG_PROFILES := emqx-pkg emqx-edge-pkg PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default +CT_READABLE ?= false + export REBAR_GIT_CLONE_OPTIONS += --depth=1 .PHONY: default @@ -57,7 +59,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh) .PHONY: $(APPS:%=%-ct) define gen-app-ct-target $1-ct: $(REBAR) - $(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) + $(REBAR) ct --name 'test@127.0.0.1' -v --readable $(CT_READABLE) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) @@ -65,7 +67,7 @@ $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) .PHONY: $(APPS:%=%-ct-pipeline) define gen-app-ct-target-pipeline $1-ct-pipeline: $(REBAR) - $(REBAR) ct --name 'test@127.0.0.1' -c -v --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) + $(REBAR) ct --name 'test@127.0.0.1' -c -v --readable $(CT_READABLE) --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target-pipeline,$(app)))) @@ -86,6 +88,7 @@ coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send .PHONY: $(REL_PROFILES) + $(REL_PROFILES:%=%): $(REBAR) get-dashboard @$(REBAR) as $(@) do compile,release diff --git a/apps/emqx_auth_ldap/docker-ct b/apps/emqx_auth_ldap/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_auth_mongo/docker-ct b/apps/emqx_auth_mongo/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_auth_mysql/docker-ct b/apps/emqx_auth_mysql/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_auth_pgsql/docker-ct b/apps/emqx_auth_pgsql/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/apps/emqx_auth_redis/docker-ct b/apps/emqx_auth_redis/docker-ct new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 9f188cf34..9ed84273a 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -7,11 +7,13 @@ cd -P -- "$(dirname -- "$0")/.." help() { echo - echo "-h|--help: To display this usage info" - echo "--json: Print apps in json" + echo "-h|--help: To display this usage info" + echo "--json: Print apps in json" + echo "-ct fast|docker: Only find apps for CT" } WANT_JSON='no' +CT='novalue' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -22,12 +24,17 @@ while [ "$#" -gt 0 ]; do WANT_JSON='yes' shift 1 ;; + --ct) + CT="$2" + shift 2 + ;; *) echo "unknown option $1" exit 1 ;; esac done + if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then # Otherwise windows may resolve to find.exe FIND="/usr/bin/find" @@ -41,25 +48,44 @@ find_app() { } EM="emqx" -CE="$(find_app 'apps')" - +SHARED="$(find_app 'apps')" if [ -f 'EMQX_ENTERPRISE' ]; then LIB="$(find_app 'lib-ee')" else LIB="$(find_app 'lib-ce')" fi - ## find directories in lib-extra LIBE="$(find_app 'lib-extra')" - ## find symlinks in lib-extra LIBES="$("$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print)" -APPS_ALL="$(echo -e "${EM}\n${CE}\n${LIB}\n${LIBE}\n${LIBES}")" +APPS_ALL="$(echo -e "${EM}\n${SHARED}\n${LIB}\n${LIBE}\n${LIBES}")" -if [ "$WANT_JSON" = 'yes' ]; then - echo "${APPS_ALL}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")' +if [ "$CT" = 'novalue' ]; then + RESULT="${APPS_ALL}" else - echo "${APPS_ALL}" + APPS_NORMAL_CT=( ) + APPS_DOCKER_CT=( ) + for app in ${APPS_ALL}; do + if [ "$app" = 'apps/emqx_plugin_libs' ]; then + # This app has no test SUITE + continue + fi + if [ -f "${app}/docker-ct" ]; then + APPS_DOCKER_CT+=("$app") + else + APPS_NORMAL_CT+=("$app") + fi + done + if [ "$CT" = 'docker' ]; then + RESULT="${APPS_DOCKER_CT[*]}" + else + RESULT="${APPS_NORMAL_CT[*]}" + fi fi +if [ "$WANT_JSON" = 'yes' ]; then + echo "${RESULT}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")' +else + echo "${RESULT}" | xargs +fi