Merge pull request #7305 from zmstone/chore-split-ct-jobs

ci: run ct in concurrent jobs

.github/workflows/run_test_cases.yaml

@@ -56,14 +56,39 @@ jobs:
             name: coverdata
             path: _build/test/cover
 
-    run_common_test:
+    find_apps:
+        runs-on: ubuntu-20.04
+        container: ghcr.io/emqx/emqx-builder/5.0-8:1.13.3-24.2.1-1-ubuntu20.04
+        outputs:
+          fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
+          docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
+        defaults:
+          run:
+            shell: bash
+        steps:
+        - uses: actions/checkout@v2
+        - name: find apps
+          id: run_find_apps
+          run: |
+            fast_ct_apps="$(./scripts/find-apps.sh --ct fast --json)"
+            docker_ct_apps="$(./scripts/find-apps.sh --ct docker --json)"
+            echo "::set-output name=fast_ct_apps::$fast_ct_apps"
+            echo "::set-output name=docker_ct_apps::$docker_ct_apps"
+
+    ct_docker:
+        needs: find_apps
         strategy:
+          fail-fast: false
           matrix:
+            app_name: ${{ fromJson(needs.find_apps.outputs.docker_ct_apps) }}
             otp_release:
               - "erlang23"
               - "erlang24"
 
         runs-on: ubuntu-20.04
+        defaults:
+          run:
+            shell: bash
 
         steps:
         - uses: actions/checkout@v2
@@ -85,10 +110,11 @@ jobs:
                 -f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \
                 -f .ci/docker-compose-file/docker-compose.yaml \
                 up -d --build
-          # produces ct.coverdata
+
+          # produces <app-name>.coverdata
         - name: run common test
           run: |
-            docker exec -i ${{ matrix.otp_release }} bash -c "make ct"
+            docker exec -i ${{ matrix.otp_release }} bash -c "make ${{ matrix.app_name }}-ct"
         - uses: actions/upload-artifact@v1
           if: matrix.otp_release == 'erlang24'
           with:
@@ -100,11 +126,48 @@ jobs:
             name: logs_${{ matrix.otp_release }}
             path: _build/test/logs
 
+    ct:
+        needs: find_apps
+        strategy:
+          fail-fast: false
+          matrix:
+            app_name: ${{ fromJson(needs.find_apps.outputs.fast_ct_apps) }}
+            otp:
+              - 24.2.1-1
+            elixir:
+              - 1.13.3
+            os:
+              - ubuntu20.04
+            arch:
+              - amd64
+        runs-on: ubuntu-20.04
+        container: "ghcr.io/emqx/emqx-builder/5.0-8:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+        defaults:
+          run:
+            shell: bash
+
+        steps:
+        - uses: actions/checkout@v2
+          # produces <app-name>.coverdata
+        - name: run common test
+          run: |
+            make ${{ matrix.app_name }}-ct
+        - uses: actions/upload-artifact@v1
+          if: matrix.otp == '24.2.1-1'
+          with:
+            name: coverdata
+            path: _build/test/cover
+        - uses: actions/upload-artifact@v1
+          if: failure()
+          with:
+            name: logs_${{ matrix.otp_release }}
+            path: _build/test/logs
 
     make_cover:
       needs:
         - eunit_and_proper
-        - run_common_test
+        - ct
+        - ct_docker
       strategy:
         matrix:
           otp:

Makefile

@@ -80,8 +80,8 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
 ## app/name-ct targets are intended for local tests hence cover is not enabled
 .PHONY: $(APPS:%=%-ct)
 define gen-app-ct-target
-$1-ct: conf-segs
+$1-ct: $(REBAR) conf-segs
-	@ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
+	@ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
 

apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl

@@ -85,7 +85,8 @@ init_per_suite(Config) ->
                 }
             ]
         }">>),
-    emqx_common_test_helpers:start_apps([emqx_dashboard, emqx_conf, ?APP], fun set_special_configs/1),
+    emqx_common_test_helpers:start_apps([emqx_conf, emqx_dashboard, ?APP],
+                                        fun set_special_configs/1),
     Config.
 
 set_special_configs(emqx_dashboard) ->

apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl

@@ -22,17 +22,25 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
+-define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard).
+
 -import(emqx_mgmt_api_test_util, [request_api/2, request_api/5, api_path/1, auth_header_/0]).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
+    application:load(emqx_conf),
+    ok = ekka:start(),
+    ok = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], infinity),
     emqx_retainer_SUITE:load_base_conf(),
     emqx_mgmt_api_test_util:init_suite([emqx_retainer]),
     Config.
 
 end_per_suite(Config) ->
+    ekka:stop(),
+    mria:stop(),
+    mria_mnesia:delete_schema(),
     emqx_mgmt_api_test_util:end_suite([emqx_retainer]),
     Config.
 

scripts/docker-ct-apps

@@ -0,0 +1,4 @@
+# apps need docker-compose to run CT
+apps/emqx_authn
+apps/emqx_authz
+apps/emqx_connector

scripts/find-apps.sh

@@ -5,6 +5,37 @@ set -euo pipefail
 # ensure dir
 cd -P -- "$(dirname -- "$0")/.."
 
+help() {
+    echo
+    echo "-h|--help:        To display this usage info"
+    echo "--ct fast|docker: Print apps which needs docker-compose to run ct"
+    echo "--json:           Print apps in json"
+}
+
+WANT_JSON='no'
+CT='novalue'
+while [ "$#" -gt 0 ]; do
+    case $1 in
+        -h|--help)
+            help
+            exit 0
+            ;;
+        --json)
+            WANT_JSON='yes'
+            shift 1
+            ;;
+
+        --ct)
+            CT="$2"
+            shift 2
+            ;;
+        *)
+            echo "unknown option $1"
+            exit 1
+            ;;
+    esac
+done
+
 if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then
     # Otherwise windows may resolve to find.exe
     FIND="/usr/bin/find"
@@ -17,5 +48,30 @@ find_app() {
     "$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d
 }
 
-find_app 'apps'
+CE="$(find_app 'apps')"
-find_app 'lib-ee'
+EE="$(find_app 'lib-ee')"
+
+if [ "$CT" = 'novalue' ]; then
+    echo -e "${CE}\n${EE}"
+    exit 0
+fi
+
+APPS_ALL="$(echo -e "${CE}\n${EE}")"
+APPS_DOCKER_CT="$(grep -v -E '^#.*' scripts/docker-ct-apps)"
+
+# shellcheck disable=SC2068
+for app in ${APPS_DOCKER_CT[@]}; do
+    APPS_ALL=("${APPS_ALL[@]/$app}")
+done
+
+if [ "$CT" = 'docker' ]; then
+    RESULT="${APPS_DOCKER_CT}"
+else
+    RESULT="${APPS_ALL[*]}"
+fi
+
+if [ "$WANT_JSON" = 'yes' ]; then
+    echo "${RESULT}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")'
+else
+    echo "${RESULT}" | xargs
+fi