From d6cd56558a88bbc6dee3aa96b1cb9d305732b634 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 22 Jun 2023 10:21:12 +0200
Subject: [PATCH] ci: split apps/emqx common test suites into chunks

---
 .github/workflows/run_test_cases.yaml | 18 ++++++-----
 Makefile                              |  2 +-
 scripts/ct/run.sh                     | 13 ++++++--
 scripts/find-apps.sh                  | 27 +++++++++++-----
 scripts/find-suites.sh                | 44 ++++++++++++++++++++++++---
 5 files changed, 82 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 3192c6278..a8f65b926 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -184,7 +184,9 @@ jobs:
           path: .
       - name: unzip source code
         run: unzip -q source.zip
-      - name: run tests
+
+        # produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
+      - name: run common tests
         working-directory: source
         env:
           DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
@@ -197,7 +199,8 @@ jobs:
           OPENTS_TAG: "9aa7f88"
           MINIO_TAG: "RELEASE.2023-03-20T20-16-18Z"
           PROFILE: ${{ matrix.profile }}
-          CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
+          SUITEGROUP: ${{ matrix.suitegroup }}
+          CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
         run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }}
       - uses: actions/upload-artifact@v3
         with:
@@ -206,7 +209,7 @@ jobs:
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}
+          name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
           path: source/_build/test/logs
 
   ct:
@@ -233,12 +236,13 @@ jobs:
       - name: unzip source code
         run: unzip -q source.zip
 
-        # produces $PROFILE-<app-name>.coverdata
-      - name: run common test
+        # produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
+      - name: run common tests
         working-directory: source
         env:
           PROFILE: ${{ matrix.profile }}
-          CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
+          SUITEGROUP: ${{ matrix.suitegroup }}
+          CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
         run: |
           make "${{ matrix.app }}-ct"
       - uses: actions/upload-artifact@v3
@@ -249,7 +253,7 @@ jobs:
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}
+          name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
           path: source/_build/test/logs
 
   make_cover:
diff --git a/Makefile b/Makefile
index f322a0a80..43e64359d 100644
--- a/Makefile
+++ b/Makefile
@@ -107,7 +107,7 @@ define gen-app-ct-target
 $1-ct: $(REBAR) merge-config clean-test-cluster-config
 	$(eval SUITES := $(shell $(SCRIPTS)/find-suites.sh $1))
 ifneq ($(SUITES),)
-		@ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \
+		ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \
 			--readable=$(CT_READABLE) \
 			--name $(CT_NODE_NAME) \
 			--cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \
diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh
index 135eec919..c99f153f6 100755
--- a/scripts/ct/run.sh
+++ b/scripts/ct/run.sh
@@ -285,9 +285,18 @@ elif [ "$CONSOLE" = 'yes' ]; then
     docker exec -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "make run"
 else
     if [ -z "${REBAR3CT:-}" ]; then
-        docker exec -e IS_CI="$IS_CI" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct"
+        docker exec -e IS_CI="$IS_CI" \
+                    -e PROFILE="$PROFILE" \
+                    -e SUITEGROUP="${SUITEGROUP:-}" \
+                    -e CT_COVER_EXPORT_PREFIX="${CT_COVER_EXPORT_PREFIX:-}" \
+                    -i $TTY "$ERLANG_CONTAINER" \
+                    bash -c "BUILD_WITHOUT_QUIC=1 make ${WHICH_APP}-ct"
     else
-        docker exec -e IS_CI="$IS_CI" -e PROFILE="$PROFILE" -i $TTY "$ERLANG_CONTAINER" bash -c "./rebar3 ct $REBAR3CT"
+        # this is an ad-hoc run
+        docker exec -e IS_CI="$IS_CI" \
+                    -e PROFILE="$PROFILE" \
+                    -i $TTY "$ERLANG_CONTAINER" \
+                    bash -c "./rebar3 ct $REBAR3CT"
     fi
     RESULT=$?
     if [ "$RESULT" -ne 0 ]; then
diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh
index bfb6ba2cc..9120181c9 100755
--- a/scripts/find-apps.sh
+++ b/scripts/find-apps.sh
@@ -55,13 +55,21 @@ fi
 ##################################################
 
 format_app_description() {
-    ## prefix is for github actions (they don't like slash in variables)
-    local prefix=${1//\//_}
-    echo -n -e "$(
-cat <<END
-    {"app": "${1}", "profile": "${2}", "runner": "${3}", "prefix": "${prefix}"}
+    local groups="$2"
+    local group=0
+    while [ "$groups" -gt $group ]; do
+        if [ $group -gt 0 ]; then
+            echo ", "
+        fi
+        group=$(( group + 1 ))
+        ## prefix is for github actions (they don't like slash in variables)
+        local prefix=${1//\//_}
+        echo -n -e "$(
+    cat <<END
+        {"app": "${1}", "suitegroup": "${group}_${groups}", "profile": "${3}", "runner": "${4}", "prefix": "${prefix}"}
 END
-    )"
+        )"
+    done
 }
 
 describe_app() {
@@ -87,7 +95,12 @@ describe_app() {
             exit 1
             ;;
     esac
-    format_app_description "$app" "$profile" "$runner"
+    if [[ "$app" == "apps/emqx" ]]; then
+        suitegroups=5
+    else
+        suitegroups=1
+    fi
+    format_app_description "$app" "$suitegroups" "$profile" "$runner"
 }
 
 matrix() {
diff --git a/scripts/find-suites.sh b/scripts/find-suites.sh
index e7c1b422e..685ab5ec8 100755
--- a/scripts/find-suites.sh
+++ b/scripts/find-suites.sh
@@ -1,16 +1,50 @@
 #!/usr/bin/env bash
 
-## this script prints out all test/*_SUITE.erl files of a given app,
+## If EMQX_CT_SUITES is provided, it prints the variable.
+## Otherwise this script tries to find all test/*_SUITE.erl files of then given app,
 ## file names are separated by comma for rebar3 ct's `--suite` option
 
+## If SUITEGROUP is set as M_N, it prints the Nth chunk of all suites.
+## SUITEGROUP default value is 1_1
+
 set -euo pipefail
 
 # ensure dir
 cd -P -- "$(dirname -- "$0")/.."
 
-if [ -z "${EMQX_CT_SUITES:-}" ]; then
-    TESTDIR="$1/test"
-    find "${TESTDIR}" -name "*_SUITE.erl" -print0 2>/dev/null | xargs -0 | tr ' ' ','
-else
+## EMQX_CT_SUITES is useful in ad-hoc runs
+if [ -n "${EMQX_CT_SUITES:-}" ]; then
     echo "${EMQX_CT_SUITES}"
+    exit 0
 fi
+
+TESTDIR="$1/test"
+# Get the output of the find command
+IFS=$'\n' read -r -d '' -a FILES < <(find "${TESTDIR}" -name "*_SUITE.erl" 2>/dev/null | sort && printf '\0')
+
+SUITEGROUP_RAW="${SUITEGROUP:-1_1}"
+SUITEGROUP="$(echo "$SUITEGROUP_RAW" | cut -d '_' -f1)"
+SUITEGROUP_COUNT="$(echo "$SUITEGROUP_RAW" | cut -d '_' -f2)"
+
+# Calculate the total number of files
+FILE_COUNT=${#FILES[@]}
+
+if (( SUITEGROUP > SUITEGROUP_COUNT )); then
+    echo "Error: SUITEGROUP in the format of M_N, M must be not greater than M"
+    exit 1
+fi
+
+# Calculate the number of files per group
+FILES_PER_GROUP=$(( (FILE_COUNT + SUITEGROUP_COUNT - 1) / SUITEGROUP_COUNT ))
+START_INDEX=$(( (SUITEGROUP - 1) * FILES_PER_GROUP ))
+END_INDEX=$(( START_INDEX + FILES_PER_GROUP ))
+
+# Print the desired suite group
+sep=''
+for (( i=START_INDEX; i<END_INDEX; i++ )); do
+    if (( i < FILE_COUNT )); then
+        echo -n "${sep}${FILES[$i]}"
+        sep=','
+    fi
+done
+echo