ci: make CI a little faster and less flaky

- use latest ubuntu22.04 emqx-builder image in compose files
- use xl runners for compiling and for emqx app tests
- make helm tests less flaky

.ci/docker-compose-file/docker-compose-kafka.yaml

@@ -18,7 +18,7 @@ services:
       - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret
   kdc:
     hostname: kdc.emqx.net
-    image:  ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu20.04
+    image:  ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04
     container_name: kdc.emqx.net
     expose:
       - 88 # kdc

.ci/docker-compose-file/docker-compose.yaml

@@ -3,17 +3,17 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu20.04}
+    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04}
     env_file:
       - conf.env
     environment:
-      GITHUB_ACTIONS: ${GITHUB_ACTIONS}
+      GITHUB_ACTIONS: ${GITHUB_ACTIONS:-}
-      GITHUB_TOKEN: ${GITHUB_TOKEN}
+      GITHUB_TOKEN: ${GITHUB_TOKEN:-}
-      GITHUB_RUN_ID: ${GITHUB_RUN_ID}
+      GITHUB_RUN_ID: ${GITHUB_RUN_ID:-}
-      GITHUB_SHA: ${GITHUB_SHA}
+      GITHUB_SHA: ${GITHUB_SHA:-}
-      GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER}
+      GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER:-}
-      GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}
+      GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME:-}
-      GITHUB_REF: ${GITHUB_REF}
+      GITHUB_REF: ${GITHUB_REF:-}
     networks:
       - emqx_bridge
     ports:

.github/workflows/_pr_entrypoint.yaml

@@ -25,6 +25,7 @@ jobs:
       version-emqx: ${{ steps.matrix.outputs.version-emqx }}
       version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
       runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted","ephemeral","linux","x64"]' || '["ubuntu-22.04"]' }}
+      xl_runner_labels: ${{ github.repository_owner == 'emqx' && '["self-hosted","ephemeral-xl","linux","x64"]' || '["ubuntu-22.04"]' }}
       builder: "ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04"
       builder_vsn: "5.2-3"
       otp_vsn: "25.3.2-2"
@@ -115,7 +116,7 @@ jobs:
           echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT
 
   compile:
-    runs-on: ${{ fromJSON(needs.sanity-checks.outputs.runner_labels) }}
+    runs-on: ${{ fromJSON(needs.sanity-checks.outputs.xl_runner_labels) }}
     container: ${{ needs.sanity-checks.outputs.builder }}
     needs:
       - sanity-checks
@@ -153,7 +154,7 @@ jobs:
       - compile
     uses: ./.github/workflows/run_emqx_app_tests.yaml
     with:
-      runner_labels: ${{ needs.sanity-checks.outputs.runner_labels }}
+      runner_labels: ${{ needs.sanity-checks.outputs.xl_runner_labels }}
       builder: ${{ needs.sanity-checks.outputs.builder }}
       before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
       after_ref: ${{ github.sha }}

.github/workflows/run_helm_tests.yaml

@@ -140,12 +140,13 @@ jobs:
           echo "waiting emqx started";
           sleep 10;
         done
-    - name: Get Token
+    - name: Setup 18083 port forwarding
+      run: |
+        nohup kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null &
+    - name: Get auth token
       run: |
-        kubectl port-forward service/${EMQX_NAME} 18083:18083 > /dev/null &
         curl --head -X GET --retry 10 --retry-connrefused --retry-delay 6 http://localhost:18083/status
         echo "TOKEN=$(curl --silent -X 'POST' 'http://127.0.0.1:18083/api/v5/login' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"username": "admin","password": "public"}' | jq -r ".token")" >> $GITHUB_ENV
-
     - name: Check cluster
       timeout-minutes: 1
       run: |

.github/workflows/run_test_cases.yaml

@@ -71,7 +71,7 @@ jobs:
 
   ct_docker:
     runs-on: ${{ fromJSON(inputs.runner_labels) }}
-    name: "ct_docker (${{ matrix.app }}-${{ matrix.suitegroup }})"
+    name: "${{ matrix.app }}-${{ matrix.suitegroup }} (${{ matrix.profile }})"
     strategy:
       fail-fast: false
       matrix:
@@ -122,7 +122,7 @@ jobs:
 
   ct:
     runs-on: ${{ fromJSON(inputs.runner_labels) }}
-    name: "ct (${{ matrix.app }}-${{ matrix.suitegroup }})"
+    name: "${{ matrix.app }}-${{ matrix.suitegroup }} (${{ matrix.profile }})"
     strategy:
       fail-fast: false
       matrix:

Makefile

@@ -104,6 +104,11 @@ ifneq ($(CASES),)
 CASES_ARG := --case $(CASES)
 endif
 
+# Allow user-set GROUPS environment variable
+ifneq ($(GROUPS),)
+GROUPS_ARG := --groups $(GROUPS)
+endif
+
 ## example:
 ## env SUITES=apps/appname/test/test_SUITE.erl CASES=t_foo make apps/appname-ct
 define gen-app-ct-target
@@ -115,6 +120,7 @@ ifneq ($(SUITES),)
 		--name $(CT_NODE_NAME) \
 		--cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \
 		--suite $(SUITES) \
+		$(GROUPS_ARG) \
 		$(CASES_ARG)
 else
 	@echo 'No suites found for $1'

scripts/ct/run.sh

@@ -109,22 +109,37 @@ fi
 ERLANG_CONTAINER='erlang'
 DOCKER_CT_ENVS_FILE="${WHICH_APP}/docker-ct"
 
-case "${WHICH_APP}" in
+if [ -z "${PROFILE+x}" ]; then
-    lib-ee*)
+    case "${WHICH_APP}" in
-        ## ensure enterprise profile when testing lib-ee applications
+        apps/emqx)
-        export PROFILE='emqx-enterprise'
+            export PROFILE='emqx-enterprise'
-        ;;
+            ;;
-    apps/*)
+        apps/emqx_bridge)
-        if [[ -f "${WHICH_APP}/BSL.txt" ]]; then
+            export PROFILE='emqx-enterprise'
-          export PROFILE='emqx-enterprise'
+            ;;
-        else
+        # emqx_connector test suite is using kafka bridge which is only available in emqx-enterprise
-          export PROFILE='emqx'
+        apps/emqx_connector)
-        fi
+            export PROFILE='emqx-enterprise'
-        ;;
+            ;;
-    *)
+        apps/emqx_dashboard)
-        export PROFILE="${PROFILE:-emqx}"
+            export PROFILE='emqx-enterprise'
-        ;;
+            ;;
-esac
+        lib-ee*)
+            ## ensure enterprise profile when testing lib-ee applications
+            export PROFILE='emqx-enterprise'
+            ;;
+        apps/*)
+            if [[ -f "${WHICH_APP}/BSL.txt" ]]; then
+                export PROFILE='emqx-enterprise'
+            else
+                export PROFILE='emqx'
+            fi
+            ;;
+        *)
+            export PROFILE="${PROFILE:-emqx}"
+            ;;
+    esac
+fi
 
 if [ -f "$DOCKER_CT_ENVS_FILE" ]; then
     # shellcheck disable=SC2002
@@ -276,14 +291,18 @@ if [ "$STOP" = 'no' ]; then
     set -e
 fi
 
-# rebar, mix and hex cache directory need to be writable by $DOCKER_USER
+if [ "$DOCKER_USER" != "root" ]; then
-docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "mkdir -p /.cache /.hex /.mix && chown $DOCKER_USER /.cache /.hex /.mix"
+    # the user must exist inside the container for `whoami` to work
-# need to initialize .erlang.cookie manually here because / is not writable by $DOCKER_USER
+    docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c \
-docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "openssl rand -base64 -hex 16 > /.erlang.cookie && chown $DOCKER_USER /.erlang.cookie && chmod 0400 /.erlang.cookie"
+          "useradd --uid $DOCKER_USER -M -d / emqx && \
-# the user must exist inside the container for `whoami` to work
+           mkdir -p /.cache /.hex /.mix && \
-docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "useradd --uid $DOCKER_USER -M -d / emqx" || true
+           chown $DOCKER_USER /.cache /.hex /.mix && \
-docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "chown -R $DOCKER_USER /var/lib/secret" || true
+           openssl rand -base64 -hex 16 > /.erlang.cookie && \
-docker exec -i $TTY -u root:root "$ERLANG_CONTAINER" bash -c "$INSTALL_ODBC" || true
+           chown $DOCKER_USER /.erlang.cookie && \
+           chmod 0400 /.erlang.cookie && \
+           chown -R $DOCKER_USER /var/lib/secret && \
+           $INSTALL_ODBC" || true
+fi
 
 if [ "$ONLY_UP" = 'yes' ]; then
     exit 0

scripts/find-apps.sh

@@ -54,7 +54,7 @@ fi
 ###### now deal with the github action's matrix.
 ##################################################
 
-format_app_description() {
+format_app_entry() {
     local groups="$2"
     local group=0
     while [ "$groups" -gt $group ]; do
@@ -72,48 +72,51 @@ END
     done
 }
 
-describe_app() {
-    app="$1"
-    local runner="host"
-    local profile
-    if [ -f "${app}/docker-ct" ]; then
-        runner="docker"
-    fi
-    case "${app}" in
-        apps/*)
-            if [[ -f "${app}/BSL.txt" ]]; then
-              profile='emqx-enterprise'
-            else
-              profile='emqx'
-            fi
-            ;;
-        lib-ee/*)
-            profile='emqx-enterprise'
-            ;;
-        *)
-            echo "unknown app: $app"
-            exit 1
-            ;;
-    esac
-    if [[ "$app" == "apps/emqx" ]]; then
-        suitegroups=5
-    else
-        suitegroups=1
-    fi
-    format_app_description "$app" "$suitegroups" "$profile" "$runner"
-}
-
 matrix() {
-    local sep='['
+    local runner
+    local profile
+    local entries=()
     for app in ${APPS_ALL}; do
-        row="$(describe_app "$app")"
+        if [ -f "${app}/docker-ct" ]; then
-        if [ -z "$row" ]; then
+            runner="docker"
-            continue
+        else
+            runner="host"
         fi
-        echo -n "${sep}${row}"
+        case "${app}" in
-        sep=', '
+            apps/emqx)
+                entries+=("$(format_app_entry "$app" 5 emqx "$runner")")
+                entries+=("$(format_app_entry "$app" 5 emqx-enterprise "$runner")")
+                ;;
+            apps/emqx_bridge)
+                entries+=("$(format_app_entry "$app" 1 emqx "$runner")")
+                entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")")
+                ;;
+            apps/emqx_connector)
+                entries+=("$(format_app_entry "$app" 1 emqx "$runner")")
+                entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")")
+                ;;
+            apps/emqx_dashboard)
+                entries+=("$(format_app_entry "$app" 1 emqx "$runner")")
+                entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")")
+                ;;
+            apps/*)
+                if [[ -f "${app}/BSL.txt" ]]; then
+                    profile='emqx-enterprise'
+                else
+                    profile='emqx'
+                fi
+                entries+=("$(format_app_entry "$app" 1 "$profile" "$runner")")
+                ;;
+            lib-ee/*)
+                entries+=("$(format_app_entry "$app" 1 emqx-enterprise "$runner")")
+                ;;
+            *)
+                echo "unknown app: $app"
+                exit 1
+                ;;
+        esac
     done
-    echo ']'
+    echo -n "[$(IFS=,; echo "${entries[*]}")]"
 }
 
 matrix