From 25b6a1c158cc0ffc176d6ccb9132c44334b13605 Mon Sep 17 00:00:00 2001
From: Traphalet <nicolaeandrei.varga@hotmail.com>
Date: Tue, 25 Oct 2022 09:24:16 +0300
Subject: [PATCH 01/13] ci(test): test parallel CT runs on self-hosted runners

---
 .github/workflows/run_test_cases.yaml | 362 ++++++++++++++++----------
 Makefile                              |   8 +
 scripts/find-apps.sh                  |  46 +++-
 3 files changed, 275 insertions(+), 141 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 295d1d0d9..a2b7e6e53 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -10,140 +10,234 @@ on:
   pull_request:
 
 jobs:
-    run_proper_test:
-        runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
+  prepare:
+    runs-on: ubuntu-20.04
+    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
+    outputs:
+      ct_apps: ${{ steps.run_find_apps.outputs.ct_apps }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: source
+          fetch-depth: 0
+      - name: git credentials
+        run: |
+          if make emqx-ee --dry-run > /dev/null 2>&1; then
+            echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
+            git config --global credential.helper store
+          fi
+      - name: find_ct_apps
+        working-directory: source
+        id: run_find_apps
+        # emqx_plugin_libs doesn't have a test suite -> excluded from app list
+        run: |
+          ct_apps="$(./scripts/find-apps.sh --json | jq -c 'del (.[] | select (. == "apps/emqx_plugin_libs"))')"
+          echo "ct-apps: $ct_apps"
+          echo "ct_apps=$ct_apps" >> $GITHUB_OUTPUT
+      - name: get_all_deps
+        working-directory: source
+        run: |
+          make deps-all
+          ./rebar3 as test compile
+          cd ..
+          zip -ryq source.zip source/* source/.[^.]*
+      - uses: actions/upload-artifact@v3
+        with:
+          name: source
+          path: source.zip
 
-        steps:
-        - uses: actions/checkout@v2
-        - name: set git credentials
-          run: |
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-              git config --global credential.helper store
-            fi
-        - name: proper
-          run: make proper
+  eunit_and_proper:
+    needs: prepare
+    runs-on: ubuntu-20.04
+    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        task:
+          - eunit
+          - proper
+    steps:
+      - uses: AutoModality/action-clean@v1
+      - uses: actions/download-artifact@v3
+        with:
+          name: source
+          path: .
+      - name: unzip source code
+        run: unzip -o -q source.zip
+      # produces eunit.coverdata and proper.coverdata
+      - name: eunit and proper
+        working-directory: source
+        run: make ${{ matrix.task }}
+      - uses: actions/upload-artifact@v3
+        with:
+          name: cover-${{ matrix.task }}
+          path: source/_build/test/cover
 
-    run_common_test:
-        runs-on: ${{ matrix.runs-on }}
-        strategy:
-          fail-fast: false
-          matrix:
-            runs-on:
-              - aws-amd64
-              - ubuntu-20.04
-            use-self-hosted:
-              - ${{ github.repository_owner == 'emqx' }}
-            exclude:
-              - runs-on: ubuntu-20.04
-                use-self-hosted: true
-              - runs-on: aws-amd64
-                use-self-hosted: false
-        steps:
-        - uses: actions/checkout@v2
-          # to avoid dirty self-hosted runners
-        - name: stop containers
-          run: |
-            docker rm -f $(docker ps -qa) || true
-            docker network rm $(docker network ls -q) || true
-        - name: docker compose up
-          if: endsWith(github.repository, 'emqx')
-          env:
-            MYSQL_TAG: 8
-            REDIS_TAG: 6
-            MONGO_TAG: 4
-            PGSQL_TAG: 13
-            LDAP_TAG: 2.4.50
-            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          run: |
-            docker-compose \
-                -f .ci/docker-compose-file/docker-compose.yaml \
-                -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
-                -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
-                up -d --build
-        - name: docker compose up
-          if: endsWith(github.repository, 'emqx-enterprise')
-          env:
-            MYSQL_TAG: 8
-            REDIS_TAG: 6
-            MONGO_TAG: 4
-            PGSQL_TAG: 13
-            LDAP_TAG: 2.4.50
-            OPENTSDB_TAG: latest
-            INFLUXDB_TAG: 1.7.6
-            DYNAMODB_TAG: 1.11.477
-            TIMESCALE_TAG: latest-pg11
-            CASSANDRA_TAG: 3.11.6
-            RABBITMQ_TAG: 3.7
-            KAFKA_TAG: 2.5.0
-            PULSAR_TAG: 2.3.2
-            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          timeout-minutes: 20
-          run: |
-            docker-compose \
-                -f .ci/docker-compose-file/docker-compose.yaml \
-                -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
-                -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-kafka-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-opentsdb-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-pulsar-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \
-                up -d --build
-            docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
-            docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
-            while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
-                 != $(docker ps -a --filter name=client | wc -l) ]; do
-              sleep 5
-            done
-        - name: run eunit
-          run: |
-            docker exec -i erlang bash -c "make eunit"
-        - name: run common test
-          run: |
-            docker exec -i erlang bash -c "make ct"
-        - name: run cover
-          run: |
-            printenv > .env
-            docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
-            docker exec -i erlang bash -c "make cover"
-            docker exec --env-file .env -i erlang bash -c "make coveralls"
-        - name: cat rebar.crashdump
-          if: failure()
-          run: if [ -f 'rebar3.crashdump' ];then cat 'rebar3.crashdump'; fi
-        - uses: actions/upload-artifact@v1
-          if: failure()
-          with:
-            name: logs
-            path: _build/test/logs
-        - uses: actions/upload-artifact@v1
-          with:
-            name: cover
-            path: _build/test/cover
+  ct:
+    needs: prepare
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      max-parallel: 12
+      fail-fast: false
+      matrix:
+        app_name: ${{ fromJson(needs.prepare.outputs.ct_apps) }}
+        runs-on:
+          - aws-amd64
+          - ubuntu-20.04
+        use-self-hosted:
+          - ${{ github.repository_owner == 'emqx' }}
+        exclude:
+          - runs-on: ubuntu-20.04
+            use-self-hosted: true
+          - runs-on: aws-amd64
+            use-self-hosted: false
+    steps:
+      - uses: AutoModality/action-clean@v1
+      - uses: actions/download-artifact@v3
+        with:
+          name: source
+          path: .
+      - name: unzip source code
+        run: unzip -q source.zip
+        # to avoid dirty self-hosted runners
+      - name: stop containers
+        run: |
+          docker rm -f $(docker ps -qa) || true
+          docker network rm $(docker network ls -q) || true
+      - name: docker compose up
+        working-directory: source
+        if: endsWith(github.repository, 'emqx')
+        env:
+          MYSQL_TAG: 8
+          REDIS_TAG: 6
+          MONGO_TAG: 4
+          PGSQL_TAG: 13
+          LDAP_TAG: 2.4.50
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          docker-compose \
+              -f .ci/docker-compose-file/docker-compose.yaml \
+              -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
+              -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
+              up -d --build
+          docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
+      - name: docker compose up
+        working-directory: source
+        if: endsWith(github.repository, 'emqx-enterprise')
+        env:
+          MYSQL_TAG: 8
+          REDIS_TAG: 6
+          MONGO_TAG: 4
+          PGSQL_TAG: 13
+          LDAP_TAG: 2.4.50
+          OPENTSDB_TAG: latest
+          INFLUXDB_TAG: 1.7.6
+          DYNAMODB_TAG: 1.11.477
+          TIMESCALE_TAG: latest-pg11
+          CASSANDRA_TAG: 3.11.6
+          RABBITMQ_TAG: 3.7
+          KAFKA_TAG: 2.5.0
+          PULSAR_TAG: 2.3.2
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        timeout-minutes: 20
+        run: |
+          docker-compose \
+              -f .ci/docker-compose-file/docker-compose.yaml \
+              -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
+              -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-kafka-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-opentsdb-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-pulsar-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \
+              up -d --build
+          docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
+          docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
+          while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
+               != $(docker ps -a --filter name=client | wc -l) ]; do
+            sleep 5
+          done
+      - name: run common test
+        run: docker exec -i erlang bash -c "make ${{ matrix.app_name }}-ct-pipeline"
+      - name: run cover
+        run: |
+          printenv > .env
+          docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
+          docker exec -i erlang bash -c "make cover"
+          docker exec --env-file .env -i erlang bash -c "make coveralls"
+      - name: cat rebar.crashdump
+        if: failure()
+        working-directory: source
+        run: if [ -f 'rebar3.crashdump' ];then cat 'rebar3.crashdump'; fi
+      - name: set log file name
+        if: failure()
+        run: echo "LOGFILENAME=logs-$(echo ${{ matrix.app_name }} | tr '/' '_')" >> $GITHUB_ENV
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: ${{ env.LOGFILENAME }}
+          path: source/_build/test/logs
+          if-no-files-found: warn
+      - uses: actions/upload-artifact@v3
+        with:
+          name: cover
+          path: source/_build/test/cover
+          if-no-files-found: warn
 
-    finish:
-      needs: run_common_test
-      runs-on: ubuntu-20.04
-      steps:
-        - name: Coveralls Finished
-          env:
-            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          run: |
-            curl -v -k https://coveralls.io/webhook \
-                 --header "Content-Type: application/json" \
-                 --data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true
+  make_cover:
+    needs:
+      - eunit_and_proper
+      - ct
+    runs-on: ubuntu-20.04
+    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
+    steps:
+      - uses: AutoModality/action-clean@v1
+      - uses: actions/download-artifact@v3
+        with:
+          name: source
+          path: .
+      - name: unzip source code
+        run: unzip -q source.zip
+      - uses: actions/download-artifact@v3
+        name: download cover data
+        with:
+          name: cover
+          path: source/_build/test/cover
+      - name: make cover
+        working-directory: source
+        run: make cover
+      - name: send to coveralls
+        working-directory: source
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: make coveralls
+      - name: get coveralls logs
+        working-directory: source
+        if: failure()
+        run: cat rebar3.crashdump
+
+  finish:
+    needs: make_cover
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Coveralls Finished
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          curl -v -k https://coveralls.io/webhook \
+               --header "Content-Type: application/json" \
+               --data "{\"repo_name\":\"$GITHUB_REPOSITORY\",\"repo_token\":\"$GITHUB_TOKEN\",\"payload\":{\"build_num\":$GITHUB_RUN_ID,\"status\":\"done\"}}" || true
diff --git a/Makefile b/Makefile
index aaa0b683a..dfe6ceaa0 100644
--- a/Makefile
+++ b/Makefile
@@ -61,6 +61,14 @@ $1-ct: $(REBAR)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
 
+## app/name-ct-pipeline targets are used in pipeline -> make cover data for each app
+.PHONY: $(APPS:%=%-ct-pipeline)
+define gen-app-ct-target-pipeline
+$1-ct-pipeline: $(REBAR)
+	$(REBAR) ct --name 'test@127.0.0.1' -c -v --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
+endef
+$(foreach app,$(APPS),$(eval $(call gen-app-ct-target-pipeline,$(app))))
+
 ## apps/name-prop targets
 .PHONY: $(APPS:%=%-prop)
 define gen-app-prop-target
diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh
index c4a0eec62..9f188cf34 100755
--- a/scripts/find-apps.sh
+++ b/scripts/find-apps.sh
@@ -5,6 +5,29 @@ set -euo pipefail
 # ensure dir
 cd -P -- "$(dirname -- "$0")/.."
 
+help() {
+    echo
+    echo "-h|--help:        To display this usage info"
+    echo "--json:           Print apps in json"
+}
+
+WANT_JSON='no'
+while [ "$#" -gt 0 ]; do
+    case $1 in
+        -h|--help)
+            help
+            exit 0
+            ;;
+        --json)
+            WANT_JSON='yes'
+            shift 1
+            ;;
+        *)
+            echo "unknown option $1"
+            exit 1
+            ;;
+    esac
+done
 if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then
     # Otherwise windows may resolve to find.exe
     FIND="/usr/bin/find"
@@ -17,17 +40,26 @@ find_app() {
     "$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d
 }
 
-# append emqx application first
-echo 'emqx'
+EM="emqx"
+CE="$(find_app 'apps')"
 
-find_app 'apps'
 if [ -f 'EMQX_ENTERPRISE' ]; then
-    find_app 'lib-ee'
+    LIB="$(find_app 'lib-ee')"
 else
-    find_app 'lib-ce'
+    LIB="$(find_app 'lib-ce')"
 fi
 
 ## find directories in lib-extra
-find_app 'lib-extra'
+LIBE="$(find_app 'lib-extra')"
+
 ## find symlinks in lib-extra
-"$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print
+LIBES="$("$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print)"
+
+APPS_ALL="$(echo -e "${EM}\n${CE}\n${LIB}\n${LIBE}\n${LIBES}")"
+
+if [ "$WANT_JSON" = 'yes' ]; then
+    echo "${APPS_ALL}" | xargs | tr -d '\n' | jq -R -s -c 'split(" ")'
+else
+    echo "${APPS_ALL}"
+fi
+

From 6d943181d7d30d9b61f6b5c706fc718c2270395a Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Nov 2022 14:27:28 +0100
Subject: [PATCH 02/13] test: ensure emqx_dashboard tables are created for data
 export import

---
 .../test/emqx_bridge_mqtt_data_export_import_SUITE.erl        | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl
index 067dc0a4c..fc95e3e4e 100644
--- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl
+++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl
@@ -32,6 +32,8 @@ init_per_suite(Cfg) ->
     ok = emqx_dashboard_admin:mnesia(boot),
     application:load(emqx_modules),
     application:load(emqx_bridge_mqtt),
+    ekka_mnesia:start(),
+    emqx_dashboard_admin:mnesia(boot),
     emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]),
     application:ensure_all_started(emqx_dashboard),
     Cfg.
@@ -185,4 +187,4 @@ remove_resources() ->
     lists:foreach(fun(#resource{id = Id}) ->
         emqx_rule_engine:delete_resource(Id)
     end, emqx_rule_registry:get_resources()),
-    timer:sleep(500).
\ No newline at end of file
+    timer:sleep(500).

From 25863b677354b36751dd646cc6c9de853860cc21 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Nov 2022 14:49:16 +0100
Subject: [PATCH 03/13] test: ensure clean start for emqx_mod_sup_SUITE

---
 lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl
index 055ccd2cb..e75d2379c 100644
--- a/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl
+++ b/lib-ce/emqx_modules/test/emqx_mod_sup_SUITE.erl
@@ -23,6 +23,18 @@
 
 all() -> emqx_ct:all(?MODULE).
 
+init_per_suite(Config) ->
+    %% do not start the application
+    %% only testing the root supervisor in this suite
+    application:stop(emqx_modules),
+    {ok, Pid} = emqx_mod_sup:start_link(),
+    unlink(Pid),
+    Config.
+
+end_per_suite(_Config) ->
+    exit(whereis(emqx_mod_sup), kill),
+    ok.
+
 %%--------------------------------------------------------------------
 %% Test cases
 %%--------------------------------------------------------------------

From 3d7dd4f738a82aec55dbea6d8e01fc8673fdd6dc Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Nov 2022 15:28:01 +0100
Subject: [PATCH 04/13] test: ensure $mqtt_pub_caps is cleared after test run

---
 test/emqx_channel_SUITE.erl          | 21 ++++++++++++---------
 test/emqx_mqtt_caps_SUITE.erl        | 22 ++++++++++++----------
 test/emqx_mqtt_protocol_v5_SUITE.erl | 12 ++++++------
 3 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/test/emqx_channel_SUITE.erl b/test/emqx_channel_SUITE.erl
index 4f250bd4a..d46d95ff2 100644
--- a/test/emqx_channel_SUITE.erl
+++ b/test/emqx_channel_SUITE.erl
@@ -822,15 +822,18 @@ t_enrich_connack_caps(_) ->
                           wildcard_subscription => true
                          }
                      end),
-    AckProps = emqx_channel:enrich_connack_caps(#{}, channel()),
-    ?assertMatch(#{'Retain-Available' := 1,
-                   'Maximum-Packet-Size' := 1024,
-                   'Topic-Alias-Maximum' := 10,
-                   'Wildcard-Subscription-Available' := 1,
-                   'Subscription-Identifier-Available' := 1,
-                   'Shared-Subscription-Available' := 1
-                  }, AckProps),
-    ok = meck:unload(emqx_mqtt_caps).
+    try
+        AckProps = emqx_channel:enrich_connack_caps(#{}, channel()),
+        ?assertMatch(#{'Retain-Available' := 1,
+                    'Maximum-Packet-Size' := 1024,
+                    'Topic-Alias-Maximum' := 10,
+                    'Wildcard-Subscription-Available' := 1,
+                    'Subscription-Identifier-Available' := 1,
+                    'Shared-Subscription-Available' := 1
+                    }, AckProps)
+    after
+        ok = meck:unload(emqx_mqtt_caps)
+    end.
 
 %%--------------------------------------------------------------------
 %% Test cases for terminate
diff --git a/test/emqx_mqtt_caps_SUITE.erl b/test/emqx_mqtt_caps_SUITE.erl
index c2b1a477b..8afd6a3be 100644
--- a/test/emqx_mqtt_caps_SUITE.erl
+++ b/test/emqx_mqtt_caps_SUITE.erl
@@ -52,13 +52,15 @@ t_check_sub(_) ->
                 wildcard_subscription => false
                },
     emqx_zone:set_env(zone, '$mqtt_sub_caps', SubCaps),
-    timer:sleep(50),
-    ClientInfo = #{zone => zone},
-    ok = emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts),
-    ?assertEqual({error, ?RC_TOPIC_FILTER_INVALID},
-                 emqx_mqtt_caps:check_sub(ClientInfo, <<"a/b/c/d">>, SubOpts)),
-    ?assertEqual({error, ?RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED},
-                 emqx_mqtt_caps:check_sub(ClientInfo, <<"+/#">>, SubOpts)),
-    ?assertEqual({error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED},
-                 emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts#{share => true})),
-    emqx_zone:unset_env(zone, '$mqtt_pub_caps').
+    try
+        ClientInfo = #{zone => zone},
+        ok = emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts),
+        ?assertEqual({error, ?RC_TOPIC_FILTER_INVALID},
+                    emqx_mqtt_caps:check_sub(ClientInfo, <<"a/b/c/d">>, SubOpts)),
+        ?assertEqual({error, ?RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED},
+                    emqx_mqtt_caps:check_sub(ClientInfo, <<"+/#">>, SubOpts)),
+        ?assertEqual({error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED},
+                    emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts#{share => true}))
+    after
+        emqx_zone:unset_env(zone, '$mqtt_pub_caps')
+    end.
diff --git a/test/emqx_mqtt_protocol_v5_SUITE.erl b/test/emqx_mqtt_protocol_v5_SUITE.erl
index 05424d642..33592b849 100644
--- a/test/emqx_mqtt_protocol_v5_SUITE.erl
+++ b/test/emqx_mqtt_protocol_v5_SUITE.erl
@@ -468,8 +468,8 @@ t_connack_max_qos_allowed(_) ->
 
     %% max_qos_allowed = 0
     emqx_zone:set_env(external, max_qos_allowed, 0),
-    persistent_term:erase({emqx_zone, external, '$mqtt_caps'}),
-    persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}),
+    emqx_zone:unset_env(external, '$mqtt_caps'),
+    emqx_zone:unset_env(external, '$mqtt_pub_caps'),
 
     {ok, Client1} = emqtt:start_link([{proto_ver, v5}]),
     {ok, Connack1} = emqtt:connect(Client1),
@@ -496,8 +496,8 @@ t_connack_max_qos_allowed(_) ->
 
     %% max_qos_allowed = 1
     emqx_zone:set_env(external, max_qos_allowed, 1),
-    persistent_term:erase({emqx_zone, external, '$mqtt_caps'}),
-    persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}),
+    emqx_zone:unset_env(external, '$mqtt_caps'),
+    emqx_zone:unset_env(external, '$mqtt_pub_caps'),
 
     {ok, Client3} = emqtt:start_link([{proto_ver, v5}]),
     {ok, Connack3} = emqtt:connect(Client3),
@@ -524,8 +524,8 @@ t_connack_max_qos_allowed(_) ->
 
     %% max_qos_allowed = 2
     emqx_zone:set_env(external, max_qos_allowed, 2),
-    persistent_term:erase({emqx_zone, external, '$mqtt_caps'}),
-    persistent_term:erase({emqx_zone, external, '$mqtt_pub_caps'}),
+    emqx_zone:unset_env(external, '$mqtt_caps'),
+    emqx_zone:unset_env(external, '$mqtt_pub_caps'),
 
     {ok, Client5} = emqtt:start_link([{proto_ver, v5}]),
     {ok, Connack5} = emqtt:connect(Client5),

From 18486a7e13e8d40f3bd3baae1ce5f711b8a030bc Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Nov 2022 16:01:12 +0100
Subject: [PATCH 05/13] ci: run emqx-ct-pipeline without docker dependencies

---
 .github/workflows/run_test_cases.yaml | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index a2b7e6e53..d1578b3c2 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -30,8 +30,9 @@ jobs:
         working-directory: source
         id: run_find_apps
         # emqx_plugin_libs doesn't have a test suite -> excluded from app list
+        # emqx ct is run independently -> exclude it from the app list
         run: |
-          ct_apps="$(./scripts/find-apps.sh --json | jq -c 'del (.[] | select (. == "apps/emqx_plugin_libs"))')"
+          ct_apps="$(./scripts/find-apps.sh --json | jq -c 'del (.[] | select (. == "apps/emqx_plugin_libs" or . ==  "emqx"))')"
           echo "ct-apps: $ct_apps"
           echo "ct_apps=$ct_apps" >> $GITHUB_OUTPUT
       - name: get_all_deps
@@ -73,6 +74,28 @@ jobs:
           name: cover-${{ matrix.task }}
           path: source/_build/test/cover
 
+  emqx_ct:
+    needs: prepare
+    runs-on: ubuntu-20.04
+    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: AutoModality/action-clean@v1
+      - uses: actions/download-artifact@v3
+        with:
+          name: source
+          path: .
+      - name: unzip source code
+        run: unzip -o -q source.zip
+      - name: emqx-ct-pipeline
+        working-directory: source
+        run: make emqx-ct-pipeline
+      - uses: actions/upload-artifact@v3
+        with:
+          name: cover-emqx-ct-pipeline
+          path: source/_build/test/cover
+
   ct:
     needs: prepare
     runs-on: ${{ matrix.runs-on }}
@@ -201,6 +224,7 @@ jobs:
   make_cover:
     needs:
       - eunit_and_proper
+      - emqx_ct
       - ct
     runs-on: ubuntu-20.04
     container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04

From f4621d3c71a31cc3659fa12b1de2ba7b4d86e83c Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Nov 2022 17:25:59 +0100
Subject: [PATCH 06/13] test: connect MQTT client with a retry

---
 test/emqx_takeover_SUITE.erl | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/test/emqx_takeover_SUITE.erl b/test/emqx_takeover_SUITE.erl
index 3317317c5..882967ab3 100644
--- a/test/emqx_takeover_SUITE.erl
+++ b/test/emqx_takeover_SUITE.erl
@@ -32,11 +32,11 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([]),
+    emqx_ct_helpers:start_apps([emqx]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([]),
+    emqx_ct_helpers:stop_apps([emqx]),
     ok.
 %%--------------------------------------------------------------------
 %% Testcases
@@ -45,10 +45,14 @@ t_takeover(_) ->
     process_flag(trap_exit, true),
     AllMsgs = messages(?CNT),
     Pos = rand:uniform(?CNT),
-
     ClientId = <<"clientid">>,
-    {ok, C1} = emqtt:start_link([{clientid, ClientId}, {clean_start, false}]),
-    {ok, _} = emqtt:connect(C1),
+    C1 =
+        with_retry(
+          fun() ->
+              {ok, C} = emqtt:start_link([{clientid, ClientId}, {clean_start, false}]),
+              {ok, _} = emqtt:connect(C),
+              C
+          end, 5),
     emqtt:subscribe(C1, <<"t">>, 1),
 
     spawn(fun() ->
@@ -78,12 +82,19 @@ t_takeover(_) ->
     emqtt:disconnect(C2),
     unload_meck(ClientId).
 
-t_takover_in_cluster(_) ->
-    todo.
-
 %%--------------------------------------------------------------------
 %% Helpers
 
+with_retry(Fun, 1) -> Fun();
+with_retry(Fun, N) when N > 1 ->
+    try
+        Fun()
+    catch
+        _ : _ ->
+            ct:sleep(1000),
+            with_retry(Fun, N - 1)
+    end.
+
 load_meck(ClientId) ->
     meck:new(fake_conn_mod, [non_strict]),
     HookTakeover = fun(Pid, Msg = {takeover, 'begin'}) ->

From a779a58ff94b0aecaab3e926ea6f0baa051ea74d Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Nov 2022 17:27:28 +0100
Subject: [PATCH 07/13] ci: no need to make cover after per app ct run

---
 .github/workflows/run_test_cases.yaml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index d1578b3c2..34b803173 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -196,12 +196,6 @@ jobs:
           done
       - name: run common test
         run: docker exec -i erlang bash -c "make ${{ matrix.app_name }}-ct-pipeline"
-      - name: run cover
-        run: |
-          printenv > .env
-          docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
-          docker exec -i erlang bash -c "make cover"
-          docker exec --env-file .env -i erlang bash -c "make coveralls"
       - name: cat rebar.crashdump
         if: failure()
         working-directory: source

From 16a306f6f897c6ee1b7e128ea2f90c6ef39ce167 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Nov 2022 19:17:53 +0100
Subject: [PATCH 08/13] test: ensure rule engine resource providers are loaded

---
 .../test/emqx_bridge_mqtt_data_export_import_SUITE.erl           | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl
index fc95e3e4e..2edfca790 100644
--- a/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl
+++ b/apps/emqx_management/test/emqx_bridge_mqtt_data_export_import_SUITE.erl
@@ -36,6 +36,7 @@ init_per_suite(Cfg) ->
     emqx_dashboard_admin:mnesia(boot),
     emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_management]),
     application:ensure_all_started(emqx_dashboard),
+    ok = emqx_rule_engine:load_providers(),
     Cfg.
 
 end_per_suite(Cfg) ->

From 7d3c7e1cc36efa08a30a3ac099130fa65af7ef4f Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Nov 2022 20:08:38 +0100
Subject: [PATCH 09/13] test: clear mqtt_admin table after bootstrap test suite

---
 apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl
index 7748c0224..e3894beb1 100644
--- a/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_bootstrap_app_SUITE.erl
@@ -41,6 +41,8 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_) ->
+    ok = application:unset_env(emqx_management, bootstrap_apps_file),
+    _ = mnesia:clear_table(mqtt_app),
     emqx_ct_helpers:stop_apps([]),
     ok.
 

From 0ab5ac95bb9f6b68540242357b0d7d6922a52a3b Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Wed, 2 Nov 2022 21:04:22 +0100
Subject: [PATCH 10/13] test: refactor emqx_takeover_SUITE

---
 test/emqx_shared_sub_SUITE.erl |  2 +-
 test/emqx_takeover_SUITE.erl   | 89 +++++++++++++++++++++++++---------
 2 files changed, 66 insertions(+), 25 deletions(-)

diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl
index 2112f0b8c..897a59cb6 100644
--- a/test/emqx_shared_sub_SUITE.erl
+++ b/test/emqx_shared_sub_SUITE.erl
@@ -51,7 +51,7 @@ init_per_suite(Config) ->
     PortDiscovery = application:get_env(gen_rpc, port_discovery),
     application:set_env(gen_rpc, port_discovery, stateless),
     application:ensure_all_started(gen_rpc),
-    %% ensure emqx_moduels' app modules are loaded
+    %% ensure emqx_modules app modules are loaded
     %% so the mnesia tables are created
     ok = load_app(emqx_modules),
     emqx_ct_helpers:start_apps([]),
diff --git a/test/emqx_takeover_SUITE.erl b/test/emqx_takeover_SUITE.erl
index 882967ab3..c9e84bc1c 100644
--- a/test/emqx_takeover_SUITE.erl
+++ b/test/emqx_takeover_SUITE.erl
@@ -32,29 +32,43 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx]),
+    emqx_ct_helpers:start_apps([]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx]),
+    emqx_ct_helpers:stop_apps([]),
     ok.
+
+init_per_testcase(Case, Config) ->
+    ?MODULE:Case({'init', Config}).
+
+end_per_testcase(Case, Config) ->
+    ?MODULE:Case({'end', Config}).
+
 %%--------------------------------------------------------------------
 %% Testcases
 
-t_takeover(_) ->
-    process_flag(trap_exit, true),
+t_takeover({init, Config}) when is_list(Config) ->
+    Config;
+t_takeover({'end', Config}) when is_list(Config) ->
+    ok;
+t_takeover(Config) when is_list(Config) ->
     AllMsgs = messages(?CNT),
     Pos = rand:uniform(?CNT),
-    ClientId = <<"clientid">>,
+    ClientId = random_clientid(),
+    ClientOpts = [{clientid, ClientId},
+                  {clean_start, false},
+                  {host, "127.0.0.1"},
+                  {port, 1883}
+                 ],
     C1 =
         with_retry(
           fun() ->
-              {ok, C} = emqtt:start_link([{clientid, ClientId}, {clean_start, false}]),
+              {ok, C} = emqtt:start_link(ClientOpts),
               {ok, _} = emqtt:connect(C),
               C
           end, 5),
     emqtt:subscribe(C1, <<"t">>, 1),
-
     spawn(fun() ->
             [begin
                 emqx:publish(lists:nth(I, AllMsgs)),
@@ -63,28 +77,55 @@ t_takeover(_) ->
           end),
     emqtt:pause(C1),
     timer:sleep(?CNT*10),
-
     load_meck(ClientId),
-    spawn(fun() ->
-            [begin
-                emqx:publish(lists:nth(I, AllMsgs)),
-                timer:sleep(rand:uniform(10))
-             end || I <- lists:seq(Pos+1, ?CNT)]
-          end),
-    {ok, C2} = emqtt:start_link([{clientid, ClientId}, {clean_start, false}]),
-    {ok, _} = emqtt:connect(C2),
-
-    Received = all_received_publishs(),
-    ct:pal("middle: ~p, received: ~p", [Pos, [P || {publish, #{payload := P}} <- Received]]),
-    assert_messages_missed(AllMsgs, Received),
-    assert_messages_order(AllMsgs, Received),
-
-    emqtt:disconnect(C2),
-    unload_meck(ClientId).
+    try
+        spawn(fun() ->
+                [begin
+                    emqx:publish(lists:nth(I, AllMsgs)),
+                    timer:sleep(rand:uniform(10))
+                end || I <- lists:seq(Pos+1, ?CNT)]
+            end),
+        {ok, C2} = emqtt:start_link(ClientOpts),
+        %% C1 is going down, unlink it so the test can continue to run
+        _ = monitor(process, C1),
+        ?assert(erlang:is_process_alive(C1)),
+        unlink(C1),
+        {ok, _} = emqtt:connect(C2),
+        receive
+            {'DOWN', _, process, C1, _} ->
+                ok
+        after 1000 ->
+                  ct:fail("timedout_waiting_for_old_connection_shutdown")
+        end,
+        Received = all_received_publishs(),
+        ct:pal("middle: ~p, received: ~p", [Pos, [P || {publish, #{payload := P}} <- Received]]),
+        assert_messages_missed(AllMsgs, Received),
+        assert_messages_order(AllMsgs, Received),
+        kill_process(C2, fun emqtt:stop/1)
+    after
+        unload_meck(ClientId)
+    end.
 
 %%--------------------------------------------------------------------
 %% Helpers
 
+random_clientid() ->
+    iolist_to_binary(["clientid", "-", integer_to_list(erlang:system_time())]).
+
+kill_process(Pid, WithFun) ->
+    _ = unlink(Pid),
+    _ = monitor(process, Pid),
+    try WithFun(Pid)
+    catch _:_ -> ok
+    end,
+    receive
+        {'DOWN', _, process, Pid, _} ->
+            ok
+    after 10_000 ->
+              exit(Pid, kill),
+              error(timeout)
+    end.
+
 with_retry(Fun, 1) -> Fun();
 with_retry(Fun, N) when N > 1 ->
     try

From a73eb29196c0d688caed8aa363f4f8f47e801305 Mon Sep 17 00:00:00 2001
From: JianBo He <heeejianbo@gmail.com>
Date: Fri, 4 Nov 2022 00:41:05 +0800
Subject: [PATCH 11/13] chore: avoid side effects from boot modules

---
 test/emqx_router_SUITE.erl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/emqx_router_SUITE.erl b/test/emqx_router_SUITE.erl
index 5e15f858e..f787d70c1 100644
--- a/test/emqx_router_SUITE.erl
+++ b/test/emqx_router_SUITE.erl
@@ -32,6 +32,7 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
+    emqx_ct_helpers:boot_modules(all),
     emqx_ct_helpers:stop_apps([]).
 
 init_per_testcase(_TestCase, Config) ->

From c2bc6b56ecc9112aaf4f4d0cf49a15852d4a7943 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 3 Nov 2022 20:33:00 +0100
Subject: [PATCH 12/13] test: ensure all modules are loaded for takeover_SUITE

---
 test/emqx_takeover_SUITE.erl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/emqx_takeover_SUITE.erl b/test/emqx_takeover_SUITE.erl
index c9e84bc1c..c7e3c9703 100644
--- a/test/emqx_takeover_SUITE.erl
+++ b/test/emqx_takeover_SUITE.erl
@@ -32,6 +32,7 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
+    emqx_ct_helpers:boot_modules(all),
     emqx_ct_helpers:start_apps([]),
     Config.
 

From b7a04c50ff21e1db3e8e87e48f977e006b1205f8 Mon Sep 17 00:00:00 2001
From: "Zaiming (Stone) Shi" <zmstone@gmail.com>
Date: Thu, 3 Nov 2022 21:49:18 +0100
Subject: [PATCH 13/13] ci: upload all cover data to the same artifact name

all concurrent jobs produces different coverdata file names
so there is clashing
---
 .github/workflows/run_test_cases.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 34b803173..d67ed1395 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -71,8 +71,9 @@ jobs:
         run: make ${{ matrix.task }}
       - uses: actions/upload-artifact@v3
         with:
-          name: cover-${{ matrix.task }}
+          name: cover
           path: source/_build/test/cover
+          if-no-files-found: warn
 
   emqx_ct:
     needs: prepare
@@ -88,13 +89,15 @@ jobs:
           path: .
       - name: unzip source code
         run: unzip -o -q source.zip
+      # produces emqx-emqx.coverdata
       - name: emqx-ct-pipeline
         working-directory: source
         run: make emqx-ct-pipeline
       - uses: actions/upload-artifact@v3
         with:
-          name: cover-emqx-ct-pipeline
+          name: cover
           path: source/_build/test/cover
+          if-no-files-found: warn
 
   ct:
     needs: prepare