diff --git a/.ci/fvt_tests/docker-compose.yaml b/.ci/fvt_tests/docker-compose.yaml
index 00b5712bc..6f2ad1be2 100644
--- a/.ci/fvt_tests/docker-compose.yaml
+++ b/.ci/fvt_tests/docker-compose.yaml
@@ -11,6 +11,7 @@ services:
     - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io"
     - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s"
     - "EMQX_MQTT__MAX_TOPIC_ALIAS=10"
+    - "EMQX_LOG__LEVEL=debug"
     command:
         - /bin/sh
         - -c
@@ -38,6 +39,7 @@ services:
     - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io"
     - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s"
     - "EMQX_MQTT__MAX_TOPIC_ALIAS=10"
+    - "EMQX_LOG__LEVEL=debug"
     command:
       - /bin/sh
       - -c
@@ -64,6 +66,8 @@ services:
     tty: true
     networks:
         emqx-bridge:
+    volumes:
+      - ./scripts:/scripts
 
 networks:
   emqx-bridge:
diff --git a/.ci/fvt_tests/scripts/pytest.sh b/.ci/fvt_tests/scripts/pytest.sh
new file mode 100755
index 000000000..c93c4a769
--- /dev/null
+++ b/.ci/fvt_tests/scripts/pytest.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+## This script is to run emqx cluster smoke tests (fvt) in github action
+## This script is executed in pacho_client
+
+set -x
+set +e
+
+NODE1="node1.emqx.io"
+NODE2="node2.emqx.io"
+
+apk update && apk add git curl
+git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing
+pip install pytest
+pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$NODE1"
+RESULT=$?
+pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "$NODE1" --host2 "$NODE2"
+RESULT=$(( RESULT + $? ))
+pytest -v /paho.mqtt.testing/interoperability/test_client --host "$NODE1"
+RESULT=$(( RESULT + $? ))
+
+exit $RESULT
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 6aa1e6844..e945f1c2d 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -44,12 +44,10 @@ jobs:
             done
         - name: make paho tests
           run: |
-            docker exec -i paho_client sh -c "apk update && apk add git curl \
-              && git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing \
-              && pip install pytest \
-              && pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host node1.emqx.io \
-              && pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 node1.emqx.io --host2 node2.emqx.io \
-              && pytest -v /paho.mqtt.testing/interoperability/test_client --host node1.emqx.io"
+            if ! docker exec -i paho_client /scripts/pytest.sh; then
+              docker logs node1.emqx.io
+              docker logs node2.emqx.io
+            fi
 
     helm_test:
         runs-on: ubuntu-20.04
@@ -138,7 +136,14 @@ jobs:
             emqx2=$(kubectl get pods emqx-2 -o jsonpath='{.status.podIP}')
 
             pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host $emqx_svc
+            RESULT=$?
             pytest -v paho.mqtt.testing/interoperability/test_cluster --host1 $emqx1 --host2 $emqx2
+            RESULT=$((RESULT + $?))
+            if [ 0 -ne $RESULT ]; then
+                kubectl logs emqx-1
+                kubectl logs emqx-2
+            fi
+            exit $RESULT
 
     relup_test:
         runs-on: ubuntu-20.04