test(fvt): extend functional verification tests to use replicant node

This parameterizes the Functional Verification Tests (FVTs) that run
in CI to use a replication log (RLOG) role of "replicant" for one of
the nodes.  With this addition, our FVTs may explore more scenarios
with data replication.

.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml

@@ -0,0 +1,27 @@
+x-default-emqx: &default-emqx
+    image: $TARGET:$EMQX_TAG
+    env_file:
+      - conf.cluster.env
+    healthcheck:
+      test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
+      interval: 5s
+      timeout: 25s
+      retries: 5
+
+services:
+  emqx1:
+    <<: *default-emqx
+    environment:
+      - "EMQX_HOST=node1.emqx.io"
+      - "EMQX_CLUSTER__DB_BACKEND=rlog"
+      - "EMQX_CLUSTER__RLOG__ROLE=core"
+      - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]"
+
+  emqx2:
+    <<: *default-emqx
+    environment:
+      - "EMQX_HOST=node2.emqx.io"
+      - "EMQX_CLUSTER__DB_BACKEND=rlog"
+      - "EMQX_CLUSTER__RLOG__ROLE=replicant"
+      - "EMQX_CLUSTER__RLOG__CORE_NODES=emqx@node1.emqx.io"
+      - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]"

.ci/docker-compose-file/docker-compose-emqx-cluster.yaml

@@ -1,5 +1,15 @@
 version: '3.9'
 
+x-default-emqx: &default-emqx
+    image: $TARGET:$EMQX_TAG
+    env_file:
+      - conf.cluster.env
+    healthcheck:
+      test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
+      interval: 5s
+      timeout: 25s
+      retries: 5
+
 services:
   haproxy:
     container_name: haproxy
@@ -28,34 +38,20 @@ services:
         haproxy -f /usr/local/etc/haproxy/haproxy.cfg
 
   emqx1:
+    <<: *default-emqx
     container_name: node1.emqx.io
-    image: $TARGET:$EMQX_TAG
-    env_file:
-      - conf.cluster.env
     environment:
       - "EMQX_HOST=node1.emqx.io"
-    healthcheck:
-      test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
-      interval: 5s
-      timeout: 25s
-      retries: 5
     networks:
       emqx_bridge:
         aliases:
         - node1.emqx.io
 
   emqx2:
+    <<: *default-emqx
     container_name: node2.emqx.io
-    image: $TARGET:$EMQX_TAG
-    env_file:
-      - conf.cluster.env
     environment:
       - "EMQX_HOST=node2.emqx.io"
-    healthcheck:
-      test: ["CMD", "/opt/emqx/bin/emqx", "ping"]
-      interval: 5s
-      timeout: 25s
-      retries: 5
     networks:
       emqx_bridge:
         aliases:

.ci/docker-compose-file/haproxy/haproxy.cfg

@@ -54,7 +54,6 @@ backend emqx_dashboard_back
     server emqx-1 node1.emqx.io:18083
     server emqx-2 node2.emqx.io:18083
 
-
 ##----------------------------------------------------------------
 ## public
 ##----------------------------------------------------------------

.ci/docker-compose-file/python/pytest.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 ## This script is to run emqx cluster smoke tests (fvt) in github action
-## This script is executed in pacho_client
+## This script is executed in paho_client
 
 set -x
 set +e

.ci/docker-compose-file/scripts/run-emqx.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -euxo pipefail
+
+if [ "$EMQX_TEST_DB_BACKEND" = "rlog" ]
+then
+  CLUSTER_OVERRIDES="-f .ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml"
+else
+  CLUSTER_OVERRIDES=""
+fi
+
+{
+  echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_"
+  echo "EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s"
+  echo "EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10"
+} >> .ci/docker-compose-file/conf.cluster.env
+
+is_cluster_up() {
+  docker exec -i node1.emqx.io \
+         bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1
+}
+
+docker-compose \
+  -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
+  $CLUSTER_OVERRIDES \
+  -f .ci/docker-compose-file/docker-compose-python.yaml \
+  up -d
+
+while ! is_cluster_up; do
+  echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx";
+  sleep 5;
+done

.github/workflows/run_fvt_tests.yaml

@@ -71,6 +71,9 @@ jobs:
         otp:
           - 23.2.7.2-emqx-2
           - 24.1.1-emqx-1
+        cluster_db_backend:
+          - "mnesia"
+          - "rlog"
 
     steps:
     - uses: actions/download-artifact@v2
@@ -91,18 +94,9 @@ jobs:
       timeout-minutes: 5
       working-directory: source
       run: |
-        set -e -u -x
+        set -x
-        echo "HOCON_ENV_OVERRIDE_PREFIX=EMQX_" >> .ci/docker-compose-file/conf.cluster.env
+        export EMQX_TEST_DB_BACKEND="${{ matrix.cluster_db_backend }}"
-        echo "EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s" >> .ci/docker-compose-file/conf.cluster.env
+        ./.ci/docker-compose-file/scripts/run-emqx.sh
-        echo "EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10" >> .ci/docker-compose-file/conf.cluster.env
-        docker-compose \
-            -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
-            -f .ci/docker-compose-file/docker-compose-python.yaml \
-            up -d
-        while ! docker exec -i node1.emqx.io bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1; do
-            echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx";
-            sleep 5;
-        done
     - name: make paho tests
       run: |
         if ! docker exec -i python /scripts/pytest.sh; then