chore(fvt test): update emqx cluster compose file

.ci/docker-compose-file/.env

@@ -3,3 +3,6 @@ REDIS_TAG=6
 MONGO_TAG=4
 PGSQL_TAG=13
 LDAP_TAG=2.4.50
+
+TARGET=emqx/emqx
+EMQX_TAG=build-alpine-amd64

.ci/docker-compose-file/docker-compose-emqx-cluster.yaml

@@ -1,14 +1,40 @@
-version: '3'
+version: '3.9'
 
 services:
+  haproxy:
+    container_name: haproxy
+    image: haproxy:2.3
+    depends_on:
+      - emqx1
+      - emqx2
+    volumes:
+      - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+      - ../../etc/certs:/usr/local/etc/haproxy/certs
+#   ports:
+#     - "1883:1883"
+#     - "8883:8883"
+#     - "8083:8083"
+#     - "8084:8084"
+    networks:
+      - emqx_bridge
+    working_dir: /usr/local/etc/haproxy
+    command:
+      - bash
+      - -c
+      - |
+        cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /usr/local/etc/haproxy/certs/emqx.pem
+        haproxy -f /usr/local/etc/haproxy/haproxy.cfg
+
   emqx1:
     container_name: node1.emqx.io
-    image: ${TARGET}:build-alpine-amd64
+    image: $TARGET:$EMQX_TAG
     environment:
     - "EMQX_NAME=emqx"
     - "EMQX_HOST=node1.emqx.io"
     - "EMQX_CLUSTER__DISCOVERY=static"
     - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io"
+    - "EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on"
+    - "EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on"
     - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s"
     - "EMQX_MQTT__MAX_TOPIC_ALIAS=10"
     - "EMQX_LOG__LEVEL=debug"
@@ -25,18 +51,20 @@ services:
       timeout: 25s
       retries: 5
     networks:
-      emqx-bridge:
+      emqx_bridge:
         aliases:
         - node1.emqx.io
 
   emqx2:
     container_name: node2.emqx.io
-    image: ${TARGET}:build-alpine-amd64
+    image: $TARGET:$EMQX_TAG
     environment:
     - "EMQX_NAME=emqx"
     - "EMQX_HOST=node2.emqx.io"
     - "EMQX_CLUSTER__DISCOVERY=static"
     - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io"
+    - "EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on"
+    - "EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on"
     - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s"
     - "EMQX_MQTT__MAX_TOPIC_ALIAS=10"
     - "EMQX_LOG__LEVEL=debug"
@@ -53,22 +81,16 @@ services:
       timeout: 25s
       retries: 5
     networks:
-      emqx-bridge:
+      emqx_bridge:
         aliases:
         - node2.emqx.io
 
-  client:
-    container_name: paho_client
-    image: python:3.7.2-alpine3.9
-    depends_on:
-      - emqx1
-      - emqx2
-    tty: true
 networks:
-        emqx-bridge:
+  emqx_bridge:
-    volumes:
-      - ./scripts:/scripts
-
-networks:
-  emqx-bridge:
     driver: bridge
+    name: emqx_bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.100.239.0/24
+          gateway: 172.100.239.1

.ci/docker-compose-file/docker-compose-python.yaml

@@ -0,0 +1,15 @@
+version: '3.9'
+
+services:
+  python:
+    container_name: python 
+    image: python:3.7.2-alpine3.9
+    depends_on:
+      - emqx1
+      - emqx2
+    tty: true
+    networks:
+        emqx_bridge:
+    volumes:
+      - ./python:/scripts
+

.ci/docker-compose-file/haproxy/haproxy.cfg

@@ -0,0 +1,109 @@
+##----------------------------------------------------------------
+## global 2021/04/05
+##----------------------------------------------------------------
+global
+    log 127.0.0.1:514 local0 notice
+    # Replace 1024000 with deployment connections
+    maxconn 1000
+    nbproc 1
+    nbthread 2
+    cpu-map auto:1/1-2 0-1
+    tune.ssl.default-dh-param 2048
+    ssl-default-bind-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:SEED:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!RSAPSK:!aDH:!aECDH:!EDH-DSS-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!SRP
+    # Enable the HAProxy Runtime API
+    stats socket :9999 level admin expose-fd listeners
+
+##----------------------------------------------------------------
+## defaults
+##----------------------------------------------------------------
+defaults
+    log global
+    mode tcp
+    option tcplog
+    # Replace 1024000 with deployment connections
+    maxconn 1000
+    timeout connect 30000
+    timeout client 600s
+    timeout server 600s
+
+##----------------------------------------------------------------
+## API
+##----------------------------------------------------------------
+frontend emqx_mgmt
+   mode tcp
+   option tcplog
+   bind *:8081
+   default_backend emqx_mgmt_back
+
+frontend emqx_dashboard
+   mode tcp
+   option tcplog
+   bind *:18083
+   default_backend emqx_dashboard_back
+
+backend emqx_mgmt_back
+    mode http
+    # balance static-rr
+    server emqx-1 node1.emqx.io:8081
+    server emqx-2 node2.emqx.io:8081
+
+backend emqx_dashboard_back
+    mode http
+    # balance static-rr
+    server emqx-1 node1.emqx.io:18083
+    server emqx-2 node2.emqx.io:18083
+
+
+##----------------------------------------------------------------
+## public
+##----------------------------------------------------------------
+frontend emqx_tcp
+   mode tcp
+   option tcplog
+   bind *:1883
+   default_backend emqx_tcp_back
+
+frontend emqx_ws
+   mode tcp
+   option tcplog
+   bind *:8083
+   default_backend emqx_ws_back
+
+backend emqx_tcp_back
+    mode tcp
+    balance static-rr
+    server emqx-1 node1.emqx.io:1883 check-send-proxy send-proxy-v2
+    server emqx-2 node2.emqx.io:1883 check-send-proxy send-proxy-v2
+
+backend emqx_ws_back
+    mode tcp
+    balance static-rr
+    server emqx-1 node1.emqx.io:8083 check-send-proxy send-proxy-v2
+    server emqx-2 node2.emqx.io:8083 check-send-proxy send-proxy-v2
+
+##----------------------------------------------------------------
+## TLS
+##----------------------------------------------------------------
+frontend emqx_ssl
+    mode tcp
+    option tcplog
+    bind *:8883 ssl crt /usr/local/etc/haproxy/certs/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3
+    default_backend emqx_ssl_back
+
+frontend emqx_wss
+    mode tcp
+    option tcplog
+    bind *:8084 ssl crt /usr/local/etc/haproxy/certs/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3
+    default_backend emqx_wss_back
+
+backend emqx_ssl_back
+    mode tcp
+    balance static-rr
+    server emqx-1 node1.emqx.io:1883 check-send-proxy send-proxy-v2-ssl-cn 
+    server emqx-2 node2.emqx.io:1883 check-send-proxy send-proxy-v2-ssl-cn
+
+backend emqx_wss_back
+    mode tcp
+    balance static-rr
+    server emqx-1 node1.emqx.io:8083 check-send-proxy send-proxy-v2-ssl-cn
+    server emqx-2 node2.emqx.io:8083 check-send-proxy send-proxy-v2-ssl-cn

.ci/docker-compose-file/python/pytest.sh

@@ -6,17 +6,19 @@
 set -x
 set +e
 
-NODE1="node1.emqx.io"
+LB="haproxy"
-NODE2="node2.emqx.io"
 
 apk update && apk add git curl
 git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing
 pip install pytest
-pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$NODE1"
+
+pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB"
 RESULT=$?
-pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "$NODE1" --host2 "$NODE2"
+
-RESULT=$(( RESULT + $? ))
+pytest -v /paho.mqtt.testing/interoperability/test_client --host "$LB"
-pytest -v /paho.mqtt.testing/interoperability/test_client --host "$NODE1"
 RESULT=$(( RESULT + $? ))
 
+# pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "node1.emqx.io" --host2 "node2.emqx.io"
+# RESULT=$(( RESULT + $? ))
+
 exit $RESULT

.github/workflows/run_fvt_tests.yaml

@@ -24,8 +24,10 @@ jobs:
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             fi
         - name: make emqx image
           run: make docker
@@ -33,7 +35,10 @@ jobs:
           timeout-minutes: 5
           run: |
             set -e -u -x
-            docker-compose -f .ci/fvt_tests/docker-compose.yaml up -d
+            docker-compose \
+                -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
+                -f .ci/docker-compose-file/docker-compose-python.yaml \
+                up -d
             while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
               if [ $(docker ps -a -f name=fvt_tests_emqx -f status=exited -q | wc -l) -ne 0 ]; then
                   echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:emqx stop";
@@ -45,7 +50,7 @@ jobs:
             done
         - name: make paho tests
           run: |
-            if ! docker exec -i paho_client /scripts/pytest.sh; then
+            if ! docker exec -i python /scripts/pytest.sh; then
               docker logs node1.emqx.io
               docker logs node2.emqx.io
               exit 1