From ed2a675404a5607a996a62d94cfcb9aee1681c87 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 14 Apr 2021 18:27:19 +0800 Subject: [PATCH] chore(fvt test): update emqx cluster compose file --- .ci/docker-compose-file/.env | 3 + .../docker-compose-emqx-cluster.yaml} | 58 +++++++--- .../docker-compose-python.yaml | 15 +++ .ci/docker-compose-file/haproxy/haproxy.cfg | 109 ++++++++++++++++++ .../python}/pytest.sh | 14 ++- .github/workflows/run_fvt_tests.yaml | 9 +- 6 files changed, 182 insertions(+), 26 deletions(-) rename .ci/{fvt_tests/docker-compose.yaml => docker-compose-file/docker-compose-emqx-cluster.yaml} (61%) create mode 100644 .ci/docker-compose-file/docker-compose-python.yaml create mode 100644 .ci/docker-compose-file/haproxy/haproxy.cfg rename .ci/{fvt_tests/scripts => docker-compose-file/python}/pytest.sh (68%) diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index d474e2637..8c9d056cc 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -3,3 +3,6 @@ REDIS_TAG=6 MONGO_TAG=4 PGSQL_TAG=13 LDAP_TAG=2.4.50 + +TARGET=emqx/emqx +EMQX_TAG=build-alpine-amd64 diff --git a/.ci/fvt_tests/docker-compose.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml similarity index 61% rename from .ci/fvt_tests/docker-compose.yaml rename to .ci/docker-compose-file/docker-compose-emqx-cluster.yaml index 6f2ad1be2..39e231367 100644 --- a/.ci/fvt_tests/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml @@ -1,14 +1,40 @@ -version: '3' +version: '3.9' services: + haproxy: + container_name: haproxy + image: haproxy:2.3 + depends_on: + - emqx1 + - emqx2 + volumes: + - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg + - ../../etc/certs:/usr/local/etc/haproxy/certs +# ports: +# - "1883:1883" +# - "8883:8883" +# - "8083:8083" +# - "8084:8084" + networks: + - emqx_bridge + working_dir: /usr/local/etc/haproxy + command: + - bash + - -c + - | + cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /usr/local/etc/haproxy/certs/emqx.pem + haproxy -f /usr/local/etc/haproxy/haproxy.cfg + emqx1: container_name: node1.emqx.io - image: ${TARGET}:build-alpine-amd64 + image: $TARGET:$EMQX_TAG environment: - "EMQX_NAME=emqx" - "EMQX_HOST=node1.emqx.io" - "EMQX_CLUSTER__DISCOVERY=static" - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io" + - "EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on" + - "EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on" - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" - "EMQX_MQTT__MAX_TOPIC_ALIAS=10" - "EMQX_LOG__LEVEL=debug" @@ -25,18 +51,20 @@ services: timeout: 25s retries: 5 networks: - emqx-bridge: + emqx_bridge: aliases: - node1.emqx.io emqx2: container_name: node2.emqx.io - image: ${TARGET}:build-alpine-amd64 + image: $TARGET:$EMQX_TAG environment: - "EMQX_NAME=emqx" - "EMQX_HOST=node2.emqx.io" - "EMQX_CLUSTER__DISCOVERY=static" - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io" + - "EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on" + - "EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on" - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s" - "EMQX_MQTT__MAX_TOPIC_ALIAS=10" - "EMQX_LOG__LEVEL=debug" @@ -53,22 +81,16 @@ services: timeout: 25s retries: 5 networks: - emqx-bridge: + emqx_bridge: aliases: - node2.emqx.io - client: - container_name: paho_client - image: python:3.7.2-alpine3.9 - depends_on: - - emqx1 - - emqx2 - tty: true - networks: - emqx-bridge: - volumes: - - ./scripts:/scripts - networks: - emqx-bridge: + emqx_bridge: driver: bridge + name: emqx_bridge + ipam: + driver: default + config: + - subnet: 172.100.239.0/24 + gateway: 172.100.239.1 diff --git a/.ci/docker-compose-file/docker-compose-python.yaml b/.ci/docker-compose-file/docker-compose-python.yaml new file mode 100644 index 000000000..0b9af4517 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-python.yaml @@ -0,0 +1,15 @@ +version: '3.9' + +services: + python: + container_name: python + image: python:3.7.2-alpine3.9 + depends_on: + - emqx1 + - emqx2 + tty: true + networks: + emqx_bridge: + volumes: + - ./python:/scripts + diff --git a/.ci/docker-compose-file/haproxy/haproxy.cfg b/.ci/docker-compose-file/haproxy/haproxy.cfg new file mode 100644 index 000000000..217fb3048 --- /dev/null +++ b/.ci/docker-compose-file/haproxy/haproxy.cfg @@ -0,0 +1,109 @@ +##---------------------------------------------------------------- +## global 2021/04/05 +##---------------------------------------------------------------- +global + log 127.0.0.1:514 local0 notice + # Replace 1024000 with deployment connections + maxconn 1000 + nbproc 1 + nbthread 2 + cpu-map auto:1/1-2 0-1 + tune.ssl.default-dh-param 2048 + ssl-default-bind-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:SEED:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!RSAPSK:!aDH:!aECDH:!EDH-DSS-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!SRP + # Enable the HAProxy Runtime API + stats socket :9999 level admin expose-fd listeners + +##---------------------------------------------------------------- +## defaults +##---------------------------------------------------------------- +defaults + log global + mode tcp + option tcplog + # Replace 1024000 with deployment connections + maxconn 1000 + timeout connect 30000 + timeout client 600s + timeout server 600s + +##---------------------------------------------------------------- +## API +##---------------------------------------------------------------- +frontend emqx_mgmt + mode tcp + option tcplog + bind *:8081 + default_backend emqx_mgmt_back + +frontend emqx_dashboard + mode tcp + option tcplog + bind *:18083 + default_backend emqx_dashboard_back + +backend emqx_mgmt_back + mode http + # balance static-rr + server emqx-1 node1.emqx.io:8081 + server emqx-2 node2.emqx.io:8081 + +backend emqx_dashboard_back + mode http + # balance static-rr + server emqx-1 node1.emqx.io:18083 + server emqx-2 node2.emqx.io:18083 + + +##---------------------------------------------------------------- +## public +##---------------------------------------------------------------- +frontend emqx_tcp + mode tcp + option tcplog + bind *:1883 + default_backend emqx_tcp_back + +frontend emqx_ws + mode tcp + option tcplog + bind *:8083 + default_backend emqx_ws_back + +backend emqx_tcp_back + mode tcp + balance static-rr + server emqx-1 node1.emqx.io:1883 check-send-proxy send-proxy-v2 + server emqx-2 node2.emqx.io:1883 check-send-proxy send-proxy-v2 + +backend emqx_ws_back + mode tcp + balance static-rr + server emqx-1 node1.emqx.io:8083 check-send-proxy send-proxy-v2 + server emqx-2 node2.emqx.io:8083 check-send-proxy send-proxy-v2 + +##---------------------------------------------------------------- +## TLS +##---------------------------------------------------------------- +frontend emqx_ssl + mode tcp + option tcplog + bind *:8883 ssl crt /usr/local/etc/haproxy/certs/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 + default_backend emqx_ssl_back + +frontend emqx_wss + mode tcp + option tcplog + bind *:8084 ssl crt /usr/local/etc/haproxy/certs/emqx.pem ca-file /usr/local/etc/haproxy/certs/cacert.pem verify required no-sslv3 + default_backend emqx_wss_back + +backend emqx_ssl_back + mode tcp + balance static-rr + server emqx-1 node1.emqx.io:1883 check-send-proxy send-proxy-v2-ssl-cn + server emqx-2 node2.emqx.io:1883 check-send-proxy send-proxy-v2-ssl-cn + +backend emqx_wss_back + mode tcp + balance static-rr + server emqx-1 node1.emqx.io:8083 check-send-proxy send-proxy-v2-ssl-cn + server emqx-2 node2.emqx.io:8083 check-send-proxy send-proxy-v2-ssl-cn diff --git a/.ci/fvt_tests/scripts/pytest.sh b/.ci/docker-compose-file/python/pytest.sh similarity index 68% rename from .ci/fvt_tests/scripts/pytest.sh rename to .ci/docker-compose-file/python/pytest.sh index c93c4a769..eacbecc3b 100755 --- a/.ci/fvt_tests/scripts/pytest.sh +++ b/.ci/docker-compose-file/python/pytest.sh @@ -6,17 +6,19 @@ set -x set +e -NODE1="node1.emqx.io" -NODE2="node2.emqx.io" +LB="haproxy" apk update && apk add git curl git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing pip install pytest -pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$NODE1" + +pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB" RESULT=$? -pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "$NODE1" --host2 "$NODE2" -RESULT=$(( RESULT + $? )) -pytest -v /paho.mqtt.testing/interoperability/test_client --host "$NODE1" + +pytest -v /paho.mqtt.testing/interoperability/test_client --host "$LB" RESULT=$(( RESULT + $? )) +# pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "node1.emqx.io" --host2 "node2.emqx.io" +# RESULT=$(( RESULT + $? )) + exit $RESULT diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 815a5f7ae..4a6069720 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -24,8 +24,10 @@ jobs: echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token make deps-emqx-ee echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV else echo "TARGET=emqx/emqx" >> $GITHUB_ENV + echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV fi - name: make emqx image run: make docker @@ -33,7 +35,10 @@ jobs: timeout-minutes: 5 run: | set -e -u -x - docker-compose -f .ci/fvt_tests/docker-compose.yaml up -d + docker-compose \ + -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \ + -f .ci/docker-compose-file/docker-compose-python.yaml \ + up -d while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do if [ $(docker ps -a -f name=fvt_tests_emqx -f status=exited -q | wc -l) -ne 0 ]; then echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:emqx stop"; @@ -45,7 +50,7 @@ jobs: done - name: make paho tests run: | - if ! docker exec -i paho_client /scripts/pytest.sh; then + if ! docker exec -i python /scripts/pytest.sh; then docker logs node1.emqx.io docker logs node2.emqx.io exit 1