chore(autotest): start emqx

.github/workflows/run_broker_performance_cluster.yaml

@@ -0,0 +1,587 @@
+name: Cluster Performance Test Suite
+
+concurrency:
+  group: slim-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+#  schedule:
+#    - cron: '0 13 * * 1-5'
+#  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        profile:
+        - emqx
+        otp:
+        - 24.1.5-4
+        elixir:
+        - 1.13.2
+        os:
+        - ubuntu20.04
+
+    container: "ghcr.io/emqx/emqx-builder/5.0-5:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: prepare
+      run: |
+        echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
+        echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
+        echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV
+        echo "EMQX_ELIXIRPKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh ${{ matrix.profile }})-elixir${{ matrix.elixir }}-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV
+    - name: Get deps git refs for cache
+      id: deps-refs
+      run: |
+        scripts/get-dep-refs.sh
+        make clean-all
+    - name: load rocksdb cache
+      uses: actions/cache@v2
+      with:
+        path: |
+          _build/default/lib/rocksdb/
+          deps/rocksdb/
+        key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
+    - name: load quicer cache
+      uses: actions/cache@v2
+      with:
+        path: |
+          _build/default/lib/quicer/
+          deps/quicer/
+        key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
+    - name: build and test tgz package
+      run: |
+        make ${EMQX_NAME}-tgz
+        .ci/build_packages/tests.sh "$EMQX_PKG_NAME" tgz amd64
+    - name: run static checks
+      if: contains(matrix.os, 'ubuntu')
+      run: |
+        make static_checks
+    - name: build and test deb/rpm packages
+      run: |
+        make ${EMQX_NAME}-pkg
+        .ci/build_packages/tests.sh "$EMQX_PKG_NAME" pkg amd64
+    - name: build and test tgz package (Elixir)
+      run: |
+        make ${EMQX_NAME}-elixir-tgz
+        .ci/build_packages/tests.sh "$EMQX_ELIXIRPKG_NAME" tgz amd64
+    - name: build and test deb/rpm packages (Elixir)
+      run: |
+        make ${EMQX_NAME}-elixirpkg
+        .ci/build_packages/tests.sh "$EMQX_ELIXIRPKG_NAME" pkg amd64
+    - uses: actions/upload-artifact@v2
+      with:
+        name: ${{ matrix.os }}
+        path: _packages/${{ matrix.profile}}/*.tar.gz
+
+  terraform_emqx:
+    runs-on: ubuntu-latest
+
+    needs: [build]
+    outputs:
+      emqx_public_ip: ${{ steps.emqx_private_ip.outputs.emqx_public_ip }}
+      emqx_node_ip1: ${{ steps.emqx_private_ip.outputs.emqx_node_ip1 }}
+      emqx_node_ip2: ${{ steps.emqx_private_ip.outputs.emqx_node_ip2 }}
+      emqx_node_ip3: ${{ steps.emqx_private_ip.outputs.emqx_node_ip3 }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: ubuntu20.04
+        path: /tmp
+    - name: Download emqx package
+      run: |
+        rm -rf /tmp/emqx-5.*-elixir1.*.tar.gz
+        mv /tmp/emqx-5.*.tar.gz /tmp/emqx.tar.gz
+    - name: Checkout tf-test-automation
+      uses: actions/checkout@v2
+      with:
+        repository: emqx/tf-test-automation
+        ref: emqx-broker
+        path: tf-test-automation
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v1.3.2
+      with:
+        terraform_wrapper: false
+    - name: Terraform Init tf-test-automation
+      working-directory: ./tf-test-automation/services/emqx
+      id: init1
+      run: |
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+    - name: Terraform Validate tf-test-automation
+      working-directory: ./tf-test-automation/services/emqx
+      id: validate1
+      run: terraform validate -no-color
+    - name: Terraform Apply tf-test-automation
+      working-directory: ./tf-test-automation/services/emqx
+      id: server_ip
+      run: |
+        terraform apply -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}" -var="emqx_package=/tmp/emqx.tar.gz"
+        echo "::set-output name=emqx_private_ips::$(terraform output emqx_private_ips)"
+        echo "::set-output name=emqx_public_ips::$(terraform output emqx_public_ips)"
+    - name: Rename and upload emqx tfstate
+      working-directory: ./tf-test-automation/services/emqx
+      run: |
+        mv ./terraform.tfstate ./emqx.tfstate
+    - uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: tfstate
+        path: ./tf-test-automation/services/emqx/emqx.tfstate
+    - name: Show emqx_private_ips
+      id: emqx_private_ip
+      run: |
+        ip1=`echo "${{ steps.server_ip.outputs.emqx_private_ips }}"|awk -F ',' '{print $1}'`
+        ip2=`echo "${{ steps.server_ip.outputs.emqx_private_ips }}"|awk -F ',' '{print $2}'`
+        ip3=`echo "${{ steps.server_ip.outputs.emqx_private_ips }}"|awk -F ',' '{print $3}'`
+        emqx_public_ip=`echo "${{ steps.server_ip.outputs.emqx_public_ips }}"|awk -F ',' '{print $1}'`
+        echo "::set-output name=emqx_node_ip1::$ip1"
+        echo "::set-output name=emqx_node_ip2::$ip2"
+        echo "::set-output name=emqx_node_ip3::$ip3"
+        echo "::set-output name=emqx_public_ip::$emqx_public_ip"
+    - name: Show emqx node ip
+      run: |
+        echo ${{ steps.emqx_private_ip.outputs.emqx_node_ip1 }}
+        echo ${{ steps.emqx_private_ip.outputs.emqx_node_ip2 }}
+        echo ${{ steps.emqx_private_ip.outputs.emqx_node_ip3 }}
+        echo ${{ steps.emqx_private_ip.outputs.emqx_public_ip }}
+
+#  xmeter_start:
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - name: Configure AWS Credentials
+#      uses: aws-actions/configure-aws-credentials@v1
+#      with:
+#        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}
+#        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}
+#        aws-region: ap-southeast-1
+#    - name: Start Xmeter Services
+#      run: |
+#        aws ec2 start-instances --instance-ids i-0dd6d99916baaa1a8
+#        aws ec2 start-instances --instance-ids i-05222103df01eb2d7
+#        sleep 40;
+#        aws ec2 start-instances --instance-ids i-0c5dccd394ed9be18
+#        aws ec2 start-instances --instance-ids i-04c1c50b3e4952266
+#
+#  terraform_mysql:
+#    runs-on: ubuntu-latest
+#
+#    outputs:
+#      mysql_ip: ${{ steps.mysql_ip.outputs.mysql_ip }}
+#      mysql_url: ${{ steps.mysql_url.outputs.mysql_url }}
+#    steps:
+#    - name: Checkout tf-test-automation
+#      uses: actions/checkout@v2
+#      with:
+#        repository: emqx/tf-test-automation
+#        ref: emqx-broker
+#        path: tf-test-automation
+#    - name: Setup Terraform
+#      uses: hashicorp/setup-terraform@v1.3.2
+#      with:
+#        terraform_wrapper: false
+#    - name: Terraform Init tf-test-automation
+#      working-directory: ./tf-test-automation/services/mysql
+#      id: init1
+#      run: |
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Validate tf-test-automation
+#      working-directory: ./tf-test-automation/services/mysql
+#      id: validate1
+#      run: terraform validate -no-color
+#    - name: Terraform Apply tf-test-automation
+#      working-directory: ./tf-test-automation/services/mysql
+#      id: mysql_url
+#      run: |
+#        terraform apply -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#        echo "::set-output name=mysql_url::$(terraform output endpoint)"
+#    - name: Rename and upload mysql tfstate
+#      working-directory: ./tf-test-automation/services/mysql
+#      run: |
+#        mv ./terraform.tfstate ./mysql.tfstate
+#    - uses: actions/upload-artifact@v2
+#      if: always()
+#      with:
+#        name: tfstate
+#        path: ./tf-test-automation/services/mysql/mysql.tfstate
+#    - name: Show mysql url
+#      id: mysql_ip
+#      run: |
+#        ip=`echo "${{ steps.mysql_url.outputs.mysql_url }}"|awk -F ':' '{print $1}'`
+#        echo "::set-output name=mysql_ip::$ip"
+#    - name: Show mysql ip
+#      run: |
+#        echo ${{ steps.mysql_url.outputs.mysql_url }}
+#        echo ${{ steps.mysql_ip.outputs.mysql_ip }}
+#
+#  terraform_redis:
+#    runs-on: ubuntu-latest
+#
+#    outputs:
+#      redis_private_ip: ${{ steps.redis_ip.outputs.redis_private_ips }}
+#      redis_public_ip: ${{ steps.redis_ip.outputs.redis_public_ips }}
+#    steps:
+#    - name: Checkout tf-test-automation
+#      uses: actions/checkout@v2
+#      with:
+#        repository: emqx/tf-test-automation
+#        ref: emqx-broker
+#        path: tf-test-automation
+#    - name: Setup Terraform
+#      uses: hashicorp/setup-terraform@v1.3.2
+#      with:
+#        terraform_wrapper: false
+#    - name: Terraform Init tf-test-automation
+#      working-directory: ./tf-test-automation/services/redis
+#      id: init
+#      run: |
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Validate tf-test-automation
+#      working-directory: ./tf-test-automation/services/redis
+#      id: validate
+#      run: terraform validate -no-color
+#    - name: Terraform Apply tf-test-automation
+#      working-directory: ./tf-test-automation/services/redis
+#      id: redis_ip
+#      run: |
+#        terraform apply -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
+#        echo "::set-output name=redis_private_ip::$(terraform output redis_private_ips)"
+#        echo "::set-output name=redis_public_ip::$(terraform output redis_public_ips)"
+#    - name: Rename and upload redis server tfstate
+#      working-directory: ./tf-test-automation/services/redis
+#      run: |
+#        mv ./terraform.tfstate ./redis.tfstate
+#    - uses: actions/upload-artifact@v2
+#      if: always()
+#      with:
+#        name: tfstate
+#        path: ./tf-test-automation/services/redis/redis.tfstate
+#    - name: Show redis ip
+#      run: |
+#        echo ${{ steps.redis_ip.outputs.redis_private_ip }}
+#        echo ${{ steps.redis_ip.outputs.redis_public_ip }}
+#
+#  terraform_pgsql:
+#    runs-on: ubuntu-latest
+#
+#    outputs:
+#      pgsql_url: ${{ steps.pgsql_url.outputs.pgsql_url }}
+#      pgsql_ip: ${{ steps.pgsql_ip.outputs.pgsql_ip }}
+#    steps:
+#    - name: Checkout tf-test-automation
+#      uses: actions/checkout@v2
+#      with:
+#        repository: emqx/tf-test-automation
+#        ref: emqx-broker
+#        path: tf-test-automation
+#    - name: Setup Terraform
+#      uses: hashicorp/setup-terraform@v1.3.2
+#      with:
+#        terraform_wrapper: false
+#    - name: Terraform Init tf-test-automation
+#      working-directory: ./tf-test-automation/services/pgsql
+#      id: init
+#      run: |
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Validate tf-test-automation
+#      working-directory: ./tf-test-automation/services/pgsql
+#      id: validate
+#      run: terraform validate -no-color
+#    - name: Terraform Apply tf-test-automation
+#      working-directory: ./tf-test-automation/services/pgsql
+#      id: pgsql_url
+#      run: |
+#        terraform apply -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#        echo "::set-output name=pgsql_url::$(terraform output endpoint)"
+#    - name: Rename and upload pgsql server tfstate
+#      working-directory: ./tf-test-automation/services/pgsql
+#      run: |
+#        mv ./terraform.tfstate ./pgsql.tfstate
+#    - uses: actions/upload-artifact@v2
+#      if: always()
+#      with:
+#        name: tfstate
+#        path: ./tf-test-automation/services/pgsql/pgsql.tfstate
+#    - name: Show pgsql url
+#      id: pgsql_ip
+#      run: |
+#        ip=`echo "${{ steps.pgsql_url.outputs.pgsql_url }}"|awk -F ':' '{print $1}'`
+#        echo "::set-output name=pgsql_ip::$ip"
+#    - name: Show pgsql ip
+#      run: |
+#        echo ${{ steps.pgsql_url.outputs.pgsql_url }}
+#        echo ${{ steps.pgsql_ip.outputs.pgsql_ip }}
+#
+#  terraform_mongo:
+#    runs-on: ubuntu-latest
+#
+#    outputs:
+#      mongo_private_ip: ${{ steps.mongo_ip.outputs.mongo_private_ip }}
+#      mongo_public_ip: ${{ steps.mongo_ip.outputs.mongo_public_ip }}
+#    steps:
+#    - name: Checkout tf-test-automation
+#      uses: actions/checkout@v2
+#      with:
+#        repository: emqx/tf-test-automation
+#        ref: emqx-broker
+#        path: tf-test-automation
+#    - name: Setup Terraform
+#      uses: hashicorp/setup-terraform@v1.3.2
+#      with:
+#        terraform_wrapper: false
+#    - name: Terraform Init tf-test-automation
+#      working-directory: ./tf-test-automation/services/mongodb
+#      id: init
+#      run: |
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Validate tf-test-automation
+#      working-directory: ./tf-test-automation/services/mongodb
+#      id: validate
+#      run: terraform validate -no-color
+#    - name: Terraform Apply tf-test-automation
+#      working-directory: ./tf-test-automation/services/mongodb
+#      id: mongo_ip
+#      run: |
+#        terraform apply -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
+#        echo "::set-output name=mongo_private_ip::$(terraform output mongo_private_ips)"
+#        echo "::set-output name=mongo_public_ip::$(terraform output mongo_public_ips)"
+#    - name: Rename and upload mongo server tfstate
+#      working-directory: ./tf-test-automation/services/mongodb
+#      run: |
+#        mv ./terraform.tfstate ./mongo.tfstate
+#    - uses: actions/upload-artifact@v2
+#      if: always()
+#      with:
+#        name: tfstate
+#        path: ./tf-test-automation/services/mongodb/mongo.tfstate
+#    - name: Show mongo ip
+#      run: |
+#        echo ${{ steps.mongo_ip.outputs.mongo_private_ip }}
+#        echo ${{ steps.mongo_ip.outputs.mongo_public_ip }}
+#
+#  terraform_webhook:
+#    runs-on: ubuntu-latest
+#
+#    outputs:
+#      webhook_ip: ${{ steps.webhook_ip.outputs.webhook_ip }}
+#    steps:
+#    - name: Checkout tf-test-automation
+#      uses: actions/checkout@v2
+#      with:
+#        repository: emqx/tf-test-automation
+#        ref: emqx-broker
+#        path: tf-test-automation
+#    - name: Setup Terraform
+#      uses: hashicorp/setup-terraform@v1.3.2
+#      with:
+#        terraform_wrapper: false
+#    - name: Terraform Init tf-test-automation
+#      working-directory: ./tf-test-automation/services/webhook
+#      id: init
+#      run: |
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Validate tf-test-automation
+#      working-directory: ./tf-test-automation/services/webhook
+#      id: validate
+#      run: terraform validate -no-color
+#    - name: Terraform Apply tf-test-automation
+#      working-directory: ./tf-test-automation/services/webhook
+#      id: webhook_ip
+#      run: |
+#        terraform apply -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
+#        echo "::set-output name=webhook_ip::$(terraform output webhook_private_ips)"
+#    - name: Rename and upload webhook server tfstate
+#      working-directory: ./tf-test-automation/services/webhook
+#      run: |
+#        mv ./terraform.tfstate ./webhook.tfstate
+#    - uses: actions/upload-artifact@v2
+#      if: always()
+#      with:
+#        name: tfstate
+#        path: ./tf-test-automation/services/webhook/webhook.tfstate
+#    - name: Show webhook ip
+#      run: |
+#        echo ${{ steps.webhook_ip.outputs.webhook_ip }}
+#
+#  master_control:
+#    runs-on: ubuntu-latest
+#
+#    needs: [xmeter_start, terraform_emqx, terraform_mysql, terraform_redis, terraform_pgsql, terraform_mongo, terraform_webhook]
+#    steps:
+#    - uses: actions/checkout@v2
+#    - uses: actions/setup-java@v1
+#      with:
+#        java-version: '8.0.282' # The JDK version to make available on the path.
+#        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+#        architecture: x64 # (x64 or x86) - defaults to x64
+#    - name: Checkout emqx-fvt
+#      uses: actions/checkout@v2
+#      with:
+#        repository: emqx/emqx-fvt
+#        ref:  broker_performance_test
+#        path: emqx-fvt
+#    - name: install jmeter
+#      timeout-minutes: 10
+#      env:
+#         JMETER_VERSION: 5.3
+#      run: |
+#        wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz https://downloads.apache.org/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
+#        cd /tmp && tar -xvf apache-jmeter.tgz
+#        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+#        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+#        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar
+#        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+#    - name: install jmeter plugin
+#      run: |
+#        wget --no-verbose -O "/opt/jmeter/lib/mysql-connector-java-8.0.16.jar" https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
+#        wget --no-verbose -O "/opt/jmeter/lib/jedis-3.6.0.jar" https://repo1.maven.org/maven2/redis/clients/jedis/3.6.0/jedis-3.6.0.jar
+#        wget --no-verbose -O "/opt/jmeter/lib/postgresql-42.2.18.jar" https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar
+#        wget --no-verbose -O "/opt/jmeter/lib/java-jwt-3.16.0.jar" https://repo1.maven.org/maven2/com/auth0/java-jwt/3.16.0/java-jwt-3.16.0.jar
+#        wget --no-verbose -O "/opt/jmeter/lib/gmongo-1.5.jar" https://repo1.maven.org/maven2/com/gmongo/gmongo/1.5/gmongo-1.5.jar
+#        wget --no-verbose -O "/opt/jmeter/lib/mongo-java-driver-3.12.8.jar" https://repo1.maven.org/maven2/org/mongodb/mongo-java-driver/3.12.8/mongo-java-driver-3.12.8.jar
+#        rm -rf /opt/jmeter/lib/mongo-java-driver-2.11.3.jar
+#    - name: run jmeter
+#      run: |
+#        /opt/jmeter/bin/jmeter.sh \
+#          -Jjmeter.save.saveservice.output_format=xml -n \
+#          -t ./emqx-fvt/broker-performance-test-suite/broker_attestation_test.jmx \
+#          -Demqx_ip=${{ needs.terraform_emqx.outputs.emqx_public_ip }} \
+#          -Demqx_private_ip1=${{ needs.terraform_emqx.outputs.emqx_node_ip1 }} \
+#          -Demqx_private_ip2=${{ needs.terraform_emqx.outputs.emqx_node_ip2 }} \
+#          -Demqx_private_ip3=${{ needs.terraform_emqx.outputs.emqx_node_ip3 }} \
+#          -Dmysql_ip=${{ needs.terraform_mysql.outputs.mysql_ip }} \
+#          -Dpgsql_ip=${{ needs.terraform_pgsql.outputs.pgsql_ip }} \
+#          -Dredis_private_ip=${{ needs.terraform_redis.outputs.redis_private_ip }} \
+#          -Dredis_public_ip=${{ needs.terraform_redis.outputs.redis_public_ip }} \
+#          -Dmongo_private_ip=${{ needs.terraform_mongo.outputs.mongo_private_ip }} \
+#          -Dmongo_public_ip=${{ needs.terraform_mongo.outputs.mongo_public_ip }} \
+#          -Dhttp_ip=${{ needs.terraform_webhook.outputs.webhook_ip }} \
+#          -Dxmeter_ip=${{ secrets.XMETER_IP }} \
+#          -Dplugins_path="/opt/jmeter/lib/ext" \
+#          -Dxmeter_user=${{ secrets.XMETER_USER }} \
+#          -Dxmeter_pwd=${{ secrets.XMETER_PWD }} \
+#          -Dscripts_path="./emqx-fvt/broker-performance-test-suite" \
+#          -Dreport_file="jmeter_logs/report.txt" \
+#          -l jmeter_logs/xmeter_process.jtl \
+#          -j jmeter_logs/xmeter_process.log
+#    - name: check logs
+#      run: |
+#        if cat jmeter_logs/xmeter_process.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+#          echo "check logs filed"
+#        fi
+#    - name: check report
+#      run: |
+#        if cat jmeter_logs/report.txt | grep -e 'err_report_id' > /dev/null 2>&1; then
+#          echo "check report filed"
+#          exit 1
+#        fi
+#    - uses: actions/upload-artifact@v2
+#      if: always()
+#      with:
+#        name: jmeter_logs_report
+#        path: ./jmeter_logs
+#
+#  terraform_destroy:
+#    runs-on: ubuntu-latest
+#
+#    needs: [master_control]
+#    steps:
+#    - name: Checkout tf-test-automation
+#      uses: actions/checkout@v2
+#      with:
+#        repository: emqx/tf-test-automation
+#        ref: cluster
+#        path: tf-test-automation
+#    - name: Setup Terraform
+#      uses: hashicorp/setup-terraform@v1.3.2
+#      with:
+#        terraform_wrapper: false
+#    - uses: actions/download-artifact@v2
+#      with:
+#        name: tfstate
+#        path: ./tf-test-automation/services
+#    - name: Terraform Init mysql
+#      working-directory: ./tf-test-automation/services
+#      run: |
+#        cp mysql.tfstate ./mysql/terraform.tfstate
+#        cd ./mysql
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Destroy mysql
+#      working-directory: ./tf-test-automation/services/mysql
+#      run: |
+#        terraform destroy -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Init emqx
+#      working-directory: ./tf-test-automation/services
+#      run: |
+#        cp emqx.tfstate ./emqx/terraform.tfstate
+#        cd ./emqx
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Destroy emqx
+#      working-directory: ./tf-test-automation/services/emqx
+#      run: |
+#        terraform destroy -auto-approve -var="region=ap-southeast-1" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Init redis
+#      working-directory: ./tf-test-automation/services
+#      run: |
+#        cp redis.tfstate ./redis/terraform.tfstate
+#        cd ./redis
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Destroy redis
+#      working-directory: ./tf-test-automation/services/redis
+#      run: |
+#        terraform destroy -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
+#    - name: Terraform Init pgsql
+#      working-directory: ./tf-test-automation/services
+#      run: |
+#        cp pgsql.tfstate ./pgsql/terraform.tfstate
+#        cd ./pgsql
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Destroy pgsql
+#      working-directory: ./tf-test-automation/services/pgsql
+#      run: |
+#        terraform destroy -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Init mongo
+#      working-directory: ./tf-test-automation/services
+#      run: |
+#        cp mongo.tfstate ./mongodb/terraform.tfstate
+#        cd ./mongodb
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Destroy mongo
+#      working-directory: ./tf-test-automation/services/mongodb
+#      run: |
+#        terraform destroy -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Init Webhook
+#      working-directory: ./tf-test-automation/services
+#      run: |
+#        cp webhook.tfstate ./webhook/terraform.tfstate
+#        cd ./webhook
+#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#    - name: Terraform Destroy Webhook
+#      working-directory: ./tf-test-automation/services/webhook
+#      run: |
+#        terraform destroy -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+#
+#  xmeter_stop:
+#    runs-on: ubuntu-latest
+#
+#    needs: [terraform_destroy]
+#    steps:
+#    - name: Configure AWS Credentials
+#      uses: aws-actions/configure-aws-credentials@v1
+#      with:
+#        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}
+#        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}
+#        aws-region: ap-southeast-1
+#    - name: Start Xmeter Services
+#      run: |
+#        aws ec2 stop-instances --instance-ids i-0dd6d99916baaa1a8
+#        aws ec2 stop-instances --instance-ids i-05222103df01eb2d7
+#        aws ec2 stop-instances --instance-ids i-0c5dccd394ed9be18
+#        aws ec2 stop-instances --instance-ids i-04c1c50b3e4952266