chore(autotest): debug broker performance test suite

.github/workflows/run_broker_performance_cluster.yaml

@@ -147,423 +147,423 @@ jobs:
         echo ${{ steps.emqx_private_ip.outputs.emqx_node_ip3 }}
         echo ${{ steps.emqx_private_ip.outputs.emqx_public_ip }}
 
-#  xmeter_start:
+  xmeter_start:
-#    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}
+        aws-region: ap-southeast-1
+    - name: Start Xmeter Services
+      run: |
+        aws ec2 start-instances --instance-ids i-0dd6d99916baaa1a8
+        aws ec2 start-instances --instance-ids i-05222103df01eb2d7
+        sleep 40;
+        aws ec2 start-instances --instance-ids i-0c5dccd394ed9be18
+        aws ec2 start-instances --instance-ids i-04c1c50b3e4952266
+
+  terraform_mysql:
+    runs-on: ubuntu-latest
+
+    outputs:
+      mysql_ip: ${{ steps.mysql_ip.outputs.mysql_ip }}
+      mysql_url: ${{ steps.mysql_url.outputs.mysql_url }}
+    steps:
+    - name: Checkout tf-test-automation
+      uses: actions/checkout@v2
+      with:
+        repository: emqx/tf-test-automation
+        ref: emqx-broker
+        path: tf-test-automation
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v1.3.2
+      with:
+        terraform_wrapper: false
+    - name: Terraform Init tf-test-automation
+      working-directory: ./tf-test-automation/services/mysql
+      id: init1
+      run: |
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+    - name: Terraform Validate tf-test-automation
+      working-directory: ./tf-test-automation/services/mysql
+      id: validate1
+      run: terraform validate -no-color
+    - name: Terraform Apply tf-test-automation
+      working-directory: ./tf-test-automation/services/mysql
+      id: mysql_url
+      run: |
+        terraform apply -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+        echo "::set-output name=mysql_url::$(terraform output endpoint)"
+    - name: Rename and upload mysql tfstate
+      working-directory: ./tf-test-automation/services/mysql
+      run: |
+        mv ./terraform.tfstate ./mysql.tfstate
+    - uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: tfstate
+        path: ./tf-test-automation/services/mysql/mysql.tfstate
+    - name: Show mysql url
+      id: mysql_ip
+      run: |
+        ip=`echo "${{ steps.mysql_url.outputs.mysql_url }}"|awk -F ':' '{print $1}'`
+        echo "::set-output name=mysql_ip::$ip"
+    - name: Show mysql ip
+      run: |
+        echo ${{ steps.mysql_url.outputs.mysql_url }}
+        echo ${{ steps.mysql_ip.outputs.mysql_ip }}
 #
-#    steps:
+  terraform_redis:
-#    - name: Configure AWS Credentials
+    runs-on: ubuntu-latest
-#      uses: aws-actions/configure-aws-credentials@v1
+
-#      with:
+    outputs:
-#        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}
+      redis_ip: ${{ steps.redis_ip.outputs.redis_ip }}
-#        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}
+    steps:
-#        aws-region: ap-southeast-1
+    - name: Checkout tf-test-automation
-#    - name: Start Xmeter Services
+      uses: actions/checkout@v2
-#      run: |
+      with:
-#        aws ec2 start-instances --instance-ids i-0dd6d99916baaa1a8
+        repository: emqx/tf-test-automation
-#        aws ec2 start-instances --instance-ids i-05222103df01eb2d7
+        ref: emqx-broker
-#        sleep 40;
+        path: tf-test-automation
-#        aws ec2 start-instances --instance-ids i-0c5dccd394ed9be18
+    - name: Setup Terraform
-#        aws ec2 start-instances --instance-ids i-04c1c50b3e4952266
+      uses: hashicorp/setup-terraform@v1.3.2
-#
+      with:
-#  terraform_mysql:
+        terraform_wrapper: false
-#    runs-on: ubuntu-latest
+    - name: Terraform Init tf-test-automation
-#
+      working-directory: ./tf-test-automation/services/redis
-#    outputs:
+      id: init
-#      mysql_ip: ${{ steps.mysql_ip.outputs.mysql_ip }}
+      run: |
-#      mysql_url: ${{ steps.mysql_url.outputs.mysql_url }}
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    steps:
+    - name: Terraform Validate tf-test-automation
-#    - name: Checkout tf-test-automation
+      working-directory: ./tf-test-automation/services/redis
-#      uses: actions/checkout@v2
+      id: validate
-#      with:
+      run: terraform validate -no-color
-#        repository: emqx/tf-test-automation
+    - name: Terraform Apply tf-test-automation
-#        ref: emqx-broker
+      working-directory: ./tf-test-automation/services/redis
-#        path: tf-test-automation
+      id: redis_ip
-#    - name: Setup Terraform
+      run: |
-#      uses: hashicorp/setup-terraform@v1.3.2
+        terraform apply -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
-#      with:
+        echo "::set-output name=redis_ip::$(terraform output redis_private_ips)"
-#        terraform_wrapper: false
+    - name: Rename and upload redis server tfstate
-#    - name: Terraform Init tf-test-automation
+      working-directory: ./tf-test-automation/services/redis
-#      working-directory: ./tf-test-automation/services/mysql
+      run: |
-#      id: init1
+        mv ./terraform.tfstate ./redis.tfstate
-#      run: |
+    - uses: actions/upload-artifact@v2
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+      if: always()
-#    - name: Terraform Validate tf-test-automation
+      with:
-#      working-directory: ./tf-test-automation/services/mysql
+        name: tfstate
-#      id: validate1
+        path: ./tf-test-automation/services/redis/redis.tfstate
-#      run: terraform validate -no-color
+    - name: Show redis ip
-#    - name: Terraform Apply tf-test-automation
+      run: |
-#      working-directory: ./tf-test-automation/services/mysql
+        echo ${{ steps.redis_ip.outputs.redis_ip }}
-#      id: mysql_url
+
-#      run: |
+  terraform_pgsql:
-#        terraform apply -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+    runs-on: ubuntu-latest
-#        echo "::set-output name=mysql_url::$(terraform output endpoint)"
+
-#    - name: Rename and upload mysql tfstate
+    outputs:
-#      working-directory: ./tf-test-automation/services/mysql
+      pgsql_url: ${{ steps.pgsql_url.outputs.pgsql_url }}
-#      run: |
+      pgsql_ip: ${{ steps.pgsql_ip.outputs.pgsql_ip }}
-#        mv ./terraform.tfstate ./mysql.tfstate
+    steps:
-#    - uses: actions/upload-artifact@v2
+    - name: Checkout tf-test-automation
-#      if: always()
+      uses: actions/checkout@v2
-#      with:
+      with:
-#        name: tfstate
+        repository: emqx/tf-test-automation
-#        path: ./tf-test-automation/services/mysql/mysql.tfstate
+        ref: emqx-broker
-#    - name: Show mysql url
+        path: tf-test-automation
-#      id: mysql_ip
+    - name: Setup Terraform
-#      run: |
+      uses: hashicorp/setup-terraform@v1.3.2
-#        ip=`echo "${{ steps.mysql_url.outputs.mysql_url }}"|awk -F ':' '{print $1}'`
+      with:
-#        echo "::set-output name=mysql_ip::$ip"
+        terraform_wrapper: false
-#    - name: Show mysql ip
+    - name: Terraform Init tf-test-automation
-#      run: |
+      working-directory: ./tf-test-automation/services/pgsql
-#        echo ${{ steps.mysql_url.outputs.mysql_url }}
+      id: init
-#        echo ${{ steps.mysql_ip.outputs.mysql_ip }}
+      run: |
-##
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#  terraform_redis:
+    - name: Terraform Validate tf-test-automation
-#    runs-on: ubuntu-latest
+      working-directory: ./tf-test-automation/services/pgsql
-#
+      id: validate
-#    outputs:
+      run: terraform validate -no-color
-#      redis_ip: ${{ steps.redis_ip.outputs.redis_ip }}
+    - name: Terraform Apply tf-test-automation
-#    steps:
+      working-directory: ./tf-test-automation/services/pgsql
-#    - name: Checkout tf-test-automation
+      id: pgsql_url
-#      uses: actions/checkout@v2
+      run: |
-#      with:
+        terraform apply -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#        repository: emqx/tf-test-automation
+        echo "::set-output name=pgsql_url::$(terraform output endpoint)"
-#        ref: emqx-broker
+    - name: Rename and upload pgsql server tfstate
-#        path: tf-test-automation
+      working-directory: ./tf-test-automation/services/pgsql
-#    - name: Setup Terraform
+      run: |
-#      uses: hashicorp/setup-terraform@v1.3.2
+        mv ./terraform.tfstate ./pgsql.tfstate
-#      with:
+    - uses: actions/upload-artifact@v2
-#        terraform_wrapper: false
+      if: always()
-#    - name: Terraform Init tf-test-automation
+      with:
-#      working-directory: ./tf-test-automation/services/redis
+        name: tfstate
-#      id: init
+        path: ./tf-test-automation/services/pgsql/pgsql.tfstate
-#      run: |
+    - name: Show pgsql url
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+      id: pgsql_ip
-#    - name: Terraform Validate tf-test-automation
+      run: |
-#      working-directory: ./tf-test-automation/services/redis
+        ip=`echo "${{ steps.pgsql_url.outputs.pgsql_url }}"|awk -F ':' '{print $1}'`
-#      id: validate
+        echo "::set-output name=pgsql_ip::$ip"
-#      run: terraform validate -no-color
+    - name: Show pgsql ip
-#    - name: Terraform Apply tf-test-automation
+      run: |
-#      working-directory: ./tf-test-automation/services/redis
+        echo ${{ steps.pgsql_url.outputs.pgsql_url }}
-#      id: redis_ip
+        echo ${{ steps.pgsql_ip.outputs.pgsql_ip }}
-#      run: |
+
-#        terraform apply -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
+  terraform_mongo:
-#        echo "::set-output name=redis_ip::$(terraform output redis_private_ips)"
+    runs-on: ubuntu-latest
-#    - name: Rename and upload redis server tfstate
+
-#      working-directory: ./tf-test-automation/services/redis
+    outputs:
-#      run: |
+      mongo_ip: ${{ steps.mongo_ip.outputs.mongo_ip }}
-#        mv ./terraform.tfstate ./redis.tfstate
+    steps:
-#    - uses: actions/upload-artifact@v2
+    - name: Checkout tf-test-automation
-#      if: always()
+      uses: actions/checkout@v2
-#      with:
+      with:
-#        name: tfstate
+        repository: emqx/tf-test-automation
-#        path: ./tf-test-automation/services/redis/redis.tfstate
+        ref: emqx-broker
-#    - name: Show redis ip
+        path: tf-test-automation
-#      run: |
+    - name: Setup Terraform
-#        echo ${{ steps.redis_ip.outputs.redis_ip }}
+      uses: hashicorp/setup-terraform@v1.3.2
-#
+      with:
-#  terraform_pgsql:
+        terraform_wrapper: false
-#    runs-on: ubuntu-latest
+    - name: Terraform Init tf-test-automation
-#
+      working-directory: ./tf-test-automation/services/mongodb
-#    outputs:
+      id: init
-#      pgsql_url: ${{ steps.pgsql_url.outputs.pgsql_url }}
+      run: |
-#      pgsql_ip: ${{ steps.pgsql_ip.outputs.pgsql_ip }}
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    steps:
+    - name: Terraform Validate tf-test-automation
-#    - name: Checkout tf-test-automation
+      working-directory: ./tf-test-automation/services/mongodb
-#      uses: actions/checkout@v2
+      id: validate
-#      with:
+      run: terraform validate -no-color
-#        repository: emqx/tf-test-automation
+    - name: Terraform Apply tf-test-automation
-#        ref: emqx-broker
+      working-directory: ./tf-test-automation/services/mongodb
-#        path: tf-test-automation
+      id: mongo_ip
-#    - name: Setup Terraform
+      run: |
-#      uses: hashicorp/setup-terraform@v1.3.2
+        terraform apply -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
-#      with:
+        echo "::set-output name=mongo_ip::$(terraform output mongo_private_ips)"
-#        terraform_wrapper: false
+    - name: Rename and upload mongo server tfstate
-#    - name: Terraform Init tf-test-automation
+      working-directory: ./tf-test-automation/services/mongodb
-#      working-directory: ./tf-test-automation/services/pgsql
+      run: |
-#      id: init
+        mv ./terraform.tfstate ./mongo.tfstate
-#      run: |
+    - uses: actions/upload-artifact@v2
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+      if: always()
-#    - name: Terraform Validate tf-test-automation
+      with:
-#      working-directory: ./tf-test-automation/services/pgsql
+        name: tfstate
-#      id: validate
+        path: ./tf-test-automation/services/mongodb/mongo.tfstate
-#      run: terraform validate -no-color
+    - name: Show mongo ip
-#    - name: Terraform Apply tf-test-automation
+      run: |
-#      working-directory: ./tf-test-automation/services/pgsql
+        echo ${{ steps.mongo_ip.outputs.mongo_ip }}
-#      id: pgsql_url
+
-#      run: |
+  terraform_webhook:
-#        terraform apply -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+    runs-on: ubuntu-latest
-#        echo "::set-output name=pgsql_url::$(terraform output endpoint)"
+
-#    - name: Rename and upload pgsql server tfstate
+    outputs:
-#      working-directory: ./tf-test-automation/services/pgsql
+      webhook_ip: ${{ steps.webhook_ip.outputs.webhook_ip }}
-#      run: |
+    steps:
-#        mv ./terraform.tfstate ./pgsql.tfstate
+    - name: Checkout tf-test-automation
-#    - uses: actions/upload-artifact@v2
+      uses: actions/checkout@v2
-#      if: always()
+      with:
-#      with:
+        repository: emqx/tf-test-automation
-#        name: tfstate
+        ref: emqx-broker
-#        path: ./tf-test-automation/services/pgsql/pgsql.tfstate
+        path: tf-test-automation
-#    - name: Show pgsql url
+    - name: Setup Terraform
-#      id: pgsql_ip
+      uses: hashicorp/setup-terraform@v1.3.2
-#      run: |
+      with:
-#        ip=`echo "${{ steps.pgsql_url.outputs.pgsql_url }}"|awk -F ':' '{print $1}'`
+        terraform_wrapper: false
-#        echo "::set-output name=pgsql_ip::$ip"
+    - name: Terraform Init tf-test-automation
-#    - name: Show pgsql ip
+      working-directory: ./tf-test-automation/services/webhook
-#      run: |
+      id: init
-#        echo ${{ steps.pgsql_url.outputs.pgsql_url }}
+      run: |
-#        echo ${{ steps.pgsql_ip.outputs.pgsql_ip }}
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#
+    - name: Terraform Validate tf-test-automation
-#  terraform_mongo:
+      working-directory: ./tf-test-automation/services/webhook
-#    runs-on: ubuntu-latest
+      id: validate
-#
+      run: terraform validate -no-color
-#    outputs:
+    - name: Terraform Apply tf-test-automation
-#      mongo_ip: ${{ steps.mongo_ip.outputs.mongo_ip }}
+      working-directory: ./tf-test-automation/services/webhook
-#    steps:
+      id: webhook_ip
-#    - name: Checkout tf-test-automation
+      run: |
-#      uses: actions/checkout@v2
+        terraform apply -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
-#      with:
+        echo "::set-output name=webhook_ip::$(terraform output webhook_private_ips)"
-#        repository: emqx/tf-test-automation
+    - name: Rename and upload webhook server tfstate
-#        ref: emqx-broker
+      working-directory: ./tf-test-automation/services/webhook
-#        path: tf-test-automation
+      run: |
-#    - name: Setup Terraform
+        mv ./terraform.tfstate ./webhook.tfstate
-#      uses: hashicorp/setup-terraform@v1.3.2
+    - uses: actions/upload-artifact@v2
-#      with:
+      if: always()
-#        terraform_wrapper: false
+      with:
-#    - name: Terraform Init tf-test-automation
+        name: tfstate
-#      working-directory: ./tf-test-automation/services/mongodb
+        path: ./tf-test-automation/services/webhook/webhook.tfstate
-#      id: init
+    - name: Show webhook ip
-#      run: |
+      run: |
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+        echo ${{ steps.webhook_ip.outputs.webhook_ip }}
-#    - name: Terraform Validate tf-test-automation
+
-#      working-directory: ./tf-test-automation/services/mongodb
+  master_control:
-#      id: validate
+    runs-on: ubuntu-latest
-#      run: terraform validate -no-color
+
-#    - name: Terraform Apply tf-test-automation
+    needs: [xmeter_start, terraform_emqx, terraform_mysql, terraform_redis, terraform_pgsql, terraform_mongo, terraform_webhook]
-#      working-directory: ./tf-test-automation/services/mongodb
+    steps:
-#      id: mongo_ip
+    - uses: actions/checkout@v2
-#      run: |
+    - uses: actions/setup-java@v1
-#        terraform apply -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
+      with:
-#        echo "::set-output name=mongo_ip::$(terraform output mongo_private_ips)"
+        java-version: '8.0.282' # The JDK version to make available on the path.
-#    - name: Rename and upload mongo server tfstate
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
-#      working-directory: ./tf-test-automation/services/mongodb
+        architecture: x64 # (x64 or x86) - defaults to x64
-#      run: |
+    - name: Checkout emqx-fvt
-#        mv ./terraform.tfstate ./mongo.tfstate
+      uses: actions/checkout@v2
-#    - uses: actions/upload-artifact@v2
+      with:
-#      if: always()
+        repository: emqx/emqx-fvt
-#      with:
+        ref:  broker_performance_test
-#        name: tfstate
+        path: emqx-fvt
-#        path: ./tf-test-automation/services/mongodb/mongo.tfstate
+    - name: install jmeter
-#    - name: Show mongo ip
+      timeout-minutes: 10
-#      run: |
+      env:
-#        echo ${{ steps.mongo_ip.outputs.mongo_ip }}
+         JMETER_VERSION: 5.3
-#
+      run: |
-#  terraform_webhook:
+        wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz https://downloads.apache.org/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
-#    runs-on: ubuntu-latest
+        cd /tmp && tar -xvf apache-jmeter.tgz
-#
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
-#    outputs:
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
-#      webhook_ip: ${{ steps.webhook_ip.outputs.webhook_ip }}
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar
-#    steps:
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
-#    - name: Checkout tf-test-automation
+    - name: install jmeter plugin
-#      uses: actions/checkout@v2
+      run: |
-#      with:
+        wget --no-verbose -O "/opt/jmeter/lib/mysql-connector-java-8.0.16.jar" https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
-#        repository: emqx/tf-test-automation
+    - name: run jmeter
-#        ref: emqx-broker
+      working-directory: ./emqx-fvt/broker-performance-test-suite
-#        path: tf-test-automation
+      run: |
-#    - name: Setup Terraform
+        /opt/jmeter/bin/jmeter.sh \
-#      uses: hashicorp/setup-terraform@v1.3.2
+          -Jjmeter.save.saveservice.output_format=xml -n \
-#      with:
+          -t broker_attestation_test.jmx \
-#        terraform_wrapper: false
+          -Demqx_ip=${{ needs.terraform_emqx.outputs.emqx_public_ip }} \
-#    - name: Terraform Init tf-test-automation
+          -Demqx_private_ip1=${{ needs.terraform_emqx.outputs.emqx_node_ip1 }} \
-#      working-directory: ./tf-test-automation/services/webhook
+          -Demqx_private_ip2=${{ needs.terraform_emqx.outputs.emqx_node_ip2 }} \
-#      id: init
+          -Demqx_private_ip3=${{ needs.terraform_emqx.outputs.emqx_node_ip3 }} \
-#      run: |
+          -Dmysql_ip=${{ needs.terraform_mysql.outputs.mysql_ip }} \
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+          -Dpgsql_ip=${{ needs.terraform_pgsql.outputs.pgsql_ip }} \
-#    - name: Terraform Validate tf-test-automation
+          -Dredis_ip=${{ needs.terraform_redis.outputs.redis_ip }} \
-#      working-directory: ./tf-test-automation/services/webhook
+          -Dmongo_ip=${{ needs.terraform_mongo.outputs.mongo_ip }} \
-#      id: validate
+          -Dhttp_ip=${{ needs.terraform_webhook.outputs.webhook_ip }} \
-#      run: terraform validate -no-color
+          -Dxmeter_ip=${{ secrets.XMETER_IP }} \
-#    - name: Terraform Apply tf-test-automation
+          -Dplugins_path="/opt/jmeter/lib/ext" \
-#      working-directory: ./tf-test-automation/services/webhook
+          -Dxmeter_user=${{ secrets.XMETER_USER }} \
-#      id: webhook_ip
+          -Dxmeter_pwd=${{ secrets.XMETER_PWD }} \
-#      run: |
+          -Dscripts_path=".ci/performance-test-suite" \
-#        terraform apply -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
+          -Dreport_file="jmeter_logs/report.txt" \
-#        echo "::set-output name=webhook_ip::$(terraform output webhook_private_ips)"
+          -l jmeter_logs/xmeter_process.jtl \
-#    - name: Rename and upload webhook server tfstate
+          -j jmeter_logs/xmeter_process.log
-#      working-directory: ./tf-test-automation/services/webhook
+    - name: check logs
-#      run: |
+      run: |
-#        mv ./terraform.tfstate ./webhook.tfstate
+        if cat jmeter_logs/xmeter_process.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
-#    - uses: actions/upload-artifact@v2
+          echo "check logs filed"
-#      if: always()
+        fi
-#      with:
+    - name: check report
-#        name: tfstate
+      run: |
-#        path: ./tf-test-automation/services/webhook/webhook.tfstate
+        if cat jmeter_logs/report.txt | grep -e 'err_report_id' > /dev/null 2>&1; then
-#    - name: Show webhook ip
+          echo "check report filed"
-#      run: |
+          exit 1
-#        echo ${{ steps.webhook_ip.outputs.webhook_ip }}
+        fi
-#
+    - uses: actions/upload-artifact@v2
-#  master_control:
+      if: always()
-#    runs-on: ubuntu-latest
+      with:
-#
+        name: jmeter_logs_report
-#    needs: [xmeter_start, terraform_emqx, terraform_mysql, terraform_redis, terraform_pgsql, terraform_mongo, terraform_webhook]
+        path: ./jmeter_logs
-#    steps:
+
-#    - uses: actions/checkout@v2
+  terraform_destroy:
-#    - uses: actions/setup-java@v1
+    runs-on: ubuntu-latest
-#      with:
+
-#        java-version: '8.0.282' # The JDK version to make available on the path.
+    needs: [master_control]
-#        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+    steps:
-#        architecture: x64 # (x64 or x86) - defaults to x64
+    - name: Checkout tf-test-automation
-#    - name: Checkout emqx-fvt
+      uses: actions/checkout@v2
-#      uses: actions/checkout@v2
+      with:
-#      with:
+        repository: emqx/tf-test-automation
-#        repository: emqx/emqx-fvt
+        ref: cluster
-#        ref:  broker_performance_test
+        path: tf-test-automation
-#        path: emqx-fvt
+    - name: Setup Terraform
-#    - name: install jmeter
+      uses: hashicorp/setup-terraform@v1.3.2
-#      timeout-minutes: 10
+      with:
-#      env:
+        terraform_wrapper: false
-#         JMETER_VERSION: 5.3
+    - uses: actions/download-artifact@v2
-#      run: |
+      with:
-#        wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz https://downloads.apache.org/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
+        name: tfstate
-#        cd /tmp && tar -xvf apache-jmeter.tgz
+        path: ./tf-test-automation/services
-#        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+    - name: Terraform Init mysql
-#        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+      working-directory: ./tf-test-automation/services
-#        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-fuse-2.0.2-jar-with-dependencies.jar
+      run: |
-#        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+        cp mysql.tfstate ./mysql/terraform.tfstate
-#    - name: install jmeter plugin
+        cd ./mysql
-#      run: |
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#        wget --no-verbose -O "/opt/jmeter/lib/mysql-connector-java-8.0.16.jar" https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
+    - name: Terraform Destroy mysql
-#    - name: run jmeter
+      working-directory: ./tf-test-automation/services/mysql
-#      working-directory: ./emqx-fvt/broker-performance-test-suite
+      run: |
-#      run: |
+        terraform destroy -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#        /opt/jmeter/bin/jmeter.sh \
+    - name: Terraform Init emqx
-#          -Jjmeter.save.saveservice.output_format=xml -n \
+      working-directory: ./tf-test-automation/services
-#          -t broker_attestation_test.jmx \
+      run: |
-#          -Demqx_ip=${{ needs.terraform_emqx.outputs.emqx_public_ip }} \
+        cp emqx.tfstate ./emqx/terraform.tfstate
-#          -Demqx_private_ip1=${{ needs.terraform_emqx.outputs.emqx_node_ip1 }} \
+        cd ./emqx
-#          -Demqx_private_ip2=${{ needs.terraform_emqx.outputs.emqx_node_ip2 }} \
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#          -Demqx_private_ip3=${{ needs.terraform_emqx.outputs.emqx_node_ip3 }} \
+    - name: Terraform Destroy emqx
-#          -Dmysql_ip=${{ needs.terraform_mysql.outputs.mysql_ip }} \
+      working-directory: ./tf-test-automation/services/emqx
-#          -Dpgsql_ip=${{ needs.terraform_pgsql.outputs.pgsql_ip }} \
+      run: |
-#          -Dredis_ip=${{ needs.terraform_redis.outputs.redis_ip }} \
+        terraform destroy -auto-approve -var="region=ap-southeast-1" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#          -Dmongo_ip=${{ needs.terraform_mongo.outputs.mongo_ip }} \
+    - name: Terraform Init redis
-#          -Dhttp_ip=${{ needs.terraform_webhook.outputs.webhook_ip }} \
+      working-directory: ./tf-test-automation/services
-#          -Dxmeter_ip=${{ secrets.XMETER_IP }} \
+      run: |
-#          -Dplugins_path="/opt/jmeter/lib/ext" \
+        cp redis.tfstate ./redis/terraform.tfstate
-#          -Dxmeter_user=${{ secrets.XMETER_USER }} \
+        cd ./redis
-#          -Dxmeter_pwd=${{ secrets.XMETER_PWD }} \
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#          -Dscripts_path=".ci/performance-test-suite" \
+    - name: Terraform Destroy redis
-#          -Dreport_file="jmeter_logs/report.txt" \
+      working-directory: ./tf-test-automation/services/redis
-#          -l jmeter_logs/xmeter_process.jtl \
+      run: |
-#          -j jmeter_logs/xmeter_process.log
+        terraform destroy -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
-#    - name: check logs
+    - name: Terraform Init pgsql
-#      run: |
+      working-directory: ./tf-test-automation/services
-#        if cat jmeter_logs/xmeter_process.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+      run: |
-#          echo "check logs filed"
+        cp pgsql.tfstate ./pgsql/terraform.tfstate
-#        fi
+        cd ./pgsql
-#    - name: check report
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#      run: |
+    - name: Terraform Destroy pgsql
-#        if cat jmeter_logs/report.txt | grep -e 'err_report_id' > /dev/null 2>&1; then
+      working-directory: ./tf-test-automation/services/pgsql
-#          echo "check report filed"
+      run: |
-#          exit 1
+        terraform destroy -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#        fi
+    - name: Terraform Init mongo
-#    - uses: actions/upload-artifact@v2
+      working-directory: ./tf-test-automation/services
-#      if: always()
+      run: |
-#      with:
+        cp mongo.tfstate ./mongodb/terraform.tfstate
-#        name: jmeter_logs_report
+        cd ./mongodb
-#        path: ./jmeter_logs
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#
+    - name: Terraform Destroy mongo
-#  terraform_destroy:
+      working-directory: ./tf-test-automation/services/mongodb
-#    runs-on: ubuntu-latest
+      run: |
-#
+        terraform destroy -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    needs: [master_control]
+    - name: Terraform Init Webhook
-#    steps:
+      working-directory: ./tf-test-automation/services
-#    - name: Checkout tf-test-automation
+      run: |
-#      uses: actions/checkout@v2
+        cp webhook.tfstate ./webhook/terraform.tfstate
-#      with:
+        cd ./webhook
-#        repository: emqx/tf-test-automation
+        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#        ref: cluster
+    - name: Terraform Destroy Webhook
-#        path: tf-test-automation
+      working-directory: ./tf-test-automation/services/webhook
-#    - name: Setup Terraform
+      run: |
-#      uses: hashicorp/setup-terraform@v1.3.2
+        terraform destroy -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#      with:
+
-#        terraform_wrapper: false
+  xmeter_stop:
-#    - uses: actions/download-artifact@v2
+    runs-on: ubuntu-latest
-#      with:
+
-#        name: tfstate
+    needs: [terraform_destroy]
-#        path: ./tf-test-automation/services
+    steps:
-#    - name: Terraform Init mysql
+    - name: Configure AWS Credentials
-#      working-directory: ./tf-test-automation/services
+      uses: aws-actions/configure-aws-credentials@v1
-#      run: |
+      with:
-#        cp mysql.tfstate ./mysql/terraform.tfstate
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}
-#        cd ./mysql
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+        aws-region: ap-southeast-1
-#    - name: Terraform Destroy mysql
+    - name: Start Xmeter Services
-#      working-directory: ./tf-test-automation/services/mysql
+      run: |
-#      run: |
+        aws ec2 stop-instances --instance-ids i-0dd6d99916baaa1a8
-#        terraform destroy -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
+        aws ec2 stop-instances --instance-ids i-05222103df01eb2d7
-#    - name: Terraform Init emqx
+        aws ec2 stop-instances --instance-ids i-0c5dccd394ed9be18
-#      working-directory: ./tf-test-automation/services
+        aws ec2 stop-instances --instance-ids i-04c1c50b3e4952266
-#      run: |
-#        cp emqx.tfstate ./emqx/terraform.tfstate
-#        cd ./emqx
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    - name: Terraform Destroy emqx
-#      working-directory: ./tf-test-automation/services/emqx
-#      run: |
-#        terraform destroy -auto-approve -var="region=ap-southeast-1" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    - name: Terraform Init redis
-#      working-directory: ./tf-test-automation/services
-#      run: |
-#        cp redis.tfstate ./redis/terraform.tfstate
-#        cd ./redis
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    - name: Terraform Destroy redis
-#      working-directory: ./tf-test-automation/services/redis
-#      run: |
-#        terraform destroy -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}" -var="private_key=${{ secrets.CI_SSH_PRIVATE_KEY }}"
-#    - name: Terraform Init pgsql
-#      working-directory: ./tf-test-automation/services
-#      run: |
-#        cp pgsql.tfstate ./pgsql/terraform.tfstate
-#        cd ./pgsql
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    - name: Terraform Destroy pgsql
-#      working-directory: ./tf-test-automation/services/pgsql
-#      run: |
-#        terraform destroy -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    - name: Terraform Init mongo
-#      working-directory: ./tf-test-automation/services
-#      run: |
-#        cp mongo.tfstate ./mongodb/terraform.tfstate
-#        cd ./mongodb
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    - name: Terraform Destroy mongo
-#      working-directory: ./tf-test-automation/services/mongodb
-#      run: |
-#        terraform destroy -auto-approve -var="db_password=public123" -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    - name: Terraform Init Webhook
-#      working-directory: ./tf-test-automation/services
-#      run: |
-#        cp webhook.tfstate ./webhook/terraform.tfstate
-#        cd ./webhook
-#        terraform init --backend-config="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" --backend-config="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#    - name: Terraform Destroy Webhook
-#      working-directory: ./tf-test-automation/services/webhook
-#      run: |
-#        terraform destroy -auto-approve -var="access_key=${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}" -var="secret_key=${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}"
-#
-#  xmeter_stop:
-#    runs-on: ubuntu-latest
-#
-#    needs: [terraform_destroy]
-#    steps:
-#    - name: Configure AWS Credentials
-#      uses: aws-actions/configure-aws-credentials@v1
-#      with:
-#        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_PERFORMANCE_TESTING }}
-#        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_PERFORMANCE_TESTING }}
-#        aws-region: ap-southeast-1
-#    - name: Start Xmeter Services
-#      run: |
-#        aws ec2 stop-instances --instance-ids i-0dd6d99916baaa1a8
-#        aws ec2 stop-instances --instance-ids i-05222103df01eb2d7
-#        aws ec2 stop-instances --instance-ids i-0c5dccd394ed9be18
-#        aws ec2 stop-instances --instance-ids i-04c1c50b3e4952266