Merge pull request #10899 from id/0530-update-perf-test

0530 update perf test

.github/workflows/performance_test.yaml

@@ -46,15 +46,16 @@ jobs:
         name: emqx-ubuntu20.04
         path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }}
 
-  tf_emqx_perf_test:
+  scenario_1on1:
     runs-on: ubuntu-latest
     needs:
       - prepare
     env:
       TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }}
-      TF_VAR_grafana_api_key: ${{ secrets.TF_EMQX_PERF_TEST_GRAFANA_API_KEY }}
+      TF_AWS_REGION: eu-west-1
-      TF_AWS_REGION: eu-north-1
       TF_VAR_test_duration: 1800
+      TF_VAR_prometheus_remote_write_url: ${{ secrets.TF_EMQX_PERF_TEST_PROMETHEUS_REMOTE_WRITE_URL }}
+      TF_VAR_prometheus_remote_write_region: eu-west-1
 
     steps:
     - name: Configure AWS Credentials
@@ -62,12 +63,13 @@ jobs:
       with:
         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
-        aws-region: eu-north-1
+        aws-region: eu-west-1
     - name: Checkout tf-emqx-performance-test
       uses: actions/checkout@v3
       with:
         repository: emqx/tf-emqx-performance-test
         path: tf-emqx-performance-test
+        ref: v0.2.2
     - uses: actions/download-artifact@v3
       with:
         name: emqx-ubuntu20.04
@@ -84,9 +86,9 @@ jobs:
         TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/1on1"
         TF_VAR_use_emqttb: 1
         TF_VAR_use_emqtt_bench: 0
-        TF_VAR_emqttb_instance_count: 2
+        TF_VAR_emqttb_instance_count: 1
-        TF_VAR_emqttb_instance_type: "c5.large"
+        TF_VAR_emqttb_instance_type: "c5.2xlarge"
-        TF_VAR_emqttb_scenario: "@pub --topic 't/%n' --pubinterval 10ms --qos 1 --publatency 50ms --size 16 --num-clients 25000 @sub --topic 't/%n' --num-clients 25000"
+        TF_VAR_emqttb_scenario: "@pubsub_fwd -n 50_000 --pub-qos 1 --sub-qos 1"
         TF_VAR_emqx_instance_type: "c5.xlarge"
         TF_VAR_emqx_instance_count: 3
       run: |
@@ -103,10 +105,10 @@ jobs:
     - name: Send notification to Slack
       uses: slackapi/slack-github-action@v1.23.0
       env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        SLACK_WEBHOOK_URL: ${{ secrets.TF_EMQX_PERF_TEST_SLACK_URL }}
       with:
         payload: |
-          {"text": "Performance test result for 1on1 scenario (50k pub, 50k sub): ${{ job.status }}\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Pub message rate*: ${{ steps.scenario_1on1.outputs.PUB_MSG_RATE }}\n*Sub message rate*: ${{ steps.scenario_1on1.outputs.SUB_MSG_RATE }}\nDropped messages: ${{ steps.scenario_1on1.outputs.MESSAGES_DROPPED }}"}
+          {"text": "Performance test result for 1on1 scenario (50k pub, 50k sub): ${{ job.status }}\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Pub message rate*: `${{ steps.scenario_1on1.outputs.PUB_MSG_RATE }}`\n*Sub message rate*: `${{ steps.scenario_1on1.outputs.SUB_MSG_RATE }}`\nDropped messages: `${{ steps.scenario_1on1.outputs.MESSAGES_DROPPED }}`"}
     - name: terraform destroy
       if: always()
       working-directory: ./tf-emqx-performance-test
@@ -116,7 +118,85 @@ jobs:
       if: success()
       with:
         name: metrics
-        path: "./tf-emqx-performance-test/metrics.json"
+        path: "./tf-emqx-performance-test/*.json"
+    - uses: actions/upload-artifact@v3
+      if: failure()
+      with:
+        name: terraform
+        path: |
+          ./tf-emqx-performance-test/.terraform
+          ./tf-emqx-performance-test/*.tfstate
+
+  scenario_1m_conns:
+    runs-on: ubuntu-latest
+    needs:
+      - prepare
+      - scenario_1on1
+    env:
+      TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }}
+      TF_AWS_REGION: eu-west-1
+      TF_VAR_test_duration: 1800
+      TF_VAR_prometheus_remote_write_url: ${{ secrets.TF_EMQX_PERF_TEST_PROMETHEUS_REMOTE_WRITE_URL }}
+      TF_VAR_prometheus_remote_write_region: eu-west-1
+
+    steps:
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
+        aws-region: eu-west-1
+    - name: Checkout tf-emqx-performance-test
+      uses: actions/checkout@v3
+      with:
+        repository: emqx/tf-emqx-performance-test
+        path: tf-emqx-performance-test
+        ref: v0.2.2
+    - uses: actions/download-artifact@v3
+      with:
+        name: emqx-ubuntu20.04
+        path: tf-emqx-performance-test/
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v2
+      with:
+        terraform_wrapper: false
+    - name: 1m conns scenario
+      id: scenario_1m_conns
+      working-directory: ./tf-emqx-performance-test
+      timeout-minutes: 60
+      env:
+        TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/scenario_1m_conns"
+        TF_VAR_use_emqttb: 1
+        TF_VAR_use_emqtt_bench: 0
+        TF_VAR_emqttb_instance_count: 5
+        TF_VAR_emqttb_instance_type: "c5.xlarge"
+        TF_VAR_emqttb_scenario: "@conn -N 200_000 --conninterval 1ms"
+        TF_VAR_emqx_instance_type: "c5.xlarge"
+        TF_VAR_emqx_instance_count: 5
+      run: |
+        terraform init
+        terraform apply -auto-approve
+        ./wait-emqttb.sh
+        ./fetch-metrics.sh
+        echo CLIENT_CONNECT=$(cat metrics.json | jq '[.[]."client.connect"] | add') >> $GITHUB_OUTPUT
+        terraform destroy -auto-approve
+    - name: Send notification to Slack
+      uses: slackapi/slack-github-action@v1.23.0
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.TF_EMQX_PERF_TEST_SLACK_URL }}
+      with:
+        payload: |
+          {"text": "Performance test result for 1m conns: ${{ job.status }}\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*client.connect*: `${{ steps.scenario_1m_conns.outputs.CLIENT_CONNECT }}`"}
+    - name: terraform destroy
+      if: always()
+      working-directory: ./tf-emqx-performance-test
+      run: |
+        terraform destroy -auto-approve
+    - uses: actions/upload-artifact@v3
+      if: success()
+      with:
+        name: metrics
+        path: "./tf-emqx-performance-test/*.json"
     - uses: actions/upload-artifact@v3
       if: failure()
       with: