diff --git a/.github/workflows/performance_test.yaml b/.github/workflows/performance_test.yaml index 5283de51a..02021d3de 100644 --- a/.github/workflows/performance_test.yaml +++ b/.github/workflows/performance_test.yaml @@ -46,15 +46,16 @@ jobs: name: emqx-ubuntu20.04 path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }} - tf_emqx_perf_test: + scenario_1on1: runs-on: ubuntu-latest needs: - prepare env: TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }} - TF_VAR_grafana_api_key: ${{ secrets.TF_EMQX_PERF_TEST_GRAFANA_API_KEY }} - TF_AWS_REGION: eu-north-1 + TF_AWS_REGION: eu-west-1 TF_VAR_test_duration: 1800 + TF_VAR_prometheus_remote_write_url: ${{ secrets.TF_EMQX_PERF_TEST_PROMETHEUS_REMOTE_WRITE_URL }} + TF_VAR_prometheus_remote_write_region: eu-west-1 steps: - name: Configure AWS Credentials @@ -62,12 +63,13 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }} - aws-region: eu-north-1 + aws-region: eu-west-1 - name: Checkout tf-emqx-performance-test uses: actions/checkout@v3 with: repository: emqx/tf-emqx-performance-test path: tf-emqx-performance-test + ref: v0.2.2 - uses: actions/download-artifact@v3 with: name: emqx-ubuntu20.04 @@ -84,9 +86,9 @@ jobs: TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/1on1" TF_VAR_use_emqttb: 1 TF_VAR_use_emqtt_bench: 0 - TF_VAR_emqttb_instance_count: 2 - TF_VAR_emqttb_instance_type: "c5.large" - TF_VAR_emqttb_scenario: "@pub --topic 't/%n' --pubinterval 10ms --qos 1 --publatency 50ms --size 16 --num-clients 25000 @sub --topic 't/%n' --num-clients 25000" + TF_VAR_emqttb_instance_count: 1 + TF_VAR_emqttb_instance_type: "c5.2xlarge" + TF_VAR_emqttb_scenario: "@pubsub_fwd -n 50_000 --pub-qos 1 --sub-qos 1" TF_VAR_emqx_instance_type: "c5.xlarge" TF_VAR_emqx_instance_count: 3 run: | @@ -103,10 +105,10 @@ jobs: - name: Send notification to Slack uses: slackapi/slack-github-action@v1.23.0 env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SLACK_WEBHOOK_URL: ${{ secrets.TF_EMQX_PERF_TEST_SLACK_URL }} with: payload: | - {"text": "Performance test result for 1on1 scenario (50k pub, 50k sub): ${{ job.status }}\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Pub message rate*: ${{ steps.scenario_1on1.outputs.PUB_MSG_RATE }}\n*Sub message rate*: ${{ steps.scenario_1on1.outputs.SUB_MSG_RATE }}\nDropped messages: ${{ steps.scenario_1on1.outputs.MESSAGES_DROPPED }}"} + {"text": "Performance test result for 1on1 scenario (50k pub, 50k sub): ${{ job.status }}\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Pub message rate*: `${{ steps.scenario_1on1.outputs.PUB_MSG_RATE }}`\n*Sub message rate*: `${{ steps.scenario_1on1.outputs.SUB_MSG_RATE }}`\nDropped messages: `${{ steps.scenario_1on1.outputs.MESSAGES_DROPPED }}`"} - name: terraform destroy if: always() working-directory: ./tf-emqx-performance-test @@ -116,7 +118,85 @@ jobs: if: success() with: name: metrics - path: "./tf-emqx-performance-test/metrics.json" + path: "./tf-emqx-performance-test/*.json" + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: terraform + path: | + ./tf-emqx-performance-test/.terraform + ./tf-emqx-performance-test/*.tfstate + + scenario_1m_conns: + runs-on: ubuntu-latest + needs: + - prepare + - scenario_1on1 + env: + TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }} + TF_AWS_REGION: eu-west-1 + TF_VAR_test_duration: 1800 + TF_VAR_prometheus_remote_write_url: ${{ secrets.TF_EMQX_PERF_TEST_PROMETHEUS_REMOTE_WRITE_URL }} + TF_VAR_prometheus_remote_write_region: eu-west-1 + + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }} + aws-region: eu-west-1 + - name: Checkout tf-emqx-performance-test + uses: actions/checkout@v3 + with: + repository: emqx/tf-emqx-performance-test + path: tf-emqx-performance-test + ref: v0.2.2 + - uses: actions/download-artifact@v3 + with: + name: emqx-ubuntu20.04 + path: tf-emqx-performance-test/ + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_wrapper: false + - name: 1m conns scenario + id: scenario_1m_conns + working-directory: ./tf-emqx-performance-test + timeout-minutes: 60 + env: + TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/scenario_1m_conns" + TF_VAR_use_emqttb: 1 + TF_VAR_use_emqtt_bench: 0 + TF_VAR_emqttb_instance_count: 5 + TF_VAR_emqttb_instance_type: "c5.xlarge" + TF_VAR_emqttb_scenario: "@conn -N 200_000 --conninterval 1ms" + TF_VAR_emqx_instance_type: "c5.xlarge" + TF_VAR_emqx_instance_count: 5 + run: | + terraform init + terraform apply -auto-approve + ./wait-emqttb.sh + ./fetch-metrics.sh + echo CLIENT_CONNECT=$(cat metrics.json | jq '[.[]."client.connect"] | add') >> $GITHUB_OUTPUT + terraform destroy -auto-approve + - name: Send notification to Slack + uses: slackapi/slack-github-action@v1.23.0 + env: + SLACK_WEBHOOK_URL: ${{ secrets.TF_EMQX_PERF_TEST_SLACK_URL }} + with: + payload: | + {"text": "Performance test result for 1m conns: ${{ job.status }}\nhttps://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*client.connect*: `${{ steps.scenario_1m_conns.outputs.CLIENT_CONNECT }}`"} + - name: terraform destroy + if: always() + working-directory: ./tf-emqx-performance-test + run: | + terraform destroy -auto-approve + - uses: actions/upload-artifact@v3 + if: success() + with: + name: metrics + path: "./tf-emqx-performance-test/*.json" - uses: actions/upload-artifact@v3 if: failure() with: