emqx/.github/workflows/performance_test.yaml

name: Performance Test Suite

on:
  push:
    branches:
      - 'perf/**'
  schedule:
    - cron:  '0 1 * * MON-FRI'
  workflow_dispatch:
    inputs:
      ref:
        required: false

env:
  TF_AWS_REGION: eu-west-1
  TF_VAR_s3_bucket_name: tf-emqx-performance-test2
  TF_VAR_test_duration: 1800
  TF_VAR_prometheus_remote_write_region: eu-west-1
  TF_VAR_prometheus_remote_write_url: ${{ secrets.TF_EMQX_PERF_TEST_PROMETHEUS_REMOTE_WRITE_URL }}
  SLACK_WEBHOOK_URL: ${{ secrets.TF_EMQX_PERF_TEST_SLACK_URL }}

permissions:
  contents: read

jobs:
  prepare:
    runs-on: ubuntu-latest
    if: github.repository_owner == 'emqx'
    container: ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu20.04
    outputs:
      BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }}
      PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }}

    steps:
    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      with:
        fetch-depth: 0
        ref: ${{ github.event.inputs.ref }}
    - name: Work around https://github.com/actions/checkout/issues/766
      run: |
        git config --global --add safe.directory "$GITHUB_WORKSPACE"        
    - id: prepare
      run: |
        echo "EMQX_NAME=emqx" >> $GITHUB_ENV
        echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
        echo "BENCH_ID=$(date --utc +%F)/emqx-$(./pkg-vsn.sh emqx)" >> $GITHUB_OUTPUT        
    - name: Build deb package
      run: |
        make ${EMQX_NAME}-pkg
        ./scripts/pkg-tests.sh ${EMQX_NAME}-pkg        
    - name: Get package file name
      id: package_file
      run: |
        echo "PACKAGE_FILE=$(find _packages/emqx -name 'emqx-*.deb' | head -n 1 | xargs basename)" >> $GITHUB_OUTPUT        
    - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
      with:
        name: emqx-ubuntu20.04
        path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }}

  scenario_1on1:
    runs-on: ubuntu-latest
    needs:
      - prepare
    env:
      TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }}

    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
        aws-region: eu-west-1
    - name: Checkout tf-emqx-performance-test
      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      with:
        repository: emqx/tf-emqx-performance-test
        path: tf-emqx-performance-test
        ref: v0.2.3
    - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
      with:
        name: emqx-ubuntu20.04
        path: tf-emqx-performance-test/
    - name: Setup Terraform
      uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
      with:
        terraform_wrapper: false
    - name: run scenario
      working-directory: ./tf-emqx-performance-test
      timeout-minutes: 60
      env:
        TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/1on1"
        TF_VAR_use_emqttb: 1
        TF_VAR_use_emqtt_bench: 0
        TF_VAR_emqttb_instance_count: 1
        TF_VAR_emqttb_instance_type: c5.2xlarge
        TF_VAR_emqttb_scenario: '@pubsub_fwd -n 50_000 --pub-qos 1 --sub-qos 1'
        TF_VAR_emqx_instance_type: c5.2xlarge
        TF_VAR_emqx_instance_count: 3
      run: |
        terraform init
        terraform apply -auto-approve
        ./wait-emqttb.sh
        ./fetch-metrics.sh
        terraform destroy -auto-approve
        aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .        
    - name: Send notification to Slack
      uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
      with:
        payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
    - name: terraform destroy
      if: always()
      working-directory: ./tf-emqx-performance-test
      run: |
        terraform destroy -auto-approve        
    - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
      if: success()
      with:
        name: metrics
        path: |
          "./tf-emqx-performance-test/*.tar.gz"          
    - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
      if: failure()
      with:
        name: terraform
        path: |
          ./tf-emqx-performance-test/.terraform
          ./tf-emqx-performance-test/*.tfstate          

  scenario_fanout:
    runs-on: ubuntu-latest
    needs:
      - prepare
      - scenario_1on1
    env:
      TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }}

    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
        aws-region: eu-west-1
    - name: Checkout tf-emqx-performance-test
      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      with:
        repository: emqx/tf-emqx-performance-test
        path: tf-emqx-performance-test
        ref: v0.2.3
    - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
      with:
        name: emqx-ubuntu20.04
        path: tf-emqx-performance-test/
    - name: Setup Terraform
      uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
      with:
        terraform_wrapper: false
    - name: run scenario
      working-directory: ./tf-emqx-performance-test
      timeout-minutes: 60
      env:
        TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/fan-out"
        TF_VAR_use_emqttb: 1
        TF_VAR_use_emqtt_bench: 0
        TF_VAR_emqttb_instance_count: 1
        TF_VAR_emqttb_instance_type: c5.2xlarge
        TF_VAR_emqttb_scenario: '@pub --topic "t/%n" --conninterval 10ms --pubinterval 20ms --num-clients 5 --size 16 @sub --topic "t/#" --conninterval 10ms --num-clients 1000'
        TF_VAR_emqx_instance_type: c5.large
        TF_VAR_emqx_instance_count: 3
      run: |
        terraform init
        terraform apply -auto-approve
        ./wait-emqttb.sh
        ./fetch-metrics.sh
        terraform destroy -auto-approve
        aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .        
    - name: Send notification to Slack
      uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
      with:
        payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
    - name: terraform destroy
      if: always()
      working-directory: ./tf-emqx-performance-test
      run: |
        terraform destroy -auto-approve        
    - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
      if: success()
      with:
        name: metrics
        path: |
          "./tf-emqx-performance-test/*.tar.gz"          
    - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
      if: failure()
      with:
        name: terraform
        path: |
          ./tf-emqx-performance-test/.terraform
          ./tf-emqx-performance-test/*.tfstate          

  scenario_fanin:
    runs-on: ubuntu-latest
    needs:
      - prepare
      - scenario_1on1
      - scenario_fanout
    env:
      TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }}

    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
        aws-region: eu-west-1
    - name: Checkout tf-emqx-performance-test
      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      with:
        repository: emqx/tf-emqx-performance-test
        path: tf-emqx-performance-test
        ref: v0.2.3
    - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
      with:
        name: emqx-ubuntu20.04
        path: tf-emqx-performance-test/
    - name: Setup Terraform
      uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
      with:
        terraform_wrapper: false
    - name: run scenario
      working-directory: ./tf-emqx-performance-test
      timeout-minutes: 60
      env:
        TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/fan-in"
        TF_VAR_use_emqttb: 1
        TF_VAR_use_emqtt_bench: 0
        TF_VAR_emqttb_instance_count: 2
        TF_VAR_emqttb_start_n_multiplier: 25000
        TF_VAR_emqttb_instance_type: c5.xlarge
        TF_VAR_emqttb_scenario: '@pub --topic t/%n --conninterval 10ms --pubinterval 1s --num-clients 25_000 --start-n $START_N --size 16 @sub --topic \$share/perf/t/# --conninterval 10ms --num-clients 250'
        TF_VAR_emqx_instance_type: c5.2xlarge
        TF_VAR_emqx_instance_count: 3
      run: |
        terraform init
        terraform apply -auto-approve
        ./wait-emqttb.sh
        ./fetch-metrics.sh
        terraform destroy -auto-approve
        aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .        
    - name: Send notification to Slack
      uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
      with:
        payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
    - name: terraform destroy
      if: always()
      working-directory: ./tf-emqx-performance-test
      run: |
        terraform destroy -auto-approve        
    - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
      if: success()
      with:
        name: metrics
        path: |
          "./tf-emqx-performance-test/*.tar.gz"          
    - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
      if: failure()
      with:
        name: terraform
        path: |
          ./tf-emqx-performance-test/.terraform
          ./tf-emqx-performance-test/*.tfstate          

  scenario_1m_conns:
    runs-on: ubuntu-latest
    needs:
      - prepare
      - scenario_fanin
      - scenario_fanout
      - scenario_1on1
    env:
      TF_VAR_package_file: ${{ needs.prepare.outputs.PACKAGE_FILE }}

    steps:
    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
        aws-region: eu-west-1
    - name: Checkout tf-emqx-performance-test
      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      with:
        repository: emqx/tf-emqx-performance-test
        path: tf-emqx-performance-test
        ref: v0.2.3
    - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
      with:
        name: emqx-ubuntu20.04
        path: tf-emqx-performance-test/
    - name: Setup Terraform
      uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
      with:
        terraform_wrapper: false
    - name: run scenario
      working-directory: ./tf-emqx-performance-test
      timeout-minutes: 60
      env:
        TF_VAR_bench_id: "${{ needs.prepare.outputs.BENCH_ID }}/1m-connections"
        TF_VAR_use_emqttb: 1
        TF_VAR_use_emqtt_bench: 0
        TF_VAR_emqttb_instance_count: 5
        TF_VAR_emqttb_instance_type: c5.2xlarge
        TF_VAR_emqttb_scenario: '@conn -N 200_000 --conninterval 1ms'
        TF_VAR_emqx_instance_type: c5.2xlarge
        TF_VAR_emqx_instance_count: 3
      run: |
        terraform init
        terraform apply -auto-approve
        ./wait-emqttb.sh
        ./fetch-metrics.sh
        terraform destroy -auto-approve
        aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .        
    - name: Send notification to Slack
      uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
      with:
        payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
    - name: terraform destroy
      if: always()
      working-directory: ./tf-emqx-performance-test
      run: |
        terraform destroy -auto-approve        
    - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
      if: success()
      with:
        name: metrics
        path: |
          "./tf-emqx-performance-test/*.tar.gz"          
    - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
      if: failure()
      with:
        name: terraform
        path: |
          ./tf-emqx-performance-test/.terraform
          ./tf-emqx-performance-test/*.tfstate