Merge pull request #11515 from kjellwinblad/kjell/release-51/v5.1.6

Changes for release of v5.1.6

.ci/docker-compose-file/.env

@@ -10,6 +10,8 @@ CASSANDRA_TAG=3.11.6
 MINIO_TAG=RELEASE.2023-03-20T20-16-18Z
 OPENTS_TAG=9aa7f88
 KINESIS_TAG=2.1
+HSTREAMDB_TAG=v0.15.0
+HSTREAMDB_ZK_TAG=3.8.1
 
 MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server
 SQLSERVER_TAG=2019-CU19-ubuntu-20.04

.ci/docker-compose-file/docker-compose-hstreamdb.yaml

@@ -2,11 +2,13 @@ version: "3.5"
 
 services:
   hserver:
-    image: hstreamdb/hstream:v0.15.0
+    image: hstreamdb/hstream:${HSTREAMDB_TAG}
     container_name: hstreamdb
     depends_on:
-      - zookeeper
+      zookeeper:
-      - hstore
+        condition: service_started
+      hstore:
+        condition: service_healthy
     # ports:
     #   - "127.0.0.1:6570:6570"
     expose:
@@ -37,7 +39,7 @@ services:
         --io-tasks-network emqx_bridge
 
   hstore:
-    image: hstreamdb/hstream:v0.15.0
+    image: hstreamdb/hstream:${HSTREAMDB_TAG}
     networks:
       - emqx_bridge
     volumes:
@@ -53,10 +55,17 @@ services:
         --use-tcp --tcp-host $$(hostname -I | awk '{print $$1}') \
         --user-admin-port 6440 \
         --param enable-dscp-reflection=false \
-        --no-interactive
+        --no-interactive \
+        > /data/store/hstore.log 2>&1
+    healthcheck:
+      test: ["CMD", "grep", "LogDevice Cluster running", "/data/store/hstore.log"]
+      interval: 10s
+      timeout: 10s
+      retries: 60
+      start_period: 60s
 
   zookeeper:
-    image: zookeeper
+    image: zookeeper:${HSTREAMDB_ZK_TAG}
     expose:
       - 2181
     networks:

.ci/docker-compose-file/docker-compose-kafka.yaml

@@ -18,7 +18,7 @@ services:
       - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret
   kdc:
     hostname: kdc.emqx.net
-    image:  ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu20.04
+    image:  ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu20.04
     container_name: kdc.emqx.net
     expose:
       - 88 # kdc

.ci/docker-compose-file/docker-compose.yaml

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu20.04}
+    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu20.04}
     env_file:
       - conf.env
     environment:

.ci/docker-compose-file/python/pytest.sh

@@ -20,7 +20,7 @@ fi
 apk update && apk add git curl
 git clone -b develop-5.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing
 
-pip install pytest==7.1.2 pytest-retry
+pip install pytest==7.1.2 pytest-retry==1.3.0
 
 pytest --retries 3 -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$TARGET_HOST"
 RESULT=$?

.github/actions/package-macos/action.yaml

@@ -3,7 +3,7 @@ inputs:
   profile: # emqx, emqx-enterprise
     required: true
     type: string
-  otp: # 25.3.2-1
+  otp: # 25.3.2-2
     required: true
     type: string
   os:

.github/pull_request_template.md

@@ -1,6 +1,6 @@
 Fixes <issue-or-jira-number>
 
-<!-- Make sure to target release-51 branch if this PR is intended to fix the issues for the release candidate. -->
+<!-- Make sure to target release-52 branch if this PR is intended to fix the issues for the release candidate. -->
 
 ## Summary
 copilot:summary

.github/workflows/_pr_entrypoint.yaml

@@ -17,7 +17,7 @@ env:
 jobs:
   sanity-checks:
     runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-    container: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04"
     outputs:
       ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
       ct-host: ${{ steps.matrix.outputs.ct-host }}
@@ -25,9 +25,9 @@ jobs:
       version-emqx: ${{ steps.matrix.outputs.version-emqx }}
       version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
       runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-      builder: "ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04"
+      builder: "ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04"
-      builder_vsn: "5.1-3"
+      builder_vsn: "5.1-4"
-      otp_vsn: "25.3.2-1"
+      otp_vsn: "25.3.2-2"
       elixir_vsn: "1.14.5"
 
     steps:
@@ -63,7 +63,8 @@ jobs:
           ./actionlint -color \
             -shellcheck= \
             -ignore 'label ".+" is unknown' \
-            -ignore 'value "emqx-enterprise" in "exclude"'
+            -ignore 'value "emqx-enterprise" in "exclude"' \
+            -ignore 'value "emqx-enterprise-elixir" in "exclude"'
       - name: Check line-break at EOF
         run: |
           ./scripts/check-nl-at-eof.sh
@@ -92,13 +93,13 @@ jobs:
           MATRIX="$(echo "${APPS}" | jq -c '
             [
               (.[] | select(.profile == "emqx") | . + {
-                builder: "5.1-3",
+                builder: "5.1-4",
-                otp: "25.3.2-1",
+                otp: "25.3.2-2",
                 elixir: "1.14.5"
               }),
               (.[] | select(.profile == "emqx-enterprise") | . + {
-                builder: "5.1-3",
+                builder: "5.1-4",
-                otp: ["25.3.2-1"][],
+                otp: ["25.3.2-2"][],
                 elixir: "1.14.5"
               })
             ]
@@ -146,6 +147,17 @@ jobs:
           path: ${{ matrix.profile }}.zip
           retention-days: 1
 
+  run_emqx_app_tests:
+    needs:
+      - sanity-checks
+      - compile
+    uses: ./.github/workflows/run_emqx_app_tests.yaml
+    with:
+      runner: ${{ needs.sanity-checks.outputs.runner }}
+      builder: ${{ needs.sanity-checks.outputs.builder }}
+      before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
+      after_ref: ${{ github.sha }}
+
   run_test_cases:
     needs:
       - sanity-checks

.github/workflows/_push-entrypoint.yaml

@@ -12,6 +12,7 @@ on:
     branches:
       - 'master'
       - 'release-51'
+      - 'release-52'
       - 'ci/**'
 
 env:
@@ -20,7 +21,7 @@ env:
 jobs:
   prepare:
     runs-on: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-    container: 'ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04'
+    container: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04'
     outputs:
       profile: ${{ steps.parse-git-ref.outputs.profile }}
       release: ${{ steps.parse-git-ref.outputs.release }}
@@ -30,9 +31,9 @@ jobs:
       ct-host: ${{ steps.matrix.outputs.ct-host }}
       ct-docker: ${{ steps.matrix.outputs.ct-docker }}
       runner: ${{ github.repository_owner == 'emqx' && 'aws-amd64' || 'ubuntu-22.04' }}
-      builder: 'ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04'
+      builder: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04'
-      builder_vsn: '5.1-3'
+      builder_vsn: '5.1-4'
-      otp_vsn: '25.3.2-1'
+      otp_vsn: '25.3.2-2'
       elixir_vsn: '1.14.5'
 
     steps:
@@ -63,13 +64,13 @@ jobs:
           MATRIX="$(echo "${APPS}" | jq -c '
             [
               (.[] | select(.profile == "emqx") | . + {
-                builder: "5.1-3",
+                builder: "5.1-4",
-                otp: "25.3.2-1",
+                otp: "25.3.2-2",
                 elixir: "1.14.5"
               }),
               (.[] | select(.profile == "emqx-enterprise") | . + {
-                builder: "5.1-3",
+                builder: "5.1-4",
-                otp: ["25.3.2-1"][],
+                otp: ["25.3.2-2"][],
                 elixir: "1.14.5"
               })
             ]
@@ -82,20 +83,8 @@ jobs:
           echo "ct-host=${CT_HOST}"     | tee -a $GITHUB_OUTPUT
           echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT
 
-  build_slim_packages:
-    if: ${{ needs.prepare.outputs.release != 'true' }}
-    needs:
-      - prepare
-    uses: ./.github/workflows/build_slim_packages.yaml
-    with:
-      runner: ${{ needs.prepare.outputs.runner }}
-      builder: ${{ needs.prepare.outputs.builder }}
-      builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
-      otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
-      elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
-
   build_packages:
-    if: ${{ needs.prepare.outputs.release == 'true' }}
+    if: needs.prepare.outputs.release == 'true'
     needs:
       - prepare
     uses: ./.github/workflows/build_packages.yaml
@@ -109,7 +98,7 @@ jobs:
     secrets: inherit
 
   build_and_push_docker_images:
-    if: ${{ needs.prepare.outputs.release == 'true' }}
+    if: needs.prepare.outputs.release == 'true'
     needs:
       - prepare
     uses: ./.github/workflows/build_and_push_docker_images.yaml
@@ -124,7 +113,20 @@ jobs:
       runner: ${{ needs.prepare.outputs.runner }}
     secrets: inherit
 
+  build_slim_packages:
+    if: needs.prepare.outputs.release != 'true'
+    needs:
+      - prepare
+    uses: ./.github/workflows/build_slim_packages.yaml
+    with:
+      runner: ${{ needs.prepare.outputs.runner }}
+      builder: ${{ needs.prepare.outputs.builder }}
+      builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
+      otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
+      elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
+
   compile:
+    if: needs.prepare.outputs.release != 'true'
     runs-on: ${{ needs.prepare.outputs.runner }}
     container: ${{ needs.prepare.outputs.builder }}
     needs:
@@ -156,7 +158,19 @@ jobs:
           path: ${{ matrix.profile }}.zip
           retention-days: 1
 
+  run_emqx_app_tests:
+    needs:
+      - prepare
+      - compile
+    uses: ./.github/workflows/run_emqx_app_tests.yaml
+    with:
+      runner: ${{ needs.prepare.outputs.runner }}
+      builder: ${{ needs.prepare.outputs.builder }}
+      before_ref: ${{ github.event.before }}
+      after_ref: ${{ github.sha }}
+
   run_test_cases:
+    if: needs.prepare.outputs.release != 'true'
     needs:
       - prepare
       - compile
@@ -169,6 +183,7 @@ jobs:
       ct-docker: ${{ needs.prepare.outputs.ct-docker }}
 
   run_conf_tests:
+    if: needs.prepare.outputs.release != 'true'
     needs:
       - prepare
       - compile
@@ -178,6 +193,7 @@ jobs:
       builder: ${{ needs.prepare.outputs.builder }}
 
   static_checks:
+    if: needs.prepare.outputs.release != 'true'
     needs:
       - prepare
       - compile

.github/workflows/build_and_push_docker_images.yaml

@@ -61,7 +61,7 @@ on:
       otp_vsn:
         required: false
         type: string
-        default: '25.3.2-1'
+        default: '25.3.2-2'
       elixir_vsn:
         required: false
         type: string
@@ -69,7 +69,7 @@ on:
       builder_vsn:
         required: false
         type: string
-        default: '5.1-3'
+        default: '5.1-4'
       runner:
         required: false
         type: string
@@ -88,6 +88,11 @@ jobs:
         registry:
           - 'docker.io'
           - 'public.ecr.aws'
+        exclude:
+          - profile: emqx-enterprise
+            registry: 'public.ecr.aws'
+          - profile: emqx-enterprise-elixir
+            registry: 'public.ecr.aws'
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/build_packages.yaml

@@ -57,7 +57,7 @@ on:
       otp_vsn:
         required: false
         type: string
-        default: '25.3.2-1'
+        default: '25.3.2-2'
       elixir_vsn:
         required: false
         type: string
@@ -69,7 +69,7 @@ on:
       builder_vsn:
         required: false
         type: string
-        default: '5.1-3'
+        default: '5.1-4'
 
 jobs:
   windows:
@@ -264,7 +264,7 @@ jobs:
         path: _packages/${{ matrix.profile }}/
 
   publish_artifacts:
-    runs-on: ${{ inputs.runner }}
+    runs-on: ubuntu-latest
     needs:
       - mac
       - linux
@@ -280,7 +280,7 @@ jobs:
         name: ${{ matrix.profile }}
         path: packages/${{ matrix.profile }}
     - name: install dos2unix
-      run: apt-get update && apt install -y dos2unix
+      run: sudo apt-get update && sudo apt install -y dos2unix
     - name: get packages
       run: |
         set -eu
@@ -300,7 +300,7 @@ jobs:
       env:
         PROFILE: ${{ matrix.profile }}
       run: |
-        set -e -u
+        set -eu
         if [ $PROFILE = 'emqx' ]; then
             s3dir='emqx-ce'
         elif [ $PROFILE = 'emqx-enterprise' ]; then

.github/workflows/build_packages_cron.yaml

@@ -13,8 +13,6 @@ jobs:
   linux:
     if: github.repository_owner == 'emqx'
     runs-on: aws-${{ matrix.arch }}
-    # always run in builder container because the host might have the wrong OTP version etc.
-    # otherwise buildx.sh does not run docker if arch and os matches the target arch and os.
     container:
       image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
 
@@ -24,15 +22,17 @@ jobs:
         profile:
           - ['emqx', 'master']
           - ['emqx-enterprise', 'release-51']
+          - ['emqx-enterprise', 'release-52']
         otp:
-          - 25.3.2-1
+          - 25.3.2-2
         arch:
           - amd64
         os:
+          - debian10
           - ubuntu22.04
           - amzn2023
         builder:
-          - 5.1-3
+          - 5.1-4
         elixir:
           - 1.14.5
 
@@ -47,25 +47,32 @@ jobs:
           ref: ${{ matrix.profile[1] }}
           fetch-depth: 0
 
-      - name: build emqx packages
+      - name: fix workdir
-        env:
-          ELIXIR: ${{ matrix.elixir }}
-          PROFILE: ${{ matrix.profile[0] }}
-          ARCH: ${{ matrix.arch }}
         run: |
           set -eu
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          PKGTYPES="tgz pkg"
+          # Align path for CMake caches
-          IS_ELIXIR="no"
+          if [ ! "$PWD" = "/emqx" ]; then
-          for PKGTYPE in ${PKGTYPES};
+            ln -s $PWD /emqx
-          do
+            cd /emqx
-            ./scripts/buildx.sh \
+          fi
-              --profile "${PROFILE}" \
+          echo "pwd is $PWD"
-              --pkgtype "${PKGTYPE}" \
+
-              --arch "${ARCH}" \
+      - name: build emqx packages
-              --elixir "${IS_ELIXIR}" \
+        env:
-              --builder "force_host"
+          PROFILE: ${{ matrix.profile[0] }}
-          done
+          ACLOCAL_PATH: "/usr/share/aclocal:/usr/local/share/aclocal"
+        run: |
+          set -eu
+          make "${PROFILE}-tgz"
+          make "${PROFILE}-pkg"
+      - name: test emqx packages
+        env:
+          PROFILE: ${{ matrix.profile[0] }}
+        run: |
+          set -eu
+          ./scripts/pkg-tests.sh "${PROFILE}-tgz"
+          ./scripts/pkg-tests.sh "${PROFILE}-pkg"
       - uses: actions/upload-artifact@v3
         if: success()
         with:
@@ -92,7 +99,7 @@ jobs:
         branch:
           - master
         otp:
-          - 25.3.2-1
+          - 25.3.2-2
         os:
           - macos-13
           - macos-12-arm64

.github/workflows/build_slim_packages.yaml

@@ -34,15 +34,15 @@ on:
       builder:
         required: false
         type: string
-        default: 'ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu22.04'
+        default: 'ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04'
       builder_vsn:
         required: false
         type: string
-        default: '5.1-3'
+        default: '5.1-4'
       otp_vsn:
         required: false
         type: string
-        default: '25.3.2-1'
+        default: '25.3.2-2'
       elixir_vsn:
         required: false
         type: string
@@ -58,8 +58,8 @@ jobs:
       fail-fast: false
       matrix:
         profile:
-          - ["emqx", "25.3.2-1", "ubuntu20.04", "elixir"]
+          - ["emqx", "25.3.2-2", "ubuntu20.04", "elixir"]
-          - ["emqx-enterprise", "25.3.2-1", "ubuntu20.04", "erlang"]
+          - ["emqx-enterprise", "25.3.2-2", "ubuntu20.04", "erlang"]
 
     container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
 

.github/workflows/codeql.yaml

@@ -18,7 +18,7 @@ jobs:
       contents: read
       security-events: write
     container:
-      image: ghcr.io/emqx/emqx-builder/5.1-1:1.14.5-25.3.2-1-ubuntu22.04
+      image: ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu22.04
 
     strategy:
       fail-fast: false

.github/workflows/performance_test.yaml

@@ -23,7 +23,7 @@ jobs:
   prepare:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'emqx'
-    container: ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-ubuntu20.04
     outputs:
       BENCH_ID: ${{ steps.prepare.outputs.BENCH_ID }}
       PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }}

.github/workflows/release.yaml

@@ -59,7 +59,7 @@ jobs:
         with:
           asset_paths: '["packages/*"]'
       - name: update to emqx.io
-        if: startsWith(github.ref_name, 'v') && (github.event_name == 'release' || inputs.publish_release_artefacts)
+        if: startsWith(github.ref_name, 'v') && ((github.event_name == 'release' && !github.event.prerelease) || inputs.publish_release_artefacts)
         run: |
           set -eux
           curl -w %{http_code} \
@@ -70,6 +70,7 @@ jobs:
                -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
                ${{ secrets.EMQX_IO_RELEASE_API }}
       - name: Push to packagecloud.io
+        if: (github.event_name == 'release' && !github.event.prerelease) || inputs.publish_release_artefacts
         env:
           PROFILE: ${{ steps.profile.outputs.profile }}
           VERSION: ${{ steps.profile.outputs.version }}

.github/workflows/run_emqx_app_tests.yaml

@@ -0,0 +1,65 @@
+name: Check emqx app standalone
+
+# These tests are needed because we provide the `emqx` application as a standalone
+# dependency for plugins.
+
+concurrency:
+  group: test-standalone-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        required: true
+        type: string
+      builder:
+        required: true
+        type: string
+      before_ref:
+        required: true
+        type: string
+      after_ref:
+        required: true
+        type: string
+
+env:
+  IS_CI: "yes"
+
+jobs:
+  run_emqx_app_tests:
+    runs-on: ${{ inputs.runner }}
+    container: ${{ inputs.builder }}
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: run
+      env:
+        BEFORE_REF: ${{ inputs.before_ref }}
+        AFTER_REF: ${{ inputs.after_ref }}
+      run: |
+        git config --global --add safe.directory "$GITHUB_WORKSPACE"
+        changed_files="$(git diff --name-only ${BEFORE_REF} ${AFTER_REF} apps/emqx)"
+        if [ "$changed_files" = '' ]; then
+          echo "nothing changed in apps/emqx, ignored."
+          exit 0
+        fi
+        make ensure-rebar3
+        cp rebar3 apps/emqx/
+        cd apps/emqx
+        ./rebar3 xref
+        ./rebar3 dialyzer
+        ./rebar3 eunit -v
+        ./rebar3 as standalone_test ct --name 'test@127.0.0.1' -v --readable=true
+        ./rebar3 proper -d test/props
+    - uses: actions/upload-artifact@v3
+      if: failure()
+      with:
+        name: logs-${{ inputs.runner }}
+        path: apps/emqx/_build/test/logs

.github/workflows/run_helm_tests.yaml

@@ -121,7 +121,7 @@ jobs:
         path: paho.mqtt.testing
     - name: install pytest
       run: |
-        pip install pytest==7.1.2 pytest-retry
+        pip install pytest==7.1.2 pytest-retry==1.3.0
         echo "$HOME/.local/bin" >> $GITHUB_PATH
     - name: run paho test
       timeout-minutes: 10

.github/workflows/run_test_cases.yaml

@@ -29,6 +29,7 @@ env:
 jobs:
   eunit_and_proper:
     runs-on: ${{ inputs.runner }}
+    name: "eunit_and_proper (${{ matrix.profile }})"
     strategy:
       fail-fast: false
       matrix:
@@ -69,6 +70,7 @@ jobs:
 
   ct_docker:
     runs-on: ${{ inputs.runner }}
+    name: "ct_docker (${{ matrix.app }}-${{ matrix.suitegroup }})"
     strategy:
       fail-fast: false
       matrix:
@@ -116,6 +118,7 @@ jobs:
 
   ct:
     runs-on: ${{ inputs.runner }}
+    name: "ct (${{ matrix.app }}-${{ matrix.suitegroup }})"
     strategy:
       fail-fast: false
       matrix:
@@ -155,6 +158,17 @@ jobs:
           name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
           path: _build/test/logs
 
+  tests_passed:
+    needs:
+      - eunit_and_proper
+      - ct
+      - ct_docker
+    runs-on: ${{ inputs.runner }}
+    strategy:
+      fail-fast: false
+    steps:
+      - run: echo "All tests passed"
+
   make_cover:
     needs:
       - eunit_and_proper

.github/workflows/static_checks.yaml

@@ -23,6 +23,7 @@ env:
 jobs:
   static_checks:
     runs-on: ${{ inputs.runner }}
+    name: "static_checks (${{ matrix.profile }})"
     strategy:
       fail-fast: false
       matrix:

.github/workflows/upload-helm-charts.yaml

@@ -42,7 +42,8 @@ jobs:
               echo "version=$(./pkg-vsn.sh emqx-enterprise)" >> $GITHUB_OUTPUT
               ;;
           esac
-      - uses: emqx/push-helm-action@v1
+      - uses: emqx/push-helm-action@v1.1
+        if: github.event_name == 'release' && !github.event.prerelease
         with:
           charts_dir: "${{ github.workspace }}/deploy/charts/${{ steps.profile.outputs.profile }}"
           version: ${{ steps.profile.outputs.version }}

.tool-versions

@@ -1,2 +1,2 @@
-erlang 25.3.2-1
+erlang 25.3.2-2
 elixir 1.14.5-otp-25

Makefile

@@ -2,7 +2,7 @@ REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
 export EMQX_RELUP ?= true
-export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.1-3:1.14.5-25.3.2-1-debian11
+export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-debian11
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
@@ -16,7 +16,7 @@ endif
 # Dashboard version
 # from https://github.com/emqx/emqx-dashboard5
 export EMQX_DASHBOARD_VERSION ?= v1.3.2
-export EMQX_EE_DASHBOARD_VERSION ?= e1.1.1
+export EMQX_EE_DASHBOARD_VERSION ?= e1.2.0-beta.4
 
 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used
 # In make 4.4+, for backward-compatibility the value from the original environment is used.

README-CN.md

@@ -1,7 +1,7 @@
 # EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)
-[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml)
+[![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml)
 [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
 [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)

README-RU.md

@@ -1,7 +1,7 @@
 # Брокер EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)
-[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml)
+[![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml)
 [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
 [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)

README.md

@@ -1,7 +1,7 @@
 # EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)
-[![Build Status](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/run_test_cases.yaml)
+[![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml)
 [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
 [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)

apps/emqx/include/emqx_mqtt.hrl

@@ -679,4 +679,8 @@ end).
 -define(THROW_FRAME_ERROR(Reason), erlang:throw({?FRAME_PARSE_ERROR, Reason})).
 -define(THROW_SERIALIZE_ERROR(Reason), erlang:throw({?FRAME_SERIALIZE_ERROR, Reason})).
 
+-define(MAX_PAYLOAD_FORMAT_SIZE, 1024).
+-define(TRUNCATED_PAYLOAD_SIZE, 100).
+-define(MAX_PAYLOAD_FORMAT_LIMIT(Bin), (byte_size(Bin) =< ?MAX_PAYLOAD_FORMAT_SIZE)).
+
 -endif.

apps/emqx/include/emqx_release.hrl

@@ -32,10 +32,10 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Opensource edition
--define(EMQX_RELEASE_CE, "5.1.5-patch.2").
+-define(EMQX_RELEASE_CE, "5.1.6").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.1.1").
+-define(EMQX_RELEASE_EE, "5.2.0-alpha.3").
 
 %% The HTTP API version
 -define(EMQX_API_VERSION, "5.0").

apps/emqx/rebar.config

@@ -30,7 +30,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.10"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.14"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.16"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
@@ -41,6 +41,16 @@
 {extra_src_dirs, [{"etc", [recursive]}]}.
 {profiles, [
     {test, [
+        {deps, [
+            {meck, "0.9.2"},
+            {proper, "1.4.0"},
+            {bbmustache, "1.10.0"},
+            {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.6"}}}
+        ]},
+        {extra_src_dirs, [{"test", [recursive]},
+                          {"integration_test", [recursive]}]}
+    ]},
+    {standalone_test, [
         {deps, [
             {meck, "0.9.2"},
             {proper, "1.4.0"},

apps/emqx/src/emqx.app.src

@@ -2,7 +2,7 @@
 {application, emqx, [
     {id, "emqx"},
     {description, "EMQX Core"},
-    {vsn, "5.1.6"},
+    {vsn, "5.1.8"},
     {modules, []},
     {registered, []},
     {applications, [
@@ -14,7 +14,6 @@
         esockd,
         cowboy,
         sasl,
-        os_mon,
         lc,
         hocon,
         emqx_durable_storage

apps/emqx/src/emqx_alarm.erl

@@ -213,7 +213,7 @@ format(Node, #deactivated_alarm{
 
 to_rfc3339(Timestamp) ->
     %% rfc3339 accuracy to millisecond
-    list_to_binary(calendar:system_time_to_rfc3339(Timestamp div 1000, [{unit, millisecond}])).
+    emqx_utils_calendar:epoch_to_rfc3339(Timestamp div 1000).
 
 %%--------------------------------------------------------------------
 %% gen_server callbacks

apps/emqx/src/emqx_banned.erl

@@ -38,7 +38,8 @@
     delete/1,
     info/1,
     format/1,
-    parse/1
+    parse/1,
+    clear/0
 ]).
 
 %% gen_server callbacks
@@ -171,7 +172,7 @@ maybe_format_host({As, Who}) ->
     {As, Who}.
 
 to_rfc3339(Timestamp) ->
-    list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])).
+    emqx_utils_calendar:epoch_to_rfc3339(Timestamp, second).
 
 -spec create(emqx_types:banned() | map()) ->
     {ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}.
@@ -226,6 +227,10 @@ delete(Who) ->
 info(InfoKey) ->
     mnesia:table_info(?BANNED_TAB, InfoKey).
 
+clear() ->
+    _ = mria:clear_table(?BANNED_TAB),
+    ok.
+
 %%--------------------------------------------------------------------
 %% gen_server callbacks
 %%--------------------------------------------------------------------

apps/emqx/src/emqx_datetime.erl

@@ -1,137 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
--module(emqx_datetime).
-
--include_lib("typerefl/include/types.hrl").
-
-%% API
--export([
-    to_epoch_millisecond/1,
-    to_epoch_second/1,
-    human_readable_duration_string/1
-]).
--export([
-    epoch_to_rfc3339/1,
-    epoch_to_rfc3339/2
-]).
-
--reflect_type([
-    epoch_millisecond/0,
-    epoch_second/0
-]).
-
--type epoch_second() :: non_neg_integer().
--type epoch_millisecond() :: non_neg_integer().
--typerefl_from_string({epoch_second/0, ?MODULE, to_epoch_second}).
--typerefl_from_string({epoch_millisecond/0, ?MODULE, to_epoch_millisecond}).
-
-to_epoch_second(DateTime) ->
-    to_epoch(DateTime, second).
-
-to_epoch_millisecond(DateTime) ->
-    to_epoch(DateTime, millisecond).
-
-to_epoch(DateTime, Unit) ->
-    try
-        case string:to_integer(DateTime) of
-            {Epoch, []} when Epoch >= 0 -> {ok, Epoch};
-            {_Epoch, []} -> {error, bad_epoch};
-            _ -> {ok, calendar:rfc3339_to_system_time(DateTime, [{unit, Unit}])}
-        end
-    catch
-        error:_ ->
-            {error, bad_rfc3339_timestamp}
-    end.
-
-epoch_to_rfc3339(TimeStamp) ->
-    epoch_to_rfc3339(TimeStamp, millisecond).
-
-epoch_to_rfc3339(TimeStamp, Unit) when is_integer(TimeStamp) ->
-    list_to_binary(calendar:system_time_to_rfc3339(TimeStamp, [{unit, Unit}])).
-
--spec human_readable_duration_string(integer()) -> string().
-human_readable_duration_string(Milliseconds) ->
-    Seconds = Milliseconds div 1000,
-    {D, {H, M, S}} = calendar:seconds_to_daystime(Seconds),
-    L0 = [{D, " days"}, {H, " hours"}, {M, " minutes"}, {S, " seconds"}],
-    L1 = lists:dropwhile(fun({K, _}) -> K =:= 0 end, L0),
-    L2 = lists:map(fun({Time, Unit}) -> [integer_to_list(Time), Unit] end, L1),
-    lists:flatten(lists:join(", ", L2)).
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
--compile(nowarn_export_all).
--compile(export_all).
-roots() -> [bar].
-
-fields(bar) ->
-    [
-        {second, ?MODULE:epoch_second()},
-        {millisecond, ?MODULE:epoch_millisecond()}
-    ].
-
--define(FORMAT(_Sec_, _Ms_),
-    lists:flatten(
-        io_lib:format("bar={second=~w,millisecond=~w}", [_Sec_, _Ms_])
-    )
-).
-
-epoch_ok_test() ->
-    Args = [
-        {0, 0, 0, 0},
-        {1, 1, 1, 1},
-        {"2022-01-01T08:00:00+08:00", "2022-01-01T08:00:00+08:00", 1640995200, 1640995200000}
-    ],
-    lists:foreach(
-        fun({Sec, Ms, EpochSec, EpochMs}) ->
-            check_ok(?FORMAT(Sec, Ms), EpochSec, EpochMs)
-        end,
-        Args
-    ),
-    ok.
-
-check_ok(Input, Sec, Ms) ->
-    {ok, Data} = hocon:binary(Input, #{}),
-    ?assertMatch(
-        #{bar := #{second := Sec, millisecond := Ms}},
-        hocon_tconf:check_plain(?MODULE, Data, #{atom_key => true}, [bar])
-    ),
-    ok.
-
-epoch_failed_test() ->
-    Args = [
-        {-1, -1},
-        {"1s", "1s"},
-        {"2022-13-13T08:00:00+08:00", "2022-13-13T08:00:00+08:00"}
-    ],
-    lists:foreach(
-        fun({Sec, Ms}) ->
-            check_failed(?FORMAT(Sec, Ms))
-        end,
-        Args
-    ),
-    ok.
-
-check_failed(Input) ->
-    {ok, Data} = hocon:binary(Input, #{}),
-    ?assertException(
-        throw,
-        _,
-        hocon_tconf:check_plain(?MODULE, Data, #{atom_key => true}, [bar])
-    ),
-    ok.
-
--endif.

apps/emqx/src/emqx_limiter/src/emqx_htb_limiter.erl

@@ -139,7 +139,7 @@ make_local_limiter(Cfg, Bucket) ->
         tokens => emqx_limiter_server:get_initial_val(Cfg),
         lasttime => ?NOW,
         bucket => Bucket,
-        capacity => emqx_limiter_schema:calc_capacity(Cfg)
+        capacity => emqx_limiter_utils:calc_capacity(Cfg)
     }.
 
 %%@doc create a limiter server's reference

apps/emqx/src/emqx_limiter/src/emqx_limiter_decimal.erl

@@ -24,13 +24,19 @@
     sub/2,
     mul/2,
     put_to_counter/3,
-    floor_div/2
+    floor_div/2,
+    precisely_add/2
 ]).
--export_type([decimal/0, zero_or_float/0]).
+-export_type([decimal/0, zero_or_float/0, correction_value/0]).
 
 -type decimal() :: infinity | number().
 -type zero_or_float() :: 0 | float().
 
+-type correction_value() :: #{
+    correction := emqx_limiter_decimal:zero_or_float(),
+    any() => any()
+}.
+
 %%--------------------------------------------------------------------
 %%% API
 %%--------------------------------------------------------------------
@@ -43,6 +49,12 @@ add(A, B) when
 add(A, B) ->
     A + B.
 
+-spec precisely_add(number(), correction_value()) -> {integer(), correction_value()}.
+precisely_add(Inc, #{correction := Correction} = Data) ->
+    FixedInc = Inc + Correction,
+    IntInc = erlang:floor(FixedInc),
+    {IntInc, Data#{correction := FixedInc - IntInc}}.
+
 -spec sub(decimal(), decimal()) -> decimal().
 sub(A, B) when
     A =:= infinity orelse

apps/emqx/src/emqx_limiter/src/emqx_limiter_manager.erl

@@ -131,7 +131,7 @@ delete_root(Type) ->
     delete_bucket(?ROOT_ID, Type).
 
 post_config_update([limiter], _Config, NewConf, _OldConf, _AppEnvs) ->
-    Conf = emqx_limiter_schema:convert_node_opts(NewConf),
+    Conf = emqx_limiter_utils:convert_node_opts(NewConf),
     _ = [on_post_config_update(Type, Cfg) || {Type, Cfg} <- maps:to_list(Conf)],
     ok.
 

apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl

@@ -33,14 +33,7 @@
     desc/1,
     types/0,
     short_paths/0,
-    calc_capacity/1,
+    short_paths_fields/1
-    extract_with_type/2,
-    default_client_config/0,
-    default_bucket_config/0,
-    short_paths_fields/1,
-    get_listener_opts/1,
-    get_node_opts/1,
-    convert_node_opts/1
 ]).
 
 -define(KILOBYTE, 1024).
@@ -263,80 +256,6 @@ types() ->
 short_paths() ->
     [max_conn_rate, messages_rate, bytes_rate].
 
-calc_capacity(#{rate := infinity}) ->
-    infinity;
-calc_capacity(#{rate := Rate, burst := Burst}) ->
-    erlang:floor(1000 * Rate / default_period()) + Burst.
-
-extract_with_type(_Type, undefined) ->
-    undefined;
-extract_with_type(Type, #{client := ClientCfg} = BucketCfg) ->
-    BucketVal = maps:find(Type, BucketCfg),
-    ClientVal = maps:find(Type, ClientCfg),
-    merge_client_bucket(Type, ClientVal, BucketVal);
-extract_with_type(Type, BucketCfg) ->
-    BucketVal = maps:find(Type, BucketCfg),
-    merge_client_bucket(Type, undefined, BucketVal).
-
-%% Since the client configuration can be absent and be a undefined value,
-%% but we must need some basic settings to control the behaviour of the limiter,
-%% so here add this helper function to generate a default setting.
-%% This is a temporary workaround until we found a better way to simplify.
-default_client_config() ->
-    #{
-        rate => infinity,
-        initial => 0,
-        low_watermark => 0,
-        burst => 0,
-        divisible => true,
-        max_retry_time => timer:hours(1),
-        failure_strategy => force
-    }.
-
-default_bucket_config() ->
-    #{
-        rate => infinity,
-        burst => 0,
-        initial => 0
-    }.
-
-get_listener_opts(Conf) ->
-    Limiter = maps:get(limiter, Conf, undefined),
-    ShortPaths = maps:with(short_paths(), Conf),
-    get_listener_opts(Limiter, ShortPaths).
-
-get_node_opts(Type) ->
-    Opts = emqx:get_config([limiter, Type], default_bucket_config()),
-    case type_to_short_path_name(Type) of
-        undefined ->
-            Opts;
-        Name ->
-            case emqx:get_config([limiter, Name], undefined) of
-                undefined ->
-                    Opts;
-                Rate ->
-                    Opts#{rate := Rate}
-            end
-    end.
-
-convert_node_opts(Conf) ->
-    DefBucket = default_bucket_config(),
-    ShorPaths = short_paths(),
-    Fun = fun
-        %% The `client` in the node options was deprecated
-        (client, _Value, Acc) ->
-            Acc;
-        (Name, Value, Acc) ->
-            case lists:member(Name, ShorPaths) of
-                true ->
-                    Type = short_path_name_to_type(Name),
-                    Acc#{Type => DefBucket#{rate => Value}};
-                _ ->
-                    Acc#{Name => Value}
-            end
-    end,
-    maps:fold(Fun, #{}, Conf).
-
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
@@ -538,51 +457,3 @@ alias_of_type(bytes) ->
     [bytes_in];
 alias_of_type(_) ->
     [].
-
-merge_client_bucket(Type, {ok, ClientVal}, {ok, BucketVal}) ->
-    #{Type => BucketVal, client => #{Type => ClientVal}};
-merge_client_bucket(Type, {ok, ClientVal}, _) ->
-    #{client => #{Type => ClientVal}};
-merge_client_bucket(Type, _, {ok, BucketVal}) ->
-    #{Type => BucketVal};
-merge_client_bucket(_, _, _) ->
-    undefined.
-
-short_path_name_to_type(max_conn_rate) ->
-    connection;
-short_path_name_to_type(messages_rate) ->
-    messages;
-short_path_name_to_type(bytes_rate) ->
-    bytes.
-
-type_to_short_path_name(connection) ->
-    max_conn_rate;
-type_to_short_path_name(messages) ->
-    messages_rate;
-type_to_short_path_name(bytes) ->
-    bytes_rate;
-type_to_short_path_name(_) ->
-    undefined.
-
-get_listener_opts(Limiter, ShortPaths) when map_size(ShortPaths) =:= 0 ->
-    Limiter;
-get_listener_opts(undefined, ShortPaths) ->
-    convert_listener_short_paths(ShortPaths);
-get_listener_opts(Limiter, ShortPaths) ->
-    Shorts = convert_listener_short_paths(ShortPaths),
-    emqx_utils_maps:deep_merge(Limiter, Shorts).
-
-convert_listener_short_paths(ShortPaths) ->
-    DefBucket = default_bucket_config(),
-    DefClient = default_client_config(),
-    Fun = fun(Name, Rate, Acc) ->
-        Type = short_path_name_to_type(Name),
-        case Name of
-            max_conn_rate ->
-                Acc#{Type => DefBucket#{rate => Rate}};
-            _ ->
-                Client = maps:get(client, Acc, #{}),
-                Acc#{client => Client#{Type => DefClient#{rate => Rate}}}
-        end
-    end,
-    maps:fold(Fun, #{}, ShortPaths).

apps/emqx/src/emqx_limiter/src/emqx_limiter_server.erl

@@ -383,7 +383,7 @@ longitudinal(
 
     case lists:min([ShouldAlloc, Flow, Capacity]) of
         Available when Available > 0 ->
-            {Inc, Bucket2} = emqx_limiter_correction:add(Available, Bucket),
+            {Inc, Bucket2} = emqx_limiter_decimal:precisely_add(Available, Bucket),
             counters:add(Counter, Index, Inc),
 
             {Available, Buckets#{Name := Bucket2#{obtained := Obtained + Available}}};
@@ -419,7 +419,7 @@ maybe_adjust_root_tokens(#{root := #{rate := Rate} = Root, counter := Counter} =
             State;
         _ ->
             Available = erlang:min(Rate - Token, InFlow),
-            {Inc, Root2} = emqx_limiter_correction:add(Available, Root),
+            {Inc, Root2} = emqx_limiter_decimal:precisely_add(Available, Root),
             counters:add(Counter, ?ROOT_COUNTER_IDX, Inc),
             State#{root := Root2}
     end.
@@ -473,7 +473,7 @@ dispatch_burst_to_buckets([Bucket | T], InFlow, Alloced, Buckets) ->
         index := Index,
         obtained := Obtained
     } = Bucket,
-    {Inc, Bucket2} = emqx_limiter_correction:add(InFlow, Bucket),
+    {Inc, Bucket2} = emqx_limiter_decimal:precisely_add(InFlow, Bucket),
 
     counters:add(Counter, Index, Inc),
 
@@ -484,7 +484,7 @@ dispatch_burst_to_buckets([], _, Alloced, Buckets) ->
 
 -spec init_tree(emqx_limiter_schema:limiter_type()) -> state().
 init_tree(Type) when is_atom(Type) ->
-    Cfg = emqx_limiter_schema:get_node_opts(Type),
+    Cfg = emqx_limiter_utils:get_node_opts(Type),
     init_tree(Type, Cfg).
 
 init_tree(Type, #{rate := Rate} = Cfg) ->
@@ -515,7 +515,7 @@ do_add_bucket(Id, #{rate := Rate} = Cfg, #{buckets := Buckets} = State) ->
         undefined ->
             make_bucket(Id, Cfg, State);
         Bucket ->
-            Bucket2 = Bucket#{rate := Rate, capacity := emqx_limiter_schema:calc_capacity(Cfg)},
+            Bucket2 = Bucket#{rate := Rate, capacity := emqx_limiter_utils:calc_capacity(Cfg)},
             State#{buckets := Buckets#{Id := Bucket2}}
     end.
 
@@ -536,7 +536,7 @@ make_bucket(
         rate => Rate,
         obtained => Initial,
         correction => 0,
-        capacity => emqx_limiter_schema:calc_capacity(Cfg),
+        capacity => emqx_limiter_utils:calc_capacity(Cfg),
         counter => Counter,
         index => NewIndex
     },
@@ -601,7 +601,7 @@ create_limiter_without_client(Id, Type, BucketCfg) ->
         false ->
             {ok, emqx_htb_limiter:make_infinity_limiter()};
         {ok, Bucket, RefCfg} ->
-            ClientCfg = emqx_limiter_schema:default_client_config(),
+            ClientCfg = emqx_limiter_utils:default_client_config(),
             create_limiter_with_ref(Bucket, ClientCfg, RefCfg);
         Error ->
             Error
@@ -627,7 +627,7 @@ find_referenced_bucket(Id, Type, #{rate := Rate} = Cfg) when Rate =/= infinity -
     end;
 %% this is a node-level reference
 find_referenced_bucket(_Id, Type, _) ->
-    case emqx_limiter_schema:get_node_opts(Type) of
+    case emqx_limiter_utils:get_node_opts(Type) of
         #{rate := infinity} ->
             false;
         NodeCfg ->

apps/emqx/src/emqx_limiter/src/emqx_limiter_server_sup.erl

@@ -86,7 +86,7 @@ init([]) ->
 %%  Internal functions
 %%--==================================================================
 make_child(Type) ->
-    Cfg = emqx_limiter_schema:get_node_opts(Type),
+    Cfg = emqx_limiter_utils:get_node_opts(Type),
     make_child(Type, Cfg).
 
 make_child(Type, Cfg) ->

apps/emqx/src/emqx_limiter/src/emqx_limiter_utils.erl

@@ -0,0 +1,158 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_limiter_utils).
+
+-export([
+    calc_capacity/1,
+    extract_with_type/2,
+    get_listener_opts/1,
+    get_node_opts/1,
+    convert_node_opts/1,
+    default_client_config/0,
+    default_bucket_config/0
+]).
+
+-import(emqx_limiter_schema, [default_period/0, short_paths/0]).
+
+%%--------------------------------------------------------------------
+%% Configuration-related runtime code
+%%--------------------------------------------------------------------
+
+calc_capacity(#{rate := infinity}) ->
+    infinity;
+calc_capacity(#{rate := Rate, burst := Burst}) ->
+    erlang:floor(1000 * Rate / default_period()) + Burst.
+
+%% @doc extract data of a type from the nested config
+extract_with_type(_Type, undefined) ->
+    undefined;
+extract_with_type(Type, #{client := ClientCfg} = BucketCfg) ->
+    BucketVal = maps:find(Type, BucketCfg),
+    ClientVal = maps:find(Type, ClientCfg),
+    merge_client_bucket(Type, ClientVal, BucketVal);
+extract_with_type(Type, BucketCfg) ->
+    BucketVal = maps:find(Type, BucketCfg),
+    merge_client_bucket(Type, undefined, BucketVal).
+
+%% @doc get the limiter configuration from the listener setting
+%% and compatible with the old version limiter schema
+get_listener_opts(Conf) ->
+    Limiter = maps:get(limiter, Conf, undefined),
+    ShortPaths = maps:with(short_paths(), Conf),
+    get_listener_opts(Limiter, ShortPaths).
+
+get_listener_opts(Limiter, ShortPaths) when map_size(ShortPaths) =:= 0 ->
+    Limiter;
+get_listener_opts(undefined, ShortPaths) ->
+    convert_listener_short_paths(ShortPaths);
+get_listener_opts(Limiter, ShortPaths) ->
+    Shorts = convert_listener_short_paths(ShortPaths),
+    emqx_utils_maps:deep_merge(Limiter, Shorts).
+
+convert_listener_short_paths(ShortPaths) ->
+    DefBucket = default_bucket_config(),
+    DefClient = default_client_config(),
+    Fun = fun(Name, Rate, Acc) ->
+        Type = short_path_name_to_type(Name),
+        case Name of
+            max_conn_rate ->
+                Acc#{Type => DefBucket#{rate => Rate}};
+            _ ->
+                Client = maps:get(client, Acc, #{}),
+                Acc#{client => Client#{Type => DefClient#{rate => Rate}}}
+        end
+    end,
+    maps:fold(Fun, #{}, ShortPaths).
+
+%% @doc get the node-level limiter configuration and compatible with the old version limiter schema
+get_node_opts(Type) ->
+    Opts = emqx:get_config([limiter, Type], default_bucket_config()),
+    case type_to_short_path_name(Type) of
+        undefined ->
+            Opts;
+        Name ->
+            case emqx:get_config([limiter, Name], undefined) of
+                undefined ->
+                    Opts;
+                Rate ->
+                    Opts#{rate := Rate}
+            end
+    end.
+
+convert_node_opts(Conf) ->
+    DefBucket = default_bucket_config(),
+    ShorPaths = short_paths(),
+    Fun = fun
+        %% The `client` in the node options was deprecated
+        (client, _Value, Acc) ->
+            Acc;
+        (Name, Value, Acc) ->
+            case lists:member(Name, ShorPaths) of
+                true ->
+                    Type = short_path_name_to_type(Name),
+                    Acc#{Type => DefBucket#{rate => Value}};
+                _ ->
+                    Acc#{Name => Value}
+            end
+    end,
+    maps:fold(Fun, #{}, Conf).
+
+merge_client_bucket(Type, {ok, ClientVal}, {ok, BucketVal}) ->
+    #{Type => BucketVal, client => #{Type => ClientVal}};
+merge_client_bucket(Type, {ok, ClientVal}, _) ->
+    #{client => #{Type => ClientVal}};
+merge_client_bucket(Type, _, {ok, BucketVal}) ->
+    #{Type => BucketVal};
+merge_client_bucket(_, _, _) ->
+    undefined.
+
+short_path_name_to_type(max_conn_rate) ->
+    connection;
+short_path_name_to_type(messages_rate) ->
+    messages;
+short_path_name_to_type(bytes_rate) ->
+    bytes.
+
+type_to_short_path_name(connection) ->
+    max_conn_rate;
+type_to_short_path_name(messages) ->
+    messages_rate;
+type_to_short_path_name(bytes) ->
+    bytes_rate;
+type_to_short_path_name(_) ->
+    undefined.
+
+%% Since the client configuration can be absent and be a undefined value,
+%% but we must need some basic settings to control the behaviour of the limiter,
+%% so here add this helper function to generate a default setting.
+%% This is a temporary workaround until we found a better way to simplify.
+default_client_config() ->
+    #{
+        rate => infinity,
+        initial => 0,
+        low_watermark => 0,
+        burst => 0,
+        divisible => true,
+        max_retry_time => timer:hours(1),
+        failure_strategy => force
+    }.
+
+default_bucket_config() ->
+    #{
+        rate => infinity,
+        burst => 0,
+        initial => 0
+    }.

apps/emqx/src/emqx_listeners.erl

@@ -587,7 +587,7 @@ esockd_opts(ListenerId, Type, Opts0) ->
     Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0),
     Limiter = limiter(Opts0),
     Opts2 =
-        case emqx_limiter_schema:extract_with_type(connection, Limiter) of
+        case emqx_limiter_utils:extract_with_type(connection, Limiter) of
             undefined ->
                 Opts1;
             BucketCfg ->
@@ -732,7 +732,7 @@ zone(Opts) ->
     maps:get(zone, Opts, undefined).
 
 limiter(Opts) ->
-    emqx_limiter_schema:get_listener_opts(Opts).
+    emqx_limiter_utils:get_listener_opts(Opts).
 
 add_limiter_bucket(_Id, undefined) ->
     ok;

apps/emqx/src/emqx_os_mon.erl

@@ -38,15 +38,14 @@
 %% gen_server callbacks
 -export([
     init/1,
+    handle_continue/2,
     handle_call/3,
     handle_cast/2,
     handle_info/2,
     terminate/2,
     code_change/3
 ]).
--ifdef(TEST).
+-export([is_os_check_supported/0]).
--export([is_sysmem_check_supported/0]).
--endif.
 
 -include("emqx.hrl").
 
@@ -56,7 +55,7 @@ start_link() ->
     gen_server:start_link({local, ?OS_MON}, ?MODULE, [], []).
 
 update(OS) ->
-    erlang:send(?MODULE, {monitor_conf_update, OS}).
+    gen_server:cast(?MODULE, {monitor_conf_update, OS}).
 
 %%--------------------------------------------------------------------
 %% API
@@ -83,12 +82,17 @@ current_sysmem_percent() ->
 %%--------------------------------------------------------------------
 
 init([]) ->
+    {ok, undefined, {continue, setup}}.
+
+handle_continue(setup, undefined) ->
+    %% start os_mon temporarily
+    {ok, _} = application:ensure_all_started(os_mon),
     %% memsup is not reliable, ignore
     memsup:set_sysmem_high_watermark(1.0),
     SysHW = init_os_monitor(),
     MemRef = start_mem_check_timer(),
     CpuRef = start_cpu_check_timer(),
-    {ok, #{sysmem_high_watermark => SysHW, mem_time_ref => MemRef, cpu_time_ref => CpuRef}}.
+    {noreply, #{sysmem_high_watermark => SysHW, mem_time_ref => MemRef, cpu_time_ref => CpuRef}}.
 
 init_os_monitor() ->
     init_os_monitor(emqx:get_config([sysmon, os])).
@@ -110,6 +114,12 @@ handle_call({set_sysmem_high_watermark, New}, _From, #{sysmem_high_watermark :=
 handle_call(Req, _From, State) ->
     {reply, {error, {unexpected_call, Req}}, State}.
 
+handle_cast({monitor_conf_update, OS}, State) ->
+    cancel_outdated_timer(State),
+    SysHW = init_os_monitor(OS),
+    MemRef = start_mem_check_timer(),
+    CpuRef = start_cpu_check_timer(),
+    {noreply, #{sysmem_high_watermark => SysHW, mem_time_ref => MemRef, cpu_time_ref => CpuRef}};
 handle_cast(Msg, State) ->
     ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
     {noreply, State}.
@@ -151,12 +161,6 @@ handle_info({timeout, _Timer, cpu_check}, State) ->
     end,
     Ref = start_cpu_check_timer(),
     {noreply, State#{cpu_time_ref => Ref}};
-handle_info({monitor_conf_update, OS}, State) ->
-    cancel_outdated_timer(State),
-    SysHW = init_os_monitor(OS),
-    MemRef = start_mem_check_timer(),
-    CpuRef = start_cpu_check_timer(),
-    {noreply, #{sysmem_high_watermark => SysHW, mem_time_ref => MemRef, cpu_time_ref => CpuRef}};
 handle_info(Info, State) ->
     ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {noreply, State}.
@@ -182,12 +186,12 @@ start_cpu_check_timer() ->
         _ -> start_timer(Interval, cpu_check)
     end.
 
-is_sysmem_check_supported() ->
+is_os_check_supported() ->
     {unix, linux} =:= os:type().
 
 start_mem_check_timer() ->
     Interval = emqx:get_config([sysmon, os, mem_check_interval]),
-    case is_integer(Interval) andalso is_sysmem_check_supported() of
+    case is_integer(Interval) andalso is_os_check_supported() of
         true ->
             start_timer(Interval, mem_check);
         false ->
@@ -205,7 +209,7 @@ update_mem_alarm_status(HWM) when HWM > 1.0 orelse HWM < 0.0 ->
         <<"Deactivated mem usage alarm due to out of range threshold">>
     );
 update_mem_alarm_status(HWM) ->
-    is_sysmem_check_supported() andalso
+    is_os_check_supported() andalso
         do_update_mem_alarm_status(HWM),
     ok.
 

apps/emqx/src/emqx_packet.erl

@@ -55,7 +55,7 @@
     format/2
 ]).
 
--export([encode_hex/1]).
+-export([format_truncated_payload/3]).
 
 -define(TYPE_NAMES,
     {'CONNECT', 'CONNACK', 'PUBLISH', 'PUBACK', 'PUBREC', 'PUBREL', 'PUBCOMP', 'SUBSCRIBE',
@@ -616,9 +616,32 @@ format_password(undefined) -> "";
 format_password(<<>>) -> "";
 format_password(_Password) -> "******".
 
-format_payload(Payload, text) -> ["Payload=", io_lib:format("~ts", [Payload])];
+format_payload(_, hidden) ->
-format_payload(Payload, hex) -> ["Payload(hex)=", encode_hex(Payload)];
+    "Payload=******";
-format_payload(_, hidden) -> "Payload=******".
+format_payload(Payload, text) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) ->
+    ["Payload=", unicode:characters_to_list(Payload)];
+format_payload(Payload, hex) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) ->
+    ["Payload(hex)=", binary:encode_hex(Payload)];
+format_payload(<<Part:?TRUNCATED_PAYLOAD_SIZE/binary, _/binary>> = Payload, Type) ->
+    [
+        "Payload=",
+        format_truncated_payload(Part, byte_size(Payload), Type)
+    ].
+
+format_truncated_payload(Bin, Size, Type) ->
+    Bin2 =
+        case Type of
+            text -> Bin;
+            hex -> binary:encode_hex(Bin)
+        end,
+    unicode:characters_to_list(
+        [
+            Bin2,
+            "... The ",
+            integer_to_list(Size - ?TRUNCATED_PAYLOAD_SIZE),
+            " bytes of this log are truncated"
+        ]
+    ).
 
 i(true) -> 1;
 i(false) -> 0;
@@ -641,71 +664,3 @@ format_topic_filters(Filters) ->
         ),
         "]"
     ].
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%% Hex encoding functions
-%% Copy from binary:encode_hex/1 (was only introduced in OTP24).
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
--define(HEX(X), (hex(X)):16).
--compile({inline, [hex/1]}).
--spec encode_hex(Bin) -> Bin2 when
-    Bin :: binary(),
-    Bin2 :: <<_:_*16>>.
-encode_hex(Data) when byte_size(Data) rem 8 =:= 0 ->
-    <<
-        <<?HEX(A), ?HEX(B), ?HEX(C), ?HEX(D), ?HEX(E), ?HEX(F), ?HEX(G), ?HEX(H)>>
-     || <<A, B, C, D, E, F, G, H>> <= Data
-    >>;
-encode_hex(Data) when byte_size(Data) rem 7 =:= 0 ->
-    <<
-        <<?HEX(A), ?HEX(B), ?HEX(C), ?HEX(D), ?HEX(E), ?HEX(F), ?HEX(G)>>
-     || <<A, B, C, D, E, F, G>> <= Data
-    >>;
-encode_hex(Data) when byte_size(Data) rem 6 =:= 0 ->
-    <<<<?HEX(A), ?HEX(B), ?HEX(C), ?HEX(D), ?HEX(E), ?HEX(F)>> || <<A, B, C, D, E, F>> <= Data>>;
-encode_hex(Data) when byte_size(Data) rem 5 =:= 0 ->
-    <<<<?HEX(A), ?HEX(B), ?HEX(C), ?HEX(D), ?HEX(E)>> || <<A, B, C, D, E>> <= Data>>;
-encode_hex(Data) when byte_size(Data) rem 4 =:= 0 ->
-    <<<<?HEX(A), ?HEX(B), ?HEX(C), ?HEX(D)>> || <<A, B, C, D>> <= Data>>;
-encode_hex(Data) when byte_size(Data) rem 3 =:= 0 ->
-    <<<<?HEX(A), ?HEX(B), ?HEX(C)>> || <<A, B, C>> <= Data>>;
-encode_hex(Data) when byte_size(Data) rem 2 =:= 0 ->
-    <<<<?HEX(A), ?HEX(B)>> || <<A, B>> <= Data>>;
-encode_hex(Data) when is_binary(Data) ->
-    <<<<?HEX(N)>> || <<N>> <= Data>>;
-encode_hex(Bin) ->
-    erlang:error(badarg, [Bin]).
-
-hex(X) ->
-    element(
-        X + 1,
-        {16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039,
-            16#3041, 16#3042, 16#3043, 16#3044, 16#3045, 16#3046, 16#3130, 16#3131, 16#3132,
-            16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3141, 16#3142,
-            16#3143, 16#3144, 16#3145, 16#3146, 16#3230, 16#3231, 16#3232, 16#3233, 16#3234,
-            16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3241, 16#3242, 16#3243, 16#3244,
-            16#3245, 16#3246, 16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336,
-            16#3337, 16#3338, 16#3339, 16#3341, 16#3342, 16#3343, 16#3344, 16#3345, 16#3346,
-            16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438,
-            16#3439, 16#3441, 16#3442, 16#3443, 16#3444, 16#3445, 16#3446, 16#3530, 16#3531,
-            16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3541,
-            16#3542, 16#3543, 16#3544, 16#3545, 16#3546, 16#3630, 16#3631, 16#3632, 16#3633,
-            16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3641, 16#3642, 16#3643,
-            16#3644, 16#3645, 16#3646, 16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735,
-            16#3736, 16#3737, 16#3738, 16#3739, 16#3741, 16#3742, 16#3743, 16#3744, 16#3745,
-            16#3746, 16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837,
-            16#3838, 16#3839, 16#3841, 16#3842, 16#3843, 16#3844, 16#3845, 16#3846, 16#3930,
-            16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939,
-            16#3941, 16#3942, 16#3943, 16#3944, 16#3945, 16#3946, 16#4130, 16#4131, 16#4132,
-            16#4133, 16#4134, 16#4135, 16#4136, 16#4137, 16#4138, 16#4139, 16#4141, 16#4142,
-            16#4143, 16#4144, 16#4145, 16#4146, 16#4230, 16#4231, 16#4232, 16#4233, 16#4234,
-            16#4235, 16#4236, 16#4237, 16#4238, 16#4239, 16#4241, 16#4242, 16#4243, 16#4244,
-            16#4245, 16#4246, 16#4330, 16#4331, 16#4332, 16#4333, 16#4334, 16#4335, 16#4336,
-            16#4337, 16#4338, 16#4339, 16#4341, 16#4342, 16#4343, 16#4344, 16#4345, 16#4346,
-            16#4430, 16#4431, 16#4432, 16#4433, 16#4434, 16#4435, 16#4436, 16#4437, 16#4438,
-            16#4439, 16#4441, 16#4442, 16#4443, 16#4444, 16#4445, 16#4446, 16#4530, 16#4531,
-            16#4532, 16#4533, 16#4534, 16#4535, 16#4536, 16#4537, 16#4538, 16#4539, 16#4541,
-            16#4542, 16#4543, 16#4544, 16#4545, 16#4546, 16#4630, 16#4631, 16#4632, 16#4633,
-            16#4634, 16#4635, 16#4636, 16#4637, 16#4638, 16#4639, 16#4641, 16#4642, 16#4643,
-            16#4644, 16#4645, 16#4646}
-    ).

apps/emqx/src/emqx_schema.erl

@@ -46,7 +46,6 @@
 -type timeout_duration_s() :: 0..?MAX_INT_TIMEOUT_S.
 -type timeout_duration_ms() :: 0..?MAX_INT_TIMEOUT_MS.
 -type bytesize() :: integer().
--type mqtt_max_packet_size() :: 1..?MAX_INT_MQTT_PACKET_SIZE.
 -type wordsize() :: bytesize().
 -type percent() :: float().
 -type file() :: string().
@@ -73,7 +72,6 @@
 -typerefl_from_string({timeout_duration_s/0, emqx_schema, to_timeout_duration_s}).
 -typerefl_from_string({timeout_duration_ms/0, emqx_schema, to_timeout_duration_ms}).
 -typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}).
--typerefl_from_string({mqtt_max_packet_size/0, emqx_schema, to_bytesize}).
 -typerefl_from_string({wordsize/0, emqx_schema, to_wordsize}).
 -typerefl_from_string({percent/0, emqx_schema, to_percent}).
 -typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}).
@@ -93,6 +91,7 @@
 
 -export([
     validate_heap_size/1,
+    validate_packet_size/1,
     user_lookup_fun_tr/2,
     validate_alarm_actions/1,
     validate_keepalive_multiplier/1,
@@ -154,7 +153,6 @@
     timeout_duration_s/0,
     timeout_duration_ms/0,
     bytesize/0,
-    mqtt_max_packet_size/0,
     wordsize/0,
     percent/0,
     file/0,
@@ -1584,7 +1582,7 @@ fields("sysmon_os") ->
             sc(
                 hoconsc:union([disabled, duration()]),
                 #{
-                    default => <<"60s">>,
+                    default => default_mem_check_interval(),
                     desc => ?DESC(sysmon_os_mem_check_interval)
                 }
             )},
@@ -2003,8 +2001,8 @@ filter(Opts) ->
 %% SSL listener and client.
 -spec common_ssl_opts_schema(map(), server | client) -> hocon_schema:field_schema().
 common_ssl_opts_schema(Defaults, Type) ->
-    D = fun(Field) -> maps:get(to_atom(Field), Defaults, undefined) end,
+    D = fun(Field) -> maps:get(Field, Defaults, undefined) end,
-    Df = fun(Field, Default) -> maps:get(to_atom(Field), Defaults, Default) end,
+    Df = fun(Field, Default) -> maps:get(Field, Defaults, Default) end,
     Collection = maps:get(versions, Defaults, tls_all_available),
     DefaultVersions = default_tls_vsns(Collection),
     [
@@ -2047,7 +2045,7 @@ common_ssl_opts_schema(Defaults, Type) ->
             sc(
                 hoconsc:enum([verify_peer, verify_none]),
                 #{
-                    default => Df("verify", verify_none),
+                    default => Df(verify, verify_none),
                     desc => ?DESC(common_ssl_opts_schema_verify)
                 }
             )},
@@ -2055,7 +2053,7 @@ common_ssl_opts_schema(Defaults, Type) ->
             sc(
                 boolean(),
                 #{
-                    default => Df("reuse_sessions", true),
+                    default => Df(reuse_sessions, true),
                     desc => ?DESC(common_ssl_opts_schema_reuse_sessions)
                 }
             )},
@@ -2063,7 +2061,7 @@ common_ssl_opts_schema(Defaults, Type) ->
             sc(
                 non_neg_integer(),
                 #{
-                    default => Df("depth", 10),
+                    default => Df(depth, 10),
                     desc => ?DESC(common_ssl_opts_schema_depth)
                 }
             )},
@@ -2090,7 +2088,7 @@ common_ssl_opts_schema(Defaults, Type) ->
                     validator => fun(Input) -> validate_tls_versions(Collection, Input) end
                 }
             )},
-        {"ciphers", ciphers_schema(D("ciphers"))},
+        {"ciphers", ciphers_schema(D(ciphers))},
         {"user_lookup_fun",
             sc(
                 typerefl:alias("string", any()),
@@ -2105,7 +2103,7 @@ common_ssl_opts_schema(Defaults, Type) ->
             sc(
                 boolean(),
                 #{
-                    default => Df("secure_renegotiate", true),
+                    default => Df(secure_renegotiate, true),
                     desc => ?DESC(common_ssl_opts_schema_secure_renegotiate)
                 }
             )},
@@ -2125,7 +2123,7 @@ common_ssl_opts_schema(Defaults, Type) ->
             sc(
                 duration(),
                 #{
-                    default => Df("hibernate_after", <<"5s">>),
+                    default => Df(hibernate_after, <<"5s">>),
                     desc => ?DESC(common_ssl_opts_schema_hibernate_after)
                 }
             )}
@@ -2134,15 +2132,15 @@ common_ssl_opts_schema(Defaults, Type) ->
 %% @doc Make schema for SSL listener options.
 -spec server_ssl_opts_schema(map(), boolean()) -> hocon_schema:field_schema().
 server_ssl_opts_schema(Defaults, IsRanchListener) ->
-    D = fun(Field) -> maps:get(to_atom(Field), Defaults, undefined) end,
+    D = fun(Field) -> maps:get(Field, Defaults, undefined) end,
-    Df = fun(Field, Default) -> maps:get(to_atom(Field), Defaults, Default) end,
+    Df = fun(Field, Default) -> maps:get(Field, Defaults, Default) end,
     common_ssl_opts_schema(Defaults, server) ++
         [
             {"dhfile",
                 sc(
                     string(),
                     #{
-                        default => D("dhfile"),
+                        default => D(dhfile),
                         required => false,
                         desc => ?DESC(server_ssl_opts_schema_dhfile)
                     }
@@ -2151,7 +2149,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) ->
                 sc(
                     boolean(),
                     #{
-                        default => Df("fail_if_no_peer_cert", false),
+                        default => Df(fail_if_no_peer_cert, false),
                         desc => ?DESC(server_ssl_opts_schema_fail_if_no_peer_cert)
                     }
                 )},
@@ -2159,7 +2157,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) ->
                 sc(
                     boolean(),
                     #{
-                        default => Df("honor_cipher_order", true),
+                        default => Df(honor_cipher_order, true),
                         desc => ?DESC(server_ssl_opts_schema_honor_cipher_order)
                     }
                 )},
@@ -2167,7 +2165,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) ->
                 sc(
                     boolean(),
                     #{
-                        default => Df("client_renegotiation", true),
+                        default => Df(client_renegotiation, true),
                         desc => ?DESC(server_ssl_opts_schema_client_renegotiation)
                     }
                 )},
@@ -2175,7 +2173,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) ->
                 sc(
                     duration(),
                     #{
-                        default => Df("handshake_timeout", <<"15s">>),
+                        default => Df(handshake_timeout, <<"15s">>),
                         desc => ?DESC(server_ssl_opts_schema_handshake_timeout)
                     }
                 )}
@@ -2618,6 +2616,16 @@ validate_heap_size(Siz) when is_integer(Siz) ->
 validate_heap_size(_SizStr) ->
     {error, invalid_heap_size}.
 
+validate_packet_size(Siz) when is_integer(Siz) andalso Siz < 1 ->
+    {error, #{reason => max_mqtt_packet_size_too_small, minimum => 1}};
+validate_packet_size(Siz) when is_integer(Siz) andalso Siz > ?MAX_INT_MQTT_PACKET_SIZE ->
+    Max = integer_to_list(round(?MAX_INT_MQTT_PACKET_SIZE / 1024 / 1024)) ++ "M",
+    {error, #{reason => max_mqtt_packet_size_too_large, maximum => Max}};
+validate_packet_size(Siz) when is_integer(Siz) ->
+    ok;
+validate_packet_size(_SizStr) ->
+    {error, invalid_packet_size}.
+
 validate_keepalive_multiplier(Multiplier) when
     is_number(Multiplier) andalso Multiplier >= 1.0 andalso Multiplier =< 65535.0
 ->
@@ -3380,9 +3388,10 @@ mqtt_general() ->
             )},
         {"max_packet_size",
             sc(
-                mqtt_max_packet_size(),
+                bytesize(),
                 #{
                     default => <<"1MB">>,
+                    validator => fun ?MODULE:validate_packet_size/1,
                     desc => ?DESC(mqtt_max_packet_size)
                 }
             )},
@@ -3648,3 +3657,9 @@ shared_subscription_strategy() ->
                 desc => ?DESC(broker_shared_subscription_strategy)
             }
         )}.
+
+default_mem_check_interval() ->
+    case emqx_os_mon:is_os_check_supported() of
+        true -> <<"60s">>;
+        false -> disabled
+    end.

apps/emqx/src/emqx_stats.erl

@@ -37,7 +37,8 @@
     setstat/2,
     setstat/3,
     statsfun/1,
-    statsfun/2
+    statsfun/2,
+    names/0
 ]).
 
 -export([
@@ -157,6 +158,28 @@ getstats() ->
         _ -> ets:tab2list(?TAB)
     end.
 
+names() ->
+    [
+        emqx_connections_count,
+        emqx_connections_max,
+        emqx_live_connections_count,
+        emqx_live_connections_max,
+        emqx_sessions_count,
+        emqx_sessions_max,
+        emqx_topics_count,
+        emqx_topics_max,
+        emqx_suboptions_count,
+        emqx_suboptions_max,
+        emqx_subscribers_count,
+        emqx_subscribers_max,
+        emqx_subscriptions_count,
+        emqx_subscriptions_max,
+        emqx_subscriptions_shared_count,
+        emqx_subscriptions_shared_max,
+        emqx_retained_count,
+        emqx_retained_max
+    ].
+
 %% @doc Get stats by name.
 -spec getstat(atom()) -> non_neg_integer().
 getstat(Name) ->

apps/emqx/src/emqx_sys_mon.erl

@@ -29,6 +29,7 @@
 %% gen_server callbacks
 -export([
     init/1,
+    handle_continue/2,
     handle_call/3,
     handle_cast/2,
     handle_info/2,
@@ -70,11 +71,14 @@ update(VM) ->
 
 init([]) ->
     emqx_logger:set_proc_metadata(#{sysmon => true}),
-    init_system_monitor(),
+    {ok, undefined, {continue, setup}}.
 
+handle_continue(setup, undefined) ->
+    init_system_monitor(),
     %% Monitor cluster partition event
     ekka:monitor(partition, fun handle_partition_event/1),
-    {ok, start_timer(#{timer => undefined, events => []})}.
+    NewState = start_timer(#{timer => undefined, events => []}),
+    {noreply, NewState, hibernate}.
 
 start_timer(State) ->
     State#{timer := emqx_utils:start_timer(timer:seconds(2), reset)}.

apps/emqx/src/emqx_sys_sup.erl

@@ -19,21 +19,25 @@
 -behaviour(supervisor).
 
 -export([start_link/0]).
-
 -export([init/1]).
 
 start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 init([]) ->
-    Childs = [
+    OsMon =
+        case emqx_os_mon:is_os_check_supported() of
+            true -> [child_spec(emqx_os_mon)];
+            false -> []
+        end,
+    Children =
+        [
             child_spec(emqx_sys),
             child_spec(emqx_alarm),
             child_spec(emqx_sys_mon),
-        child_spec(emqx_os_mon),
             child_spec(emqx_vm_mon)
-    ],
+        ] ++ OsMon,
-    {ok, {{one_for_one, 10, 100}, Childs}}.
+    {ok, {{one_for_one, 10, 100}, Children}}.
 
 %%--------------------------------------------------------------------
 %% Internal functions

apps/emqx/src/emqx_tls_lib.erl

@@ -62,6 +62,8 @@
     [ocsp, issuer_pem]
 ]).
 
+-define(ALLOW_EMPTY_PEM, [[<<"cacertfile">>], [cacertfile]]).
+
 %% non-empty string
 -define(IS_STRING(L), (is_list(L) andalso L =/= [] andalso is_integer(hd(L)))).
 %% non-empty list of strings
@@ -330,6 +332,13 @@ ensure_ssl_files_per_key(Dir, SSL, [KeyPath | KeyPaths], Opts) ->
 
 ensure_ssl_file(_Dir, _KeyPath, SSL, undefined, _Opts) ->
     {ok, SSL};
+ensure_ssl_file(_Dir, KeyPath, SSL, MaybePem, _Opts) when
+    MaybePem =:= "" orelse MaybePem =:= <<"">>
+->
+    case lists:member(KeyPath, ?ALLOW_EMPTY_PEM) of
+        true -> {ok, SSL};
+        false -> {error, #{reason => pem_file_path_or_string_is_required}}
+    end;
 ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, Opts) ->
     case is_valid_string(MaybePem) of
         true ->

apps/emqx/src/emqx_topic_index.erl

@@ -0,0 +1,242 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc Topic index for matching topics to topic filters.
+%%
+%% Works on top of ETS ordered_set table. Keys are tuples constructed from
+%% parsed topic filters and record IDs, wrapped in a tuple to order them
+%% strictly greater than unit tuple (`{}`). Existing table may be used if
+%% existing keys will not collide with index keys.
+%%
+%% Designed to effectively answer questions like:
+%% 1. Does any topic filter match given topic?
+%% 2. Which records are associated with topic filters matching given topic?
+%% 3. Which topic filters match given topic?
+%% 4. Which record IDs are associated with topic filters matching given topic?
+
+-module(emqx_topic_index).
+
+-export([new/0]).
+-export([insert/4]).
+-export([delete/3]).
+-export([match/2]).
+-export([matches/3]).
+
+-export([get_id/1]).
+-export([get_topic/1]).
+-export([get_record/2]).
+
+-type word() :: binary() | '+' | '#'.
+-type key(ID) :: {[word()], {ID}}.
+-type match(ID) :: key(ID).
+
+%% @doc Create a new ETS table suitable for topic index.
+%% Usable mostly for testing purposes.
+-spec new() -> ets:table().
+new() ->
+    ets:new(?MODULE, [public, ordered_set, {read_concurrency, true}]).
+
+%% @doc Insert a new entry into the index that associates given topic filter to given
+%% record ID, and attaches arbitrary record to the entry. This allows users to choose
+%% between regular and "materialized" indexes, for example.
+-spec insert(emqx_types:topic(), _ID, _Record, ets:table()) -> true.
+insert(Filter, ID, Record, Tab) ->
+    ets:insert(Tab, {{words(Filter), {ID}}, Record}).
+
+%% @doc Delete an entry from the index that associates given topic filter to given
+%% record ID. Deleting non-existing entry is not an error.
+-spec delete(emqx_types:topic(), _ID, ets:table()) -> true.
+delete(Filter, ID, Tab) ->
+    ets:delete(Tab, {words(Filter), {ID}}).
+
+%% @doc Match given topic against the index and return the first match, or `false` if
+%% no match is found.
+-spec match(emqx_types:topic(), ets:table()) -> match(_ID) | false.
+match(Topic, Tab) ->
+    {Words, RPrefix} = match_init(Topic),
+    match(Words, RPrefix, Tab).
+
+match(Words, RPrefix, Tab) ->
+    Prefix = lists:reverse(RPrefix),
+    match(ets:next(Tab, {Prefix, {}}), Prefix, Words, RPrefix, Tab).
+
+match(K, Prefix, Words, RPrefix, Tab) ->
+    case match_next(Prefix, K, Words) of
+        true ->
+            K;
+        skip ->
+            match(ets:next(Tab, K), Prefix, Words, RPrefix, Tab);
+        stop ->
+            false;
+        Matched ->
+            match_rest(Matched, Words, RPrefix, Tab)
+    end.
+
+match_rest([W1 | [W2 | _] = SLast], [W1 | [W2 | _] = Rest], RPrefix, Tab) ->
+    % NOTE
+    % Fast-forward through identical words in the topic and the last key suffixes.
+    % This should save us a few redundant `ets:next` calls at the cost of slightly
+    % more complex match patterns.
+    match_rest(SLast, Rest, [W1 | RPrefix], Tab);
+match_rest(SLast, [W | Rest], RPrefix, Tab) when is_list(SLast) ->
+    match(Rest, [W | RPrefix], Tab);
+match_rest(plus, [W | Rest], RPrefix, Tab) ->
+    % NOTE
+    % There's '+' in the key suffix, meaning we should consider 2 alternatives:
+    % 1. Match the rest of the topic as if there was '+' in the current position.
+    % 2. Skip this key and try to match the topic as it is.
+    case match(Rest, ['+' | RPrefix], Tab) of
+        Match = {_, _} ->
+            Match;
+        false ->
+            match(Rest, [W | RPrefix], Tab)
+    end;
+match_rest(_, [], _RPrefix, _Tab) ->
+    false.
+
+%% @doc Match given topic against the index and return _all_ matches.
+%% If `unique` option is given, return only unique matches by record ID.
+-spec matches(emqx_types:topic(), ets:table(), _Opts :: [unique]) -> [match(_ID)].
+matches(Topic, Tab, Opts) ->
+    {Words, RPrefix} = match_init(Topic),
+    AccIn =
+        case Opts of
+            [unique | _] -> #{};
+            [] -> []
+        end,
+    Matches = matches(Words, RPrefix, AccIn, Tab),
+    case Matches of
+        #{} -> maps:values(Matches);
+        _ -> Matches
+    end.
+
+matches(Words, RPrefix, Acc, Tab) ->
+    Prefix = lists:reverse(RPrefix),
+    matches(ets:next(Tab, {Prefix, {}}), Prefix, Words, RPrefix, Acc, Tab).
+
+matches(Words, RPrefix, K = {Filter, _}, Acc, Tab) ->
+    Prefix = lists:reverse(RPrefix),
+    case Prefix > Filter of
+        true ->
+            % NOTE: Prefix already greater than the last key seen, need to `ets:next/2`.
+            matches(ets:next(Tab, {Prefix, {}}), Prefix, Words, RPrefix, Acc, Tab);
+        false ->
+            % NOTE: Prefix is still less than or equal to the last key seen, reuse it.
+            matches(K, Prefix, Words, RPrefix, Acc, Tab)
+    end.
+
+matches(K, Prefix, Words, RPrefix, Acc, Tab) ->
+    case match_next(Prefix, K, Words) of
+        true ->
+            matches(ets:next(Tab, K), Prefix, Words, RPrefix, match_add(K, Acc), Tab);
+        skip ->
+            matches(ets:next(Tab, K), Prefix, Words, RPrefix, Acc, Tab);
+        stop ->
+            Acc;
+        Matched ->
+            % NOTE: Prserve next key on the stack to save on `ets:next/2` calls.
+            matches_rest(Matched, Words, RPrefix, K, Acc, Tab)
+    end.
+
+matches_rest([W1 | [W2 | _] = SLast], [W1 | [W2 | _] = Rest], RPrefix, K, Acc, Tab) ->
+    % NOTE
+    % Fast-forward through identical words in the topic and the last key suffixes.
+    % This should save us a few redundant `ets:next` calls at the cost of slightly
+    % more complex match patterns.
+    matches_rest(SLast, Rest, [W1 | RPrefix], K, Acc, Tab);
+matches_rest(SLast, [W | Rest], RPrefix, K, Acc, Tab) when is_list(SLast) ->
+    matches(Rest, [W | RPrefix], K, Acc, Tab);
+matches_rest(plus, [W | Rest], RPrefix, K, Acc, Tab) ->
+    % NOTE
+    % There's '+' in the key suffix, meaning we should accumulate all matches from
+    % each of 2 branches:
+    % 1. Match the rest of the topic as if there was '+' in the current position.
+    % 2. Skip this key and try to match the topic as it is.
+    NAcc = matches(Rest, ['+' | RPrefix], K, Acc, Tab),
+    matches(Rest, [W | RPrefix], K, NAcc, Tab);
+matches_rest(_, [], _RPrefix, _K, Acc, _Tab) ->
+    Acc.
+
+match_add(K = {_Filter, ID}, Acc = #{}) ->
+    % NOTE: ensuring uniqueness by record ID
+    Acc#{ID => K};
+match_add(K, Acc) ->
+    [K | Acc].
+
+match_next(Prefix, {Filter, _ID}, Suffix) ->
+    match_filter(Prefix, Filter, Suffix);
+match_next(_, '$end_of_table', _) ->
+    stop.
+
+match_filter([], [], []) ->
+    % NOTE: we matched the topic exactly
+    true;
+match_filter([], [], _Suffix) ->
+    % NOTE: we matched the prefix, but there may be more matches next
+    skip;
+match_filter([], ['#'], _Suffix) ->
+    % NOTE: naturally, '#' < '+', so this is already optimal for `match/2`
+    true;
+match_filter([], ['+' | _], _Suffix) ->
+    plus;
+match_filter([], [_H | _] = Rest, _Suffix) ->
+    Rest;
+match_filter([H | T1], [H | T2], Suffix) ->
+    match_filter(T1, T2, Suffix);
+match_filter([H1 | _], [H2 | _], _Suffix) when H2 > H1 ->
+    % NOTE: we're strictly past the prefix, no need to continue
+    stop.
+
+match_init(Topic) ->
+    case words(Topic) of
+        [W = <<"$", _/bytes>> | Rest] ->
+            % NOTE
+            % This will effectively skip attempts to match special topics to `#` or `+/...`.
+            {Rest, [W]};
+        Words ->
+            {Words, []}
+    end.
+
+%% @doc Extract record ID from the match.
+-spec get_id(match(ID)) -> ID.
+get_id({_Filter, {ID}}) ->
+    ID.
+
+%% @doc Extract topic (or topic filter) from the match.
+-spec get_topic(match(_ID)) -> emqx_types:topic().
+get_topic({Filter, _ID}) ->
+    emqx_topic:join(Filter).
+
+%% @doc Fetch the record associated with the match.
+%% NOTE: Only really useful for ETS tables where the record ID is the first element.
+-spec get_record(match(_ID), ets:table()) -> _Record.
+get_record(K, Tab) ->
+    ets:lookup_element(Tab, K, 2).
+
+%%
+
+-spec words(emqx_types:topic()) -> [word()].
+words(Topic) when is_binary(Topic) ->
+    % NOTE
+    % This is almost identical to `emqx_topic:words/1`, but it doesn't convert empty
+    % tokens to ''. This is needed to keep ordering of words consistent with what
+    % `match_filter/3` expects.
+    [word(W) || W <- emqx_topic:tokens(Topic)].
+
+-spec word(binary()) -> word().
+word(<<"+">>) -> '+';
+word(<<"#">>) -> '#';
+word(Bin) -> Bin.

apps/emqx/src/emqx_trace/emqx_trace_formatter.erl

@@ -14,6 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 -module(emqx_trace_formatter).
+-include("emqx_mqtt.hrl").
 
 -export([format/2]).
 -export([format_meta_map/1]).
@@ -27,7 +28,7 @@ format(
     #{level := debug, meta := Meta = #{trace_tag := Tag}, msg := Msg},
     #{payload_encode := PEncode}
 ) ->
-    Time = calendar:system_time_to_rfc3339(erlang:system_time(microsecond), [{unit, microsecond}]),
+    Time = emqx_utils_calendar:now_to_rfc3339(microsecond),
     ClientId = to_iolist(maps:get(clientid, Meta, "")),
     Peername = maps:get(peername, Meta, ""),
     MetaBin = format_meta(Meta, PEncode),
@@ -68,10 +69,15 @@ weight({K, _}) -> {1, K}.
 format_packet(undefined, _) -> "";
 format_packet(Packet, Encode) -> emqx_packet:format(Packet, Encode).
 
-format_payload(undefined, _) -> "";
+format_payload(undefined, _) ->
-format_payload(Payload, text) -> io_lib:format("~ts", [Payload]);
+    "";
-format_payload(Payload, hex) -> emqx_packet:encode_hex(Payload);
+format_payload(_, hidden) ->
-format_payload(_, hidden) -> "******".
+    "******";
+format_payload(Payload, text) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) ->
+    unicode:characters_to_list(Payload);
+format_payload(Payload, hex) when ?MAX_PAYLOAD_FORMAT_LIMIT(Payload) -> binary:encode_hex(Payload);
+format_payload(<<Part:?TRUNCATED_PAYLOAD_SIZE/binary, _/binary>> = Payload, Type) ->
+    emqx_packet:format_truncated_payload(Part, byte_size(Payload), Type).
 
 to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom);
 to_iolist(Int) when is_integer(Int) -> integer_to_list(Int);

apps/emqx/src/emqx_vm.erl

@@ -44,7 +44,7 @@
     get_otp_version/0
 ]).
 
--export([cpu_util/0]).
+-export([cpu_util/0, cpu_util/1]).
 
 -ifdef(TEST).
 -compile(export_all).
@@ -378,16 +378,25 @@ avg15() ->
 cpu_util() ->
     compat_windows(fun cpu_sup:util/0).
 
+cpu_util(Args) ->
+    compat_windows(fun cpu_sup:util/1, Args).
+
 compat_windows(Fun) ->
-    case os:type() of
+    case compat_windows(Fun, []) of
-        {win32, nt} ->
-            0.0;
-        _Type ->
-            case catch Fun() of
         Val when is_float(Val) -> floor(Val * 100) / 100;
         Val when is_number(Val) -> Val;
-                _Error -> 0.0
+        _ -> 0.0
+    end.
+
+compat_windows(Fun, Args) ->
+    try
+        case emqx_os_mon:is_os_check_supported() of
+            false -> 0.0;
+            true when Args =:= [] -> Fun();
+            true -> Fun(Args)
         end
+    catch
+        _:_ -> 0.0
     end.
 
 load(Avg) ->

apps/emqx/test/emqx_cth_suite.erl

@@ -55,6 +55,11 @@
 -type config() :: #{atom() => scalar() | [scalar()] | config() | [config()]}.
 -type scalar() :: atom() | number() | string() | binary().
 
+-type hookfun(R) ::
+    fun(() -> R)
+    | fun((appname()) -> R)
+    | fun((appname(), appspec_opts()) -> R).
+
 -type appspec_opts() :: #{
     %% 1. Enable loading application config
     %% If not defined or set to `false`, this step will be skipped.
@@ -70,19 +75,19 @@
     %% 3. Perform anything right before starting the application
     %% If not defined or set to `false`, this step will be skipped.
     %% Merging amounts to redefining.
-    before_start => fun(() -> _) | fun((appname()) -> _) | false,
+    before_start => hookfun(_) | false,
 
     %% 4. Starting the application
     %% If not defined or set to `true`, `application:ensure_all_started/1` is used.
     %% If custom function is used, it should return list of all applications that were started.
     %% If set to `false`, application will not be started.
     %% Merging amounts to redefining.
-    start => fun(() -> {ok, [appname()]}) | fun((appname()) -> {ok, [appname()]}) | boolean(),
+    start => hookfun({ok, [appname()]}) | boolean(),
 
     %% 5. Perform anything right after starting the application
     %% If not defined or set to `false`, this step will be skipped.
     %% Merging amounts to redefining.
-    after_start => fun(() -> _) | fun((appname()) -> _) | false
+    after_start => hookfun(_) | false
 }.
 
 %% @doc Start applications with a clean slate.
@@ -214,29 +219,30 @@ maybe_override_env(App, #{override_env := Env = [{_, _} | _]}) ->
 maybe_override_env(_App, #{}) ->
     ok.
 
-maybe_before_start(App, #{before_start := Fun}) when is_function(Fun, 1) ->
+maybe_before_start(App, #{before_start := Fun} = Opts) when is_function(Fun) ->
-    Fun(App);
+    apply_hookfun(Fun, App, Opts);
-maybe_before_start(_App, #{before_start := Fun}) when is_function(Fun, 0) ->
-    Fun();
 maybe_before_start(_App, #{}) ->
     ok.
 
 maybe_start(_App, #{start := false}) ->
     {ok, []};
-maybe_start(_App, #{start := Fun}) when is_function(Fun, 0) ->
+maybe_start(App, #{start := Fun} = Opts) when is_function(Fun) ->
-    Fun();
+    apply_hookfun(Fun, App, Opts);
-maybe_start(App, #{start := Fun}) when is_function(Fun, 1) ->
-    Fun(App);
 maybe_start(App, #{}) ->
     application:ensure_all_started(App).
 
-maybe_after_start(App, #{after_start := Fun}) when is_function(Fun, 1) ->
+maybe_after_start(App, #{after_start := Fun} = Opts) when is_function(Fun) ->
-    Fun(App);
+    apply_hookfun(Fun, App, Opts);
-maybe_after_start(_App, #{after_start := Fun}) when is_function(Fun, 0) ->
-    Fun();
 maybe_after_start(_App, #{}) ->
     ok.
 
+apply_hookfun(Fun, _App, _Opts) when is_function(Fun, 0) ->
+    Fun();
+apply_hookfun(Fun, App, _Opts) when is_function(Fun, 1) ->
+    Fun(App);
+apply_hookfun(Fun, App, Opts) when is_function(Fun, 2) ->
+    Fun(App, Opts).
+
 -spec merge_appspec(appspec_opts(), appspec_opts()) ->
     appspec_opts().
 merge_appspec(Opts1, Opts2) ->
@@ -270,7 +276,11 @@ default_appspec(ekka, _SuiteOpts) ->
     };
 default_appspec(emqx, SuiteOpts) ->
     #{
-        override_env => [{data_dir, maps:get(work_dir, SuiteOpts, "data")}]
+        override_env => [{data_dir, maps:get(work_dir, SuiteOpts, "data")}],
+        % NOTE
+        % We inform `emqx` of our config loader before starting it so that it won't
+        % overwrite everything with a default configuration.
+        before_start => fun inhibit_config_loader/2
     };
 default_appspec(emqx_authz, _SuiteOpts) ->
     #{
@@ -307,9 +317,7 @@ default_appspec(emqx_conf, SuiteOpts) ->
         % NOTE
         % We inform `emqx` of our config loader before starting `emqx_conf` so that it won't
         % overwrite everything with a default configuration.
-        before_start => fun() ->
+        before_start => fun inhibit_config_loader/2
-            emqx_app:set_config_loader(?MODULE)
-        end
     };
 default_appspec(emqx_dashboard, _SuiteOpts) ->
     #{
@@ -329,6 +337,11 @@ start_ekka() ->
     ok = emqx_common_test_helpers:start_ekka(),
     {ok, [mnesia, ekka]}.
 
+inhibit_config_loader(_App, #{config := Config}) when Config /= false ->
+    ok = emqx_app:set_config_loader(?MODULE);
+inhibit_config_loader(_App, #{}) ->
+    ok.
+
 %%
 
 -spec stop(_StartedApps :: [appname()]) ->

apps/emqx/test/emqx_flapping_SUITE.erl

@@ -20,31 +20,27 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
 
 all() -> emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_common_test_helpers:boot_modules(all),
+    Apps = emqx_cth_suite:start(
-    emqx_common_test_helpers:start_apps([]),
+        [
-    %% update global default config
+            {emqx,
-    {ok, _} = emqx:update_config(
+                "flapping_detect {"
-        [flapping_detect],
+                "\n enable = true"
-        #{
+                "\n max_count = 3"
-            <<"enable">> => true,
+                "\n window_time = 100ms"
-            <<"max_count">> => 3,
+                "\n ban_time = 2s"
-            % 0.1s
+                "\n }"}
-            <<"window_time">> => <<"100ms">>,
+        ],
-            %% 2s
+        #{work_dir => ?config(priv_dir, Config)}
-            <<"ban_time">> => <<"2s">>
-        }
     ),
-    Config.
+    [{suite_apps, Apps} | Config].
 
-end_per_suite(_Config) ->
+end_per_suite(Config) ->
-    emqx_common_test_helpers:stop_apps([]),
+    emqx_cth_suite:stop(?config(suite_apps, Config)).
-    %% Clean emqx_banned table
-    mria_mnesia:delete_schema(),
-    ok.
 
 t_detect_check(_) ->
     ClientInfo = #{

apps/emqx/test/emqx_os_mon_SUITE.erl

@@ -39,29 +39,47 @@ init_per_testcase(t_cpu_check_alarm, Config) ->
         %% 200ms
         cpu_check_interval => 200
     }),
-    ok = supervisor:terminate_child(emqx_sys_sup, emqx_os_mon),
+    restart_os_mon(),
-    {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_os_mon),
     Config;
 init_per_testcase(t_sys_mem_check_alarm, Config) ->
-    case emqx_os_mon:is_sysmem_check_supported() of
+    case emqx_os_mon:is_os_check_supported() of
         true ->
             SysMon = emqx_config:get([sysmon, os], #{}),
             emqx_config:put([sysmon, os], SysMon#{
                 sysmem_high_watermark => 0.51,
                 %% 200ms
                 mem_check_interval => 200
-            }),
+            });
-            ok = supervisor:terminate_child(emqx_sys_sup, emqx_os_mon),
-            {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_os_mon),
-            Config;
         false ->
-            Config
+            ok
-    end;
+    end,
+    restart_os_mon(),
+    Config;
 init_per_testcase(_, Config) ->
-    emqx_common_test_helpers:boot_modules(all),
+    restart_os_mon(),
-    emqx_common_test_helpers:start_apps([]),
     Config.
 
+restart_os_mon() ->
+    case emqx_os_mon:is_os_check_supported() of
+        true ->
+            ok = supervisor:terminate_child(emqx_sys_sup, emqx_os_mon),
+            {ok, _} = supervisor:restart_child(emqx_sys_sup, emqx_os_mon);
+        false ->
+            _ = supervisor:terminate_child(emqx_sys_sup, emqx_os_mon),
+            _ = supervisor:delete_child(emqx_sys_sup, emqx_os_mon),
+            %% run test on mac/windows.
+            Mod = emqx_os_mon,
+            OsMon = #{
+                id => Mod,
+                start => {Mod, start_link, []},
+                restart => permanent,
+                shutdown => 5000,
+                type => worker,
+                modules => [Mod]
+            },
+            {ok, _} = supervisor:start_child(emqx_sys_sup, OsMon)
+    end.
+
 t_api(_) ->
     ?assertEqual(0.7, emqx_os_mon:get_sysmem_high_watermark()),
     ?assertEqual(ok, emqx_os_mon:set_sysmem_high_watermark(0.8)),
@@ -81,7 +99,7 @@ t_api(_) ->
     ok.
 
 t_sys_mem_check_disable(Config) ->
-    case emqx_os_mon:is_sysmem_check_supported() of
+    case emqx_os_mon:is_os_check_supported() of
         true -> do_sys_mem_check_disable(Config);
         false -> skip
     end.
@@ -100,7 +118,7 @@ do_sys_mem_check_disable(_Config) ->
     ok.
 
 t_sys_mem_check_alarm(Config) ->
-    case emqx_os_mon:is_sysmem_check_supported() of
+    case emqx_os_mon:is_os_check_supported() of
         true -> do_sys_mem_check_alarm(Config);
         false -> skip
     end.
@@ -167,7 +185,7 @@ t_cpu_check_alarm(_) ->
         util,
         fun() -> CpuUtil end,
         fun() ->
-            timer:sleep(500),
+            timer:sleep(1000),
             Alarms = emqx_alarm:get_alarms(activated),
             ?assert(
                 emqx_vm_mon_SUITE:is_existing(high_cpu_usage, emqx_alarm:get_alarms(activated))
@@ -193,7 +211,7 @@ t_cpu_check_alarm(_) ->
             ?assert(is_binary(Msg)),
             emqx_config:put([sysmon, os, cpu_high_watermark], 1),
             emqx_config:put([sysmon, os, cpu_low_watermark], 0.96),
-            timer:sleep(500),
+            timer:sleep(800),
             ?assertNot(
                 emqx_vm_mon_SUITE:is_existing(high_cpu_usage, emqx_alarm:get_alarms(activated))
             )

apps/emqx/test/emqx_ratelimiter_SUITE.erl

@@ -589,11 +589,11 @@ t_extract_with_type(_) ->
         (Type, Cfg) ->
             IsOnly(Type, Cfg)
     end,
-    ?assertEqual(undefined, emqx_limiter_schema:extract_with_type(messages, undefined)),
+    ?assertEqual(undefined, emqx_limiter_utils:extract_with_type(messages, undefined)),
     ?assert(
         Checker(
             messages,
-            emqx_limiter_schema:extract_with_type(messages, #{
+            emqx_limiter_utils:extract_with_type(messages, #{
                 messages => #{rate => 1}, bytes => #{rate => 1}
             })
         )
@@ -601,7 +601,7 @@ t_extract_with_type(_) ->
     ?assert(
         Checker(
             messages,
-            emqx_limiter_schema:extract_with_type(messages, #{
+            emqx_limiter_utils:extract_with_type(messages, #{
                 messages => #{rate => 1},
                 bytes => #{rate => 1},
                 client => #{messages => #{rate => 2}}
@@ -611,7 +611,7 @@ t_extract_with_type(_) ->
     ?assert(
         Checker(
             messages,
-            emqx_limiter_schema:extract_with_type(messages, #{
+            emqx_limiter_utils:extract_with_type(messages, #{
                 client => #{messages => #{rate => 2}, bytes => #{rate => 1}}
             })
         )
@@ -622,7 +622,7 @@ t_add_bucket(_) ->
         #{buckets := Buckets} = sys:get_state(emqx_limiter_server:whereis(bytes)),
         ?assertEqual(Size, maps:size(Buckets), Buckets)
     end,
-    DefBucket = emqx_limiter_schema:default_bucket_config(),
+    DefBucket = emqx_limiter_utils:default_bucket_config(),
     ?assertEqual(ok, emqx_limiter_server:add_bucket(?FUNCTION_NAME, bytes, undefined)),
     Checker(0),
     ?assertEqual(ok, emqx_limiter_server:add_bucket(?FUNCTION_NAME, bytes, DefBucket)),
@@ -765,7 +765,7 @@ t_esockd_htb_consume(_) ->
 t_node_short_paths(_) ->
     CfgStr = <<"limiter {max_conn_rate = \"1000\", messages_rate = \"100\", bytes_rate = \"10\"}">>,
     ok = emqx_common_test_helpers:load_config(emqx_limiter_schema, CfgStr),
-    Accessor = fun emqx_limiter_schema:get_node_opts/1,
+    Accessor = fun emqx_limiter_utils:get_node_opts/1,
     ?assertMatch(#{rate := 100.0}, Accessor(connection)),
     ?assertMatch(#{rate := 10.0}, Accessor(messages)),
     ?assertMatch(#{rate := 1.0}, Accessor(bytes)),
@@ -776,7 +776,7 @@ t_compatibility_for_node_short_paths(_) ->
     CfgStr =
         <<"limiter {max_conn_rate = \"1000\", connection.rate = \"500\", bytes.rate = \"200\"}">>,
     ok = emqx_common_test_helpers:load_config(emqx_limiter_schema, CfgStr),
-    Accessor = fun emqx_limiter_schema:get_node_opts/1,
+    Accessor = fun emqx_limiter_utils:get_node_opts/1,
     ?assertMatch(#{rate := 100.0}, Accessor(connection)),
     ?assertMatch(#{rate := 20.0}, Accessor(bytes)).
 
@@ -796,7 +796,7 @@ t_listener_short_paths(_) ->
             },
             connection := #{rate := 100.0}
         },
-        emqx_limiter_schema:get_listener_opts(ListenerOpt)
+        emqx_limiter_utils:get_listener_opts(ListenerOpt)
     ).
 
 t_compatibility_for_listener_short_paths(_) ->
@@ -809,7 +809,7 @@ t_compatibility_for_listener_short_paths(_) ->
         #{
             connection := #{rate := 100.0}
         },
-        emqx_limiter_schema:get_listener_opts(ListenerOpt)
+        emqx_limiter_utils:get_listener_opts(ListenerOpt)
     ).
 
 t_no_limiter_for_listener(_) ->
@@ -818,7 +818,7 @@ t_no_limiter_for_listener(_) ->
     ListenerOpt = emqx:get_config([listeners, tcp, default]),
     ?assertEqual(
         undefined,
-        emqx_limiter_schema:get_listener_opts(ListenerOpt)
+        emqx_limiter_utils:get_listener_opts(ListenerOpt)
     ).
 
 %%--------------------------------------------------------------------
@@ -1135,5 +1135,5 @@ parse_schema(ConfigString) ->
     ).
 
 default_client_config() ->
-    Conf = emqx_limiter_schema:default_client_config(),
+    Conf = emqx_limiter_utils:default_client_config(),
     Conf#{divisible := false, max_retry_time := timer:seconds(10)}.

apps/emqx/test/emqx_tls_lib_tests.erl

@@ -113,11 +113,22 @@ ssl_files_failure_test_() ->
                 })
             )
         end},
+        {"empty_cacertfile", fun() ->
+            ?assertMatch(
+                {ok, _},
+                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                    <<"keyfile">> => test_key(),
+                    <<"certfile">> => test_key(),
+                    <<"cacertfile">> => <<"">>
+                })
+            )
+        end},
         {"bad_pem_string", fun() ->
             %% empty string
             ?assertMatch(
                 {error, #{
-                    reason := invalid_file_path_or_pem_string, which_options := [[<<"keyfile">>]]
+                    reason := pem_file_path_or_string_is_required,
+                    which_options := [[<<"keyfile">>]]
                 }},
                 emqx_tls_lib:ensure_ssl_files("/tmp", #{
                     <<"keyfile">> => <<>>,

apps/emqx/test/emqx_topic_index_SUITE.erl

@@ -0,0 +1,331 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_topic_index_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("proper/include/proper.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+-import(emqx_proper_types, [scaled/2]).
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+t_insert(_) ->
+    Tab = emqx_topic_index:new(),
+    true = emqx_topic_index:insert(<<"sensor/1/metric/2">>, t_insert_1, <<>>, Tab),
+    true = emqx_topic_index:insert(<<"sensor/+/#">>, t_insert_2, <<>>, Tab),
+    true = emqx_topic_index:insert(<<"sensor/#">>, t_insert_3, <<>>, Tab),
+    ?assertEqual(<<"sensor/#">>, topic(match(<<"sensor">>, Tab))),
+    ?assertEqual(t_insert_3, id(match(<<"sensor">>, Tab))).
+
+t_match(_) ->
+    Tab = emqx_topic_index:new(),
+    true = emqx_topic_index:insert(<<"sensor/1/metric/2">>, t_match_1, <<>>, Tab),
+    true = emqx_topic_index:insert(<<"sensor/+/#">>, t_match_2, <<>>, Tab),
+    true = emqx_topic_index:insert(<<"sensor/#">>, t_match_3, <<>>, Tab),
+    ?assertMatch(
+        [<<"sensor/#">>, <<"sensor/+/#">>],
+        [topic(M) || M <- matches(<<"sensor/1">>, Tab)]
+    ).
+
+t_match2(_) ->
+    Tab = emqx_topic_index:new(),
+    true = emqx_topic_index:insert(<<"#">>, t_match2_1, <<>>, Tab),
+    true = emqx_topic_index:insert(<<"+/#">>, t_match2_2, <<>>, Tab),
+    true = emqx_topic_index:insert(<<"+/+/#">>, t_match2_3, <<>>, Tab),
+    ?assertEqual(
+        [<<"#">>, <<"+/#">>, <<"+/+/#">>],
+        [topic(M) || M <- matches(<<"a/b/c">>, Tab)]
+    ),
+    ?assertEqual(
+        false,
+        emqx_topic_index:match(<<"$SYS/broker/zenmq">>, Tab)
+    ).
+
+t_match3(_) ->
+    Tab = emqx_topic_index:new(),
+    Records = [
+        {<<"d/#">>, t_match3_1},
+        {<<"a/b/+">>, t_match3_2},
+        {<<"a/#">>, t_match3_3},
+        {<<"#">>, t_match3_4},
+        {<<"$SYS/#">>, t_match3_sys}
+    ],
+    lists:foreach(
+        fun({Topic, ID}) -> emqx_topic_index:insert(Topic, ID, <<>>, Tab) end,
+        Records
+    ),
+    Matched = matches(<<"a/b/c">>, Tab),
+    case length(Matched) of
+        3 -> ok;
+        _ -> error({unexpected, Matched})
+    end,
+    ?assertEqual(
+        t_match3_sys,
+        id(match(<<"$SYS/a/b/c">>, Tab))
+    ).
+
+t_match4(_) ->
+    Tab = emqx_topic_index:new(),
+    Records = [{<<"/#">>, t_match4_1}, {<<"/+">>, t_match4_2}, {<<"/+/a/b/c">>, t_match4_3}],
+    lists:foreach(
+        fun({Topic, ID}) -> emqx_topic_index:insert(Topic, ID, <<>>, Tab) end,
+        Records
+    ),
+    ?assertEqual(
+        [<<"/#">>, <<"/+">>],
+        [topic(M) || M <- matches(<<"/">>, Tab)]
+    ),
+    ?assertEqual(
+        [<<"/#">>, <<"/+/a/b/c">>],
+        [topic(M) || M <- matches(<<"/0/a/b/c">>, Tab)]
+    ).
+
+t_match5(_) ->
+    Tab = emqx_topic_index:new(),
+    T = <<"a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z">>,
+    Records = [
+        {<<"#">>, t_match5_1},
+        {<<T/binary, "/#">>, t_match5_2},
+        {<<T/binary, "/+">>, t_match5_3}
+    ],
+    lists:foreach(
+        fun({Topic, ID}) -> emqx_topic_index:insert(Topic, ID, <<>>, Tab) end,
+        Records
+    ),
+    ?assertEqual(
+        [<<"#">>, <<T/binary, "/#">>],
+        [topic(M) || M <- matches(T, Tab)]
+    ),
+    ?assertEqual(
+        [<<"#">>, <<T/binary, "/#">>, <<T/binary, "/+">>],
+        [topic(M) || M <- matches(<<T/binary, "/1">>, Tab)]
+    ).
+
+t_match6(_) ->
+    Tab = emqx_topic_index:new(),
+    T = <<"a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z">>,
+    W = <<"+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/+/#">>,
+    emqx_topic_index:insert(W, ID = t_match6, <<>>, Tab),
+    ?assertEqual(ID, id(match(T, Tab))).
+
+t_match7(_) ->
+    Tab = emqx_topic_index:new(),
+    T = <<"a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z">>,
+    W = <<"a/+/c/+/e/+/g/+/i/+/k/+/m/+/o/+/q/+/s/+/u/+/w/+/y/+/#">>,
+    emqx_topic_index:insert(W, t_match7, <<>>, Tab),
+    ?assertEqual(W, topic(match(T, Tab))).
+
+t_match_fast_forward(_) ->
+    Tab = emqx_topic_index:new(),
+    emqx_topic_index:insert(<<"a/b/1/2/3/4/5/6/7/8/9/#">>, id1, <<>>, Tab),
+    emqx_topic_index:insert(<<"z/y/x/+/+">>, id2, <<>>, Tab),
+    emqx_topic_index:insert(<<"a/b/c/+">>, id3, <<>>, Tab),
+    % dbg:tracer(),
+    % dbg:p(all, c),
+    % dbg:tpl({ets, next, '_'}, x),
+    ?assertEqual(id1, id(match(<<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab))),
+    ?assertEqual([id1], [id(M) || M <- matches(<<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab)]).
+
+t_match_unique(_) ->
+    Tab = emqx_topic_index:new(),
+    emqx_topic_index:insert(<<"a/b/c">>, t_match_id1, <<>>, Tab),
+    emqx_topic_index:insert(<<"a/b/+">>, t_match_id1, <<>>, Tab),
+    emqx_topic_index:insert(<<"a/b/c/+">>, t_match_id2, <<>>, Tab),
+    ?assertEqual(
+        [t_match_id1, t_match_id1],
+        [id(M) || M <- emqx_topic_index:matches(<<"a/b/c">>, Tab, [])]
+    ),
+    ?assertEqual(
+        [t_match_id1],
+        [id(M) || M <- emqx_topic_index:matches(<<"a/b/c">>, Tab, [unique])]
+    ).
+
+t_match_wildcard_edge_cases(_) ->
+    CommonTopics = [
+        <<"a/b">>,
+        <<"a/b/#">>,
+        <<"a/b/#">>,
+        <<"a/b/c">>,
+        <<"a/b/+">>,
+        <<"a/b/d">>,
+        <<"a/+/+">>,
+        <<"a/+/#">>
+    ],
+    Datasets =
+        [
+            %% Topics, TopicName, Results
+            {CommonTopics, <<"a/b/c">>, [2, 3, 4, 5, 7, 8]},
+            {CommonTopics, <<"a/b">>, [1, 2, 3, 8]},
+            {[<<"+/b/c">>, <<"/">>], <<"a/b/c">>, [1]},
+            {[<<"#">>, <<"/">>], <<"a">>, [1]},
+            {[<<"/">>, <<"+">>], <<"a">>, [2]}
+        ],
+    F = fun({Topics, TopicName, Expected}) ->
+        Tab = emqx_topic_index:new(),
+        _ = [emqx_topic_index:insert(T, N, <<>>, Tab) || {N, T} <- lists:enumerate(Topics)],
+        ?assertEqual(
+            lists:last(Expected),
+            id(emqx_topic_index:match(TopicName, Tab)),
+            #{"Base topics" => Topics, "Topic name" => TopicName}
+        ),
+        ?assertEqual(
+            Expected,
+            [id(M) || M <- emqx_topic_index:matches(TopicName, Tab, [unique])],
+            #{"Base topics" => Topics, "Topic name" => TopicName}
+        )
+    end,
+    lists:foreach(F, Datasets).
+
+t_prop_matches(_) ->
+    ?assert(
+        proper:quickcheck(
+            topic_matches_prop(),
+            [{max_size, 100}, {numtests, 100}]
+        )
+    ),
+    Statistics = [{C, account(C)} || C <- [filters, topics, matches, maxhits]],
+    ct:pal("Statistics: ~p", [maps:from_list(Statistics)]).
+
+topic_matches_prop() ->
+    ?FORALL(
+        % Generate a longer list of topics and a shorter list of topic filter patterns.
+        #{
+            topics := TTopics,
+            patterns := Pats
+        },
+        emqx_proper_types:fixedmap(#{
+            % NOTE
+            % Beware adding non-empty contraint, proper will have a hard time with `topic_t/1`
+            % for some reason.
+            topics => scaled(4, list(topic_t([1, 2, 3, 4]))),
+            patterns => list(topic_filter_pattern_t())
+        }),
+        begin
+            Tab = emqx_topic_index:new(),
+            Topics = [emqx_topic:join(T) || T <- TTopics],
+            % Produce topic filters from generated topics and patterns.
+            % Number of filters is equal to the number of patterns, most of the time.
+            Filters = lists:enumerate(mk_filters(Pats, TTopics)),
+            _ = [emqx_topic_index:insert(F, N, <<>>, Tab) || {N, F} <- Filters],
+            % Gather some basic statistics
+            _ = account(filters, length(Filters)),
+            _ = account(topics, NTopics = length(Topics)),
+            _ = account(maxhits, NTopics * NTopics),
+            % Verify that matching each topic against index returns the same results as
+            % matching it against the list of filters one by one.
+            lists:all(
+                fun(Topic) ->
+                    Ids1 = [id(M) || M <- emqx_topic_index:matches(Topic, Tab, [unique])],
+                    Ids2 = lists:filtermap(
+                        fun({N, F}) ->
+                            case emqx_topic:match(Topic, F) of
+                                true -> {true, N};
+                                false -> false
+                            end
+                        end,
+                        Filters
+                    ),
+                    % Account a number of matches to compute hitrate later
+                    _ = account(matches, length(Ids1)),
+                    case (Ids2 -- Ids1) ++ (Ids2 -- Ids1) of
+                        [] ->
+                            true;
+                        [_ | _] = _Differences ->
+                            ct:pal(
+                                "Topic name: ~p~n"
+                                "Index results: ~p~n"
+                                "Topic match results:: ~p~n",
+                                [Topic, Ids1, Ids2]
+                            ),
+                            false
+                    end
+                end,
+                Topics
+            )
+        end
+    ).
+
+mk_filters([Pat | PRest], [Topic | TRest]) ->
+    [emqx_topic:join(mk_topic_filter(Pat, Topic)) | mk_filters(PRest, TRest)];
+mk_filters(_, _) ->
+    [].
+
+account(Counter, N) ->
+    put({?MODULE, Counter}, account(Counter) + N).
+
+account(Counter) ->
+    emqx_maybe:define(get({?MODULE, Counter}), 0).
+
+%%
+
+match(T, Tab) ->
+    emqx_topic_index:match(T, Tab).
+
+matches(T, Tab) ->
+    lists:sort(emqx_topic_index:matches(T, Tab, [])).
+
+id(Match) ->
+    emqx_topic_index:get_id(Match).
+
+topic(Match) ->
+    emqx_topic_index:get_topic(Match).
+
+%%
+
+topic_t(EntropyWeights) ->
+    EWLast = lists:last(EntropyWeights),
+    ?LET(L, scaled(1 / 4, list(EWLast)), begin
+        EWs = lists:sublist(EntropyWeights ++ L, length(L)),
+        ?SIZED(S, [oneof([topic_level_t(S * EW), topic_level_fixed_t()]) || EW <- EWs])
+    end).
+
+topic_level_t(Entropy) ->
+    S = floor(1 + math:log2(Entropy) / 4),
+    ?LET(I, range(1, Entropy), iolist_to_binary(io_lib:format("~*.16.0B", [S, I]))).
+
+topic_level_fixed_t() ->
+    oneof([
+        <<"foo">>,
+        <<"bar">>,
+        <<"baz">>,
+        <<"xyzzy">>
+    ]).
+
+topic_filter_pattern_t() ->
+    list(topic_level_pattern_t()).
+
+topic_level_pattern_t() ->
+    frequency([
+        {5, level},
+        {2, '+'},
+        {1, '#'}
+    ]).
+
+mk_topic_filter([], _) ->
+    [];
+mk_topic_filter(_, []) ->
+    [];
+mk_topic_filter(['#' | _], _) ->
+    ['#'];
+mk_topic_filter(['+' | Rest], [_ | Levels]) ->
+    ['+' | mk_topic_filter(Rest, Levels)];
+mk_topic_filter([level | Rest], [L | Levels]) ->
+    [L | mk_topic_filter(Rest, Levels)].

apps/emqx/test/emqx_trace_SUITE.erl

@@ -274,7 +274,6 @@ t_load_state(_Config) ->
     ok.
 
 t_client_event(_Config) ->
-    application:set_env(emqx, allow_anonymous, true),
     ClientId = <<"client-test">>,
     Now = erlang:system_time(second),
     Name = <<"test_client_id_event">>,
@@ -312,6 +311,60 @@ t_client_event(_Config) ->
     ?assert(erlang:byte_size(Bin3) > 0),
     ok.
 
+t_client_huge_payload_truncated(_Config) ->
+    ClientId = <<"client-truncated1">>,
+    Now = erlang:system_time(second),
+    Name = <<"test_client_id_truncated1">>,
+    {ok, _} = emqx_trace:create([
+        {<<"name">>, Name},
+        {<<"type">>, clientid},
+        {<<"clientid">>, ClientId},
+        {<<"start_at">>, Now}
+    ]),
+    ok = emqx_trace_handler_SUITE:filesync(Name, clientid),
+    {ok, Client} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]),
+    {ok, _} = emqtt:connect(Client),
+    emqtt:ping(Client),
+    NormalPayload = iolist_to_binary(lists:duplicate(1024, "x")),
+    ok = emqtt:publish(Client, <<"/test">>, #{}, NormalPayload, [{qos, 0}]),
+    HugePayload1 = iolist_to_binary(lists:duplicate(1025, "y")),
+    ok = emqtt:publish(Client, <<"/test">>, #{}, HugePayload1, [{qos, 0}]),
+    HugePayload2 = iolist_to_binary(lists:duplicate(1024 * 10, "y")),
+    ok = emqtt:publish(Client, <<"/test">>, #{}, HugePayload2, [{qos, 0}]),
+    ok = emqx_trace_handler_SUITE:filesync(Name, clientid),
+    {ok, _} = emqx_trace:create([
+        {<<"name">>, <<"test_topic">>},
+        {<<"type">>, topic},
+        {<<"topic">>, <<"/test">>},
+        {<<"start_at">>, Now}
+    ]),
+    ok = emqx_trace_handler_SUITE:filesync(<<"test_topic">>, topic),
+    {ok, Bin} = file:read_file(emqx_trace:log_file(Name, Now)),
+    ok = emqtt:publish(Client, <<"/test">>, #{}, NormalPayload, [{qos, 0}]),
+    ok = emqtt:publish(Client, <<"/test">>, #{}, HugePayload1, [{qos, 0}]),
+    ok = emqtt:publish(Client, <<"/test">>, #{}, HugePayload2, [{qos, 0}]),
+    ok = emqtt:disconnect(Client),
+    ok = emqx_trace_handler_SUITE:filesync(Name, clientid),
+    ok = emqx_trace_handler_SUITE:filesync(<<"test_topic">>, topic),
+    {ok, Bin2} = file:read_file(emqx_trace:log_file(Name, Now)),
+    {ok, Bin3} = file:read_file(emqx_trace:log_file(<<"test_topic">>, Now)),
+    ct:pal("Bin ~p Bin2 ~p Bin3 ~p", [byte_size(Bin), byte_size(Bin2), byte_size(Bin3)]),
+    ?assert(erlang:byte_size(Bin) > 1024),
+    ?assert(erlang:byte_size(Bin) < erlang:byte_size(Bin2)),
+    ?assert(erlang:byte_size(Bin3) > 1024),
+
+    %% Don't have format crash
+    CrashBin = <<"CRASH">>,
+    ?assertEqual(nomatch, binary:match(Bin, [CrashBin])),
+    ?assertEqual(nomatch, binary:match(Bin2, [CrashBin])),
+    ?assertEqual(nomatch, binary:match(Bin3, [CrashBin])),
+    %% have "this log are truncated" for huge payload
+    TruncatedLog = <<"this log are truncated">>,
+    ?assertNotEqual(nomatch, binary:match(Bin, [TruncatedLog])),
+    ?assertNotEqual(nomatch, binary:match(Bin2, [TruncatedLog])),
+    ?assertNotEqual(nomatch, binary:match(Bin3, [TruncatedLog])),
+    ok.
+
 t_get_log_filename(_Config) ->
     Now = erlang:system_time(second),
     Name = <<"name1">>,

apps/emqx_authn/src/emqx_authn.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authn, [
     {description, "EMQX Authentication"},
-    {vsn, "0.1.24"},
+    {vsn, "0.1.25"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
     {applications, [

apps/emqx_authn/src/emqx_authn_enterprise.erl

@@ -9,7 +9,10 @@
 -if(?EMQX_RELEASE_EDITION == ee).
 
 providers() ->
-    [{{password_based, ldap}, emqx_ldap_authn}].
+    [
+        {{password_based, ldap}, emqx_ldap_authn},
+        {gcp_device, emqx_gcp_device_authn}
+    ].
 
 resource_provider() ->
     [emqx_ldap_authn].

apps/emqx_bridge_azure_event_hub/rebar.config

@@ -1,6 +1,6 @@
 %% -*- mode: erlang; -*-
 {erl_opts, [debug_info]}.
-{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.6"}}}
+{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.7"}}}
        , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.3"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0"}}}
        , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}}

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_gcp_pubsub, [
     {description, "EMQX Enterprise GCP Pub/Sub Bridge"},
-    {vsn, "0.1.6"},
+    {vsn, "0.1.7"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl

@@ -113,6 +113,22 @@ fields(connector_config) ->
     ];
 fields(producer) ->
     [
+        {attributes_template,
+            sc(
+                hoconsc:array(ref(key_value_pair)),
+                #{
+                    default => [],
+                    desc => ?DESC("attributes_template")
+                }
+            )},
+        {ordering_key_template,
+            sc(
+                binary(),
+                #{
+                    default => <<>>,
+                    desc => ?DESC("ordering_key_template")
+                }
+            )},
         {payload_template,
             sc(
                 binary(),
@@ -203,6 +219,18 @@ fields("consumer_resource_opts") ->
         fun({Field, _Sc}) -> lists:member(Field, SupportedFields) end,
         ResourceFields
     );
+fields(key_value_pair) ->
+    [
+        {key,
+            mk(binary(), #{
+                required => true,
+                validator => [
+                    emqx_resource_validator:not_empty("Key templates must not be empty")
+                ],
+                desc => ?DESC(kv_pair_key)
+            })},
+        {value, mk(binary(), #{required => true, desc => ?DESC(kv_pair_value)})}
+    ];
 fields("get_producer") ->
     emqx_bridge_schema:status_fields() ++ fields("post_producer");
 fields("post_producer") ->
@@ -218,6 +246,8 @@ fields("put_consumer") ->
 
 desc("config_producer") ->
     ?DESC("desc_config");
+desc(key_value_pair) ->
+    ?DESC("kv_pair_desc");
 desc("config_consumer") ->
     ?DESC("desc_config");
 desc("consumer_resource_opts") ->

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl

@@ -205,7 +205,7 @@ get_topic(Topic, ConnectorState) ->
     Path = <<"/v1/projects/", ProjectId/binary, "/topics/", Topic/binary>>,
     Body = <<>>,
     PreparedRequest = {prepared_request, {Method, Path, Body}},
-    query_sync(PreparedRequest, ConnectorState).
+    ?MODULE:query_sync(PreparedRequest, ConnectorState).
 
 %%-------------------------------------------------------------------------------------------------
 %% Helper fns

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_consumer_worker.erl

@@ -217,7 +217,9 @@ handle_continue(?ensure_subscription, State0) ->
             {noreply, State0, {continue, ?ensure_subscription}};
         not_found ->
             %% there's nothing much to do if the topic suddenly doesn't exist anymore.
-            {stop, {error, topic_not_found}, State0}
+            {stop, {error, topic_not_found}, State0};
+        permission_denied ->
+            {stop, {error, permission_denied}, State0}
     end;
 handle_continue(?patch_subscription, State0) ->
     ?tp(gcp_pubsub_consumer_worker_patch_subscription_enter, #{}),
@@ -291,14 +293,17 @@ handle_info(Msg, State0) ->
     }),
     {noreply, State0}.
 
-terminate({error, topic_not_found} = _Reason, State) ->
+terminate({error, Reason}, State) when
+    Reason =:= topic_not_found;
+    Reason =:= permission_denied
+->
     #{
         instance_id := InstanceId,
         topic := _Topic
     } = State,
     optvar:unset(?OPTVAR_SUB_OK(self())),
-    emqx_bridge_gcp_pubsub_impl_consumer:mark_topic_as_nonexistent(InstanceId),
+    emqx_bridge_gcp_pubsub_impl_consumer:mark_as_unhealthy(InstanceId, Reason),
-    ?tp(gcp_pubsub_consumer_worker_terminate, #{reason => _Reason, topic => _Topic}),
+    ?tp(gcp_pubsub_consumer_worker_terminate, #{reason => {error, Reason}, topic => _Topic}),
     ok;
 terminate(_Reason, _State) ->
     optvar:unset(?OPTVAR_SUB_OK(self())),
@@ -329,7 +334,8 @@ ensure_pull_timer(State = #{pull_timer := TRef}) when is_reference(TRef) ->
 ensure_pull_timer(State = #{pull_retry_interval := PullRetryInterval}) ->
     State#{pull_timer := emqx_utils:start_timer(PullRetryInterval, pull)}.
 
--spec ensure_subscription_exists(state()) -> continue | retry | not_found | already_exists.
+-spec ensure_subscription_exists(state()) ->
+    continue | retry | not_found | permission_denied | already_exists.
 ensure_subscription_exists(State) ->
     ?tp(gcp_pubsub_consumer_worker_create_subscription_enter, #{}),
     #{
@@ -367,6 +373,17 @@ ensure_subscription_exists(State) ->
                 }
             ),
             not_found;
+        {error, #{status_code := 403}} ->
+            %% permission denied
+            ?tp(
+                warning,
+                "gcp_pubsub_consumer_worker_permission_denied",
+                #{
+                    instance_id => InstanceId,
+                    topic => Topic
+                }
+            ),
+            permission_denied;
         {ok, #{status_code := 200}} ->
             ?tp(
                 debug,

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl

@@ -17,9 +17,9 @@
 
 %% health check API
 -export([
-    mark_topic_as_nonexistent/1,
+    mark_as_unhealthy/2,
-    unset_nonexistent_topic/1,
+    clear_unhealthy/1,
-    is_nonexistent_topic/1
+    check_if_unhealthy/1
 ]).
 
 -include_lib("emqx/include/logger.hrl").
@@ -47,11 +47,15 @@
 
 -define(AUTO_RECONNECT_S, 2).
 -define(DEFAULT_FORGET_INTERVAL, timer:seconds(60)).
--define(OPTVAR_TOPIC_NOT_FOUND(INSTANCE_ID), {?MODULE, topic_not_found, INSTANCE_ID}).
+-define(OPTVAR_UNHEALTHY(INSTANCE_ID), {?MODULE, topic_not_found, INSTANCE_ID}).
 -define(TOPIC_MESSAGE,
     "GCP PubSub topics are invalid.  Please check the logs, check if the "
     "topics exist in GCP and if the service account has permissions to use them."
 ).
+-define(PERMISSION_MESSAGE,
+    "Permission denied while verifying topic existence.  Please check that the "
+    "provided service account has the correct permissions configured."
+).
 
 %%-------------------------------------------------------------------------------------------------
 %% `emqx_resource' API
@@ -77,7 +81,7 @@ on_start(InstanceId, Config0) ->
 -spec on_stop(resource_id(), state()) -> ok | {error, term()}.
 on_stop(InstanceId, _State) ->
     ?tp(gcp_pubsub_consumer_stop_enter, #{}),
-    unset_nonexistent_topic(InstanceId),
+    clear_unhealthy(InstanceId),
     ok = stop_consumers(InstanceId),
     emqx_bridge_gcp_pubsub_client:stop(InstanceId).
 
@@ -85,10 +89,12 @@ on_stop(InstanceId, _State) ->
 on_get_status(InstanceId, State) ->
     %% We need to check this flag separately because the workers might be gone when we
     %% check them.
-    case is_nonexistent_topic(InstanceId) of
+    case check_if_unhealthy(InstanceId) of
-        true ->
+        {error, topic_not_found} ->
             {disconnected, State, {unhealthy_target, ?TOPIC_MESSAGE}};
-        false ->
+        {error, permission_denied} ->
+            {disconnected, State, {unhealthy_target, ?PERMISSION_MESSAGE}};
+        ok ->
             #{client := Client} = State,
             check_workers(InstanceId, Client)
     end.
@@ -97,24 +103,24 @@ on_get_status(InstanceId, State) ->
 %% Health check API (signalled by consumer worker)
 %%-------------------------------------------------------------------------------------------------
 
--spec mark_topic_as_nonexistent(resource_id()) -> ok.
+-spec mark_as_unhealthy(resource_id(), topic_not_found | permission_denied) -> ok.
-mark_topic_as_nonexistent(InstanceId) ->
+mark_as_unhealthy(InstanceId, Reason) ->
-    optvar:set(?OPTVAR_TOPIC_NOT_FOUND(InstanceId), true),
+    optvar:set(?OPTVAR_UNHEALTHY(InstanceId), Reason),
     ok.
 
--spec unset_nonexistent_topic(resource_id()) -> ok.
+-spec clear_unhealthy(resource_id()) -> ok.
-unset_nonexistent_topic(InstanceId) ->
+clear_unhealthy(InstanceId) ->
-    optvar:unset(?OPTVAR_TOPIC_NOT_FOUND(InstanceId)),
+    optvar:unset(?OPTVAR_UNHEALTHY(InstanceId)),
-    ?tp(gcp_pubsub_consumer_unset_nonexistent_topic, #{}),
+    ?tp(gcp_pubsub_consumer_clear_unhealthy, #{}),
     ok.
 
--spec is_nonexistent_topic(resource_id()) -> boolean().
+-spec check_if_unhealthy(resource_id()) -> ok | {error, topic_not_found | permission_denied}.
-is_nonexistent_topic(InstanceId) ->
+check_if_unhealthy(InstanceId) ->
-    case optvar:peek(?OPTVAR_TOPIC_NOT_FOUND(InstanceId)) of
+    case optvar:peek(?OPTVAR_UNHEALTHY(InstanceId)) of
-        {ok, true} ->
+        {ok, Reason} ->
-            true;
+            {error, Reason};
-        _ ->
+        undefined ->
-            false
+            ok
     end.
 
 %%-------------------------------------------------------------------------------------------------
@@ -153,6 +159,11 @@ start_consumers(InstanceId, Client, Config) ->
             throw(
                 {unhealthy_target, ?TOPIC_MESSAGE}
             );
+        {error, permission_denied} ->
+            _ = emqx_bridge_gcp_pubsub_client:stop(InstanceId),
+            throw(
+                {unhealthy_target, ?PERMISSION_MESSAGE}
+            );
         {error, _} ->
             %% connection might be down; we'll have to check topic existence during health
             %% check, or the workers will kill themselves when they realized there's no
@@ -229,6 +240,8 @@ check_for_topic_existence(Topic, Client) ->
             ok;
         {error, #{status_code := 404}} ->
             {error, not_found};
+        {error, #{status_code := 403}} ->
+            {error, permission_denied};
         {error, Reason} ->
             ?tp(warning, "gcp_pubsub_consumer_check_topic_error", #{reason => Reason}),
             {error, Reason}

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl

@@ -9,15 +9,20 @@
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -type config() :: #{
+    attributes_template := [#{key := binary(), value := binary()}],
     connect_timeout := emqx_schema:duration_ms(),
     max_retries := non_neg_integer(),
+    ordering_key_template := binary(),
+    payload_template := binary(),
     pubsub_topic := binary(),
     resource_opts := #{request_ttl := infinity | emqx_schema:duration_ms(), any() => term()},
     service_account_json := emqx_bridge_gcp_pubsub_client:service_account_json(),
     any() => term()
 }.
 -type state() :: #{
+    attributes_template := #{emqx_placeholder:tmpl_token() => emqx_placeholder:tmpl_token()},
     client := emqx_bridge_gcp_pubsub_client:state(),
+    ordering_key_template := emqx_placeholder:tmpl_token(),
     payload_template := emqx_placeholder:tmpl_token(),
     project_id := emqx_bridge_gcp_pubsub_client:project_id(),
     pubsub_topic := binary()
@@ -57,6 +62,8 @@ on_start(InstanceId, Config0) ->
     }),
     Config = maps:update_with(service_account_json, fun emqx_utils_maps:binary_key_map/1, Config0),
     #{
+        attributes_template := AttributesTemplate,
+        ordering_key_template := OrderingKeyTemplate,
         payload_template := PayloadTemplate,
         pubsub_topic := PubSubTopic,
         service_account_json := #{<<"project_id">> := ProjectId}
@@ -65,6 +72,8 @@ on_start(InstanceId, Config0) ->
         {ok, Client} ->
             State = #{
                 client => Client,
+                attributes_template => preproc_attributes(AttributesTemplate),
+                ordering_key_template => emqx_placeholder:preproc_tmpl(OrderingKeyTemplate),
                 payload_template => emqx_placeholder:preproc_tmpl(PayloadTemplate),
                 project_id => ProjectId,
                 pubsub_topic => PubSubTopic
@@ -197,14 +206,107 @@ do_send_requests_async(State, Requests, ReplyFunAndArgs0) ->
         Request, ReplyFunAndArgs, Client
     ).
 
--spec encode_payload(state(), Selected :: map()) -> #{data := binary()}.
+-spec encode_payload(state(), Selected :: map()) ->
-encode_payload(_State = #{payload_template := PayloadTemplate}, Selected) ->
+    #{
-    Interpolated =
+        data := binary(),
-        case PayloadTemplate of
+        attributes => #{binary() => binary()},
-            [] -> emqx_utils_json:encode(Selected);
+        'orderingKey' => binary()
-            _ -> emqx_placeholder:proc_tmpl(PayloadTemplate, Selected)
+    }.
+encode_payload(State, Selected) ->
+    #{
+        attributes_template := AttributesTemplate,
+        ordering_key_template := OrderingKeyTemplate,
+        payload_template := PayloadTemplate
+    } = State,
+    Data = render_payload(PayloadTemplate, Selected),
+    OrderingKey = render_key(OrderingKeyTemplate, Selected),
+    Attributes = proc_attributes(AttributesTemplate, Selected),
+    Payload0 = #{data => base64:encode(Data)},
+    Payload1 = put_if(Payload0, attributes, Attributes, map_size(Attributes) > 0),
+    put_if(Payload1, 'orderingKey', OrderingKey, OrderingKey =/= <<>>).
+
+put_if(Acc, K, V, true) ->
+    Acc#{K => V};
+put_if(Acc, _K, _V, false) ->
+    Acc.
+
+-spec render_payload(emqx_placeholder:tmpl_token(), map()) -> binary().
+render_payload([] = _Template, Selected) ->
+    emqx_utils_json:encode(Selected);
+render_payload(Template, Selected) ->
+    render_value(Template, Selected).
+
+render_key(Template, Selected) ->
+    Opts = #{
+        return => full_binary,
+        var_trans => fun
+            (_Var, undefined) ->
+                <<>>;
+            (Var, X) when is_boolean(X) ->
+                throw({bad_value_for_key, Var, X});
+            (_Var, X) when is_binary(X); is_number(X); is_atom(X) ->
+                emqx_utils_conv:bin(X);
+            (Var, X) ->
+                throw({bad_value_for_key, Var, X})
+        end
+    },
+    try
+        emqx_placeholder:proc_tmpl(Template, Selected, Opts)
+    catch
+        throw:{bad_value_for_key, Var, X} ->
+            ?tp(
+                warning,
+                "gcp_pubsub_producer_bad_value_for_key",
+                #{
+                    placeholder => Var,
+                    value => X,
+                    action => "key ignored",
+                    hint => "only plain values like strings and numbers can be used in keys"
+                }
+            ),
+            <<>>
+    end.
+
+render_value(Template, Selected) ->
+    Opts = #{
+        return => full_binary,
+        var_trans => fun
+            (undefined) -> <<>>;
+            (X) -> emqx_utils_conv:bin(X)
+        end
+    },
+    emqx_placeholder:proc_tmpl(Template, Selected, Opts).
+
+-spec preproc_attributes([#{key := binary(), value := binary()}]) ->
+    #{emqx_placeholder:tmpl_token() => emqx_placeholder:tmpl_token()}.
+preproc_attributes(AttributesTemplate) ->
+    lists:foldl(
+        fun(#{key := K, value := V}, Acc) ->
+            KT = emqx_placeholder:preproc_tmpl(K),
+            VT = emqx_placeholder:preproc_tmpl(V),
+            Acc#{KT => VT}
         end,
-    #{data => base64:encode(Interpolated)}.
+        #{},
+        AttributesTemplate
+    ).
+
+-spec proc_attributes(#{emqx_placeholder:tmpl_token() => emqx_placeholder:tmpl_token()}, map()) ->
+    #{binary() => binary()}.
+proc_attributes(AttributesTemplate, Selected) ->
+    maps:fold(
+        fun(KT, VT, Acc) ->
+            K = render_key(KT, Selected),
+            case K =:= <<>> of
+                true ->
+                    Acc;
+                false ->
+                    V = render_value(VT, Selected),
+                    Acc#{K => V}
+            end
+        end,
+        #{},
+        AttributesTemplate
+    ).
 
 -spec to_pubsub_request([#{data := binary()}]) -> binary().
 to_pubsub_request(Payloads) ->

apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl

@@ -760,6 +760,64 @@ prop_acked_ids_eventually_forgotten(Trace) ->
     ),
     ok.
 
+permission_denied_response() ->
+    Link =
+        <<"https://console.developers.google.com/project/9999/apiui/credential">>,
+    {error, #{
+        status_code => 403,
+        headers =>
+            [
+                {<<"vary">>, <<"X-Origin">>},
+                {<<"vary">>, <<"Referer">>},
+                {<<"content-type">>, <<"application/json; charset=UTF-8">>},
+                {<<"date">>, <<"Tue, 15 Aug 2023 13:59:09 GMT">>},
+                {<<"server">>, <<"ESF">>},
+                {<<"cache-control">>, <<"private">>},
+                {<<"x-xss-protection">>, <<"0">>},
+                {<<"x-frame-options">>, <<"SAMEORIGIN">>},
+                {<<"x-content-type-options">>, <<"nosniff">>},
+                {<<"alt-svc">>, <<"h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000">>},
+                {<<"accept-ranges">>, <<"none">>},
+                {<<"vary">>, <<"Origin,Accept-Encoding">>},
+                {<<"transfer-encoding">>, <<"chunked">>}
+            ],
+        body => emqx_utils_json:encode(
+            #{
+                <<"error">> =>
+                    #{
+                        <<"code">> => 403,
+                        <<"details">> =>
+                            [
+                                #{
+                                    <<"@type">> => <<"type.googleapis.com/google.rpc.Help">>,
+                                    <<"links">> =>
+                                        [
+                                            #{
+                                                <<"description">> =>
+                                                    <<"Google developer console API key">>,
+                                                <<"url">> =>
+                                                    Link
+                                            }
+                                        ]
+                                },
+                                #{
+                                    <<"@type">> => <<"type.googleapis.com/google.rpc.ErrorInfo">>,
+                                    <<"domain">> => <<"googleapis.com">>,
+                                    <<"metadata">> =>
+                                        #{
+                                            <<"consumer">> => <<"projects/9999">>,
+                                            <<"service">> => <<"pubsub.googleapis.com">>
+                                        },
+                                    <<"reason">> => <<"CONSUMER_INVALID">>
+                                }
+                            ],
+                        <<"message">> => <<"Project #9999 has been deleted.">>,
+                        <<"status">> => <<"PERMISSION_DENIED">>
+                    }
+            }
+        )
+    }}.
+
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
@@ -785,7 +843,7 @@ t_start_stop(Config) ->
             prop_client_stopped(),
             prop_workers_stopped(PubSubTopic),
             fun(Trace) ->
-                ?assertMatch([_], ?of_kind(gcp_pubsub_consumer_unset_nonexistent_topic, Trace)),
+                ?assertMatch([_], ?of_kind(gcp_pubsub_consumer_clear_unhealthy, Trace)),
                 ok
             end
         ]
@@ -1992,6 +2050,81 @@ t_get_subscription(Config) ->
     ),
     ok.
 
+t_permission_denied_topic_check(Config) ->
+    [#{pubsub_topic := PubSubTopic}] = ?config(topic_mapping, Config),
+    ResourceId = resource_id(Config),
+    ?check_trace(
+        begin
+            %% the emulator does not check any credentials
+            emqx_common_test_helpers:with_mock(
+                emqx_bridge_gcp_pubsub_client,
+                query_sync,
+                fun(PreparedRequest = {prepared_request, {Method, Path, _Body}}, Client) ->
+                    RE = iolist_to_binary(["/topics/", PubSubTopic, "$"]),
+                    case {Method =:= get, re:run(Path, RE)} of
+                        {true, {match, _}} ->
+                            permission_denied_response();
+                        _ ->
+                            meck:passthrough([PreparedRequest, Client])
+                    end
+                end,
+                fun() ->
+                    {{ok, _}, {ok, _}} =
+                        ?wait_async_action(
+                            create_bridge(Config),
+                            #{?snk_kind := gcp_pubsub_stop},
+                            5_000
+                        ),
+                    ?assertMatch(
+                        {ok, disconnected},
+                        emqx_resource_manager:health_check(ResourceId)
+                    ),
+                    ?assertMatch(
+                        {ok, _Group, #{error := {unhealthy_target, "Permission denied" ++ _}}},
+                        emqx_resource_manager:lookup_cached(ResourceId)
+                    ),
+                    ok
+                end
+            ),
+            ok
+        end,
+        []
+    ),
+    ok.
+
+t_permission_denied_worker(Config) ->
+    ?check_trace(
+        begin
+            emqx_common_test_helpers:with_mock(
+                emqx_bridge_gcp_pubsub_client,
+                query_sync,
+                fun(PreparedRequest = {prepared_request, {Method, _Path, _Body}}, Client) ->
+                    case Method =:= put of
+                        true ->
+                            permission_denied_response();
+                        false ->
+                            meck:passthrough([PreparedRequest, Client])
+                    end
+                end,
+                fun() ->
+                    {{ok, _}, {ok, _}} =
+                        ?wait_async_action(
+                            create_bridge(
+                                Config
+                            ),
+                            #{?snk_kind := gcp_pubsub_consumer_worker_terminate},
+                            10_000
+                        ),
+
+                    ok
+                end
+            ),
+            ok
+        end,
+        []
+    ),
+    ok.
+
 t_cluster_subscription(Config) ->
     [
         #{

apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_producer_SUITE.erl

@@ -63,7 +63,9 @@ single_config_tests() ->
         t_get_status_down,
         t_get_status_no_worker,
         t_get_status_timeout_calling_workers,
-        t_on_start_ehttpc_pool_already_started
+        t_on_start_ehttpc_pool_already_started,
+        t_attributes,
+        t_bad_attributes
     ].
 
 only_sync_tests() ->
@@ -212,7 +214,9 @@ create_bridge_http(Config, GCPPubSubConfigOverrides) ->
                 Error
         end,
     ct:pal("bridge creation result: ~p", [Res]),
-    ?assertEqual(element(1, ProbeResult), element(1, Res)),
+    ?assertEqual(element(1, ProbeResult), element(1, Res), #{
+        creation_result => Res, probe_result => ProbeResult
+    }),
     case ProbeResult of
         {error, {{_, 500, _}, _, _}} -> error({bad_probe_result, ProbeResult});
         _ -> ok
@@ -456,6 +460,7 @@ assert_valid_request_headers(Headers, ServiceAccountJSON) ->
 assert_valid_request_body(Body) ->
     BodyMap = emqx_utils_json:decode(Body, [return_maps]),
     ?assertMatch(#{<<"messages">> := [_ | _]}, BodyMap),
+    ct:pal("request: ~p", [BodyMap]),
     #{<<"messages">> := Messages} = BodyMap,
     lists:map(
         fun(Msg) ->
@@ -480,6 +485,31 @@ assert_http_request(ServiceAccountJSON) ->
         error({timeout, #{mailbox => Mailbox}})
     end.
 
+receive_http_requests(ServiceAccountJSON, Opts) ->
+    Default = #{n => 1},
+    #{n := N} = maps:merge(Default, Opts),
+    lists:flatmap(fun(_) -> receive_http_request(ServiceAccountJSON) end, lists:seq(1, N)).
+
+receive_http_request(ServiceAccountJSON) ->
+    receive
+        {http, Headers, Body} ->
+            ct:pal("received publish:\n  ~p", [#{headers => Headers, body => Body}]),
+            assert_valid_request_headers(Headers, ServiceAccountJSON),
+            #{<<"messages">> := Msgs} = emqx_utils_json:decode(Body, [return_maps]),
+            lists:map(
+                fun(Msg) ->
+                    #{<<"data">> := Content64} = Msg,
+                    Content = base64:decode(Content64),
+                    Decoded = emqx_utils_json:decode(Content, [return_maps]),
+                    Msg#{<<"data">> := Decoded}
+                end,
+                Msgs
+            )
+    after 5_000 ->
+        {messages, Mailbox} = process_info(self(), messages),
+        error({timeout, #{mailbox => Mailbox}})
+    end.
+
 install_telemetry_handler(TestCase) ->
     Tid = ets:new(TestCase, [ordered_set, public]),
     HandlerId = TestCase,
@@ -585,8 +615,8 @@ t_publish_success(Config) ->
                 <<"topic">> := Topic,
                 <<"payload">> := Payload,
                 <<"metadata">> := #{<<"rule_id">> := RuleId}
-            }
+            } = Msg
-        ],
+        ] when not (is_map_key(<<"attributes">>, Msg) orelse is_map_key(<<"orderingKey">>, Msg)),
         DecodedMessages
     ),
     %% to avoid test flakiness
@@ -1524,3 +1554,251 @@ t_query_sync(Config) ->
         []
     ),
     ok.
+
+t_attributes(Config) ->
+    Name = ?config(gcp_pubsub_name, Config),
+    ServiceAccountJSON = ?config(service_account_json, Config),
+    LocalTopic = <<"t/topic">>,
+    ?check_trace(
+        begin
+            {ok, _} = create_bridge_http(
+                Config,
+                #{
+                    <<"local_topic">> => LocalTopic,
+                    <<"attributes_template">> =>
+                        [
+                            #{
+                                <<"key">> => <<"${.payload.key}">>,
+                                <<"value">> => <<"fixed_value">>
+                            },
+                            #{
+                                <<"key">> => <<"${.payload.key}2">>,
+                                <<"value">> => <<"${.payload.value}">>
+                            },
+                            #{
+                                <<"key">> => <<"fixed_key">>,
+                                <<"value">> => <<"fixed_value">>
+                            },
+                            #{
+                                <<"key">> => <<"fixed_key2">>,
+                                <<"value">> => <<"${.payload.value}">>
+                            }
+                        ],
+                    <<"ordering_key_template">> => <<"${.payload.ok}">>
+                }
+            ),
+            %% without ordering key
+            Payload0 =
+                emqx_utils_json:encode(
+                    #{
+                        <<"value">> => <<"payload_value">>,
+                        <<"key">> => <<"payload_key">>
+                    }
+                ),
+            Message0 = emqx_message:make(LocalTopic, Payload0),
+            emqx:publish(Message0),
+            DecodedMessages0 = receive_http_request(ServiceAccountJSON),
+            ?assertMatch(
+                [
+                    #{
+                        <<"attributes">> :=
+                            #{
+                                <<"fixed_key">> := <<"fixed_value">>,
+                                <<"fixed_key2">> := <<"payload_value">>,
+                                <<"payload_key">> := <<"fixed_value">>,
+                                <<"payload_key2">> := <<"payload_value">>
+                            },
+                        <<"data">> := #{
+                            <<"topic">> := _,
+                            <<"payload">> := _
+                        }
+                    } = Msg
+                ] when not is_map_key(<<"orderingKey">>, Msg),
+                DecodedMessages0
+            ),
+            %% with ordering key
+            Payload1 =
+                emqx_utils_json:encode(
+                    #{
+                        <<"value">> => <<"payload_value">>,
+                        <<"key">> => <<"payload_key">>,
+                        <<"ok">> => <<"ordering_key">>
+                    }
+                ),
+            Message1 = emqx_message:make(LocalTopic, Payload1),
+            emqx:publish(Message1),
+            DecodedMessages1 = receive_http_request(ServiceAccountJSON),
+            ?assertMatch(
+                [
+                    #{
+                        <<"attributes">> :=
+                            #{
+                                <<"fixed_key">> := <<"fixed_value">>,
+                                <<"fixed_key2">> := <<"payload_value">>,
+                                <<"payload_key">> := <<"fixed_value">>,
+                                <<"payload_key2">> := <<"payload_value">>
+                            },
+                        <<"orderingKey">> := <<"ordering_key">>,
+                        <<"data">> := #{
+                            <<"topic">> := _,
+                            <<"payload">> := _
+                        }
+                    }
+                ],
+                DecodedMessages1
+            ),
+            %% will result in empty key
+            Payload2 =
+                emqx_utils_json:encode(
+                    #{
+                        <<"value">> => <<"payload_value">>,
+                        <<"ok">> => <<"ordering_key">>
+                    }
+                ),
+            Message2 = emqx_message:make(LocalTopic, Payload2),
+            emqx:publish(Message2),
+            [DecodedMessage2] = receive_http_request(ServiceAccountJSON),
+            ?assertEqual(
+                #{
+                    <<"fixed_key">> => <<"fixed_value">>,
+                    <<"fixed_key2">> => <<"payload_value">>,
+                    <<"2">> => <<"payload_value">>
+                },
+                maps:get(<<"attributes">>, DecodedMessage2)
+            ),
+            %% ensure loading cluster override file doesn't mangle the attribute
+            %% placeholders...
+            #{<<"bridges">> := #{?BRIDGE_TYPE_BIN := #{Name := RawConf}}} =
+                emqx_config:read_override_conf(#{override_to => cluster}),
+            ?assertEqual(
+                [
+                    #{
+                        <<"key">> => <<"${.payload.key}">>,
+                        <<"value">> => <<"fixed_value">>
+                    },
+                    #{
+                        <<"key">> => <<"${.payload.key}2">>,
+                        <<"value">> => <<"${.payload.value}">>
+                    },
+                    #{
+                        <<"key">> => <<"fixed_key">>,
+                        <<"value">> => <<"fixed_value">>
+                    },
+                    #{
+                        <<"key">> => <<"fixed_key2">>,
+                        <<"value">> => <<"${.payload.value}">>
+                    }
+                ],
+                maps:get(<<"attributes_template">>, RawConf)
+            ),
+            ok
+        end,
+        []
+    ),
+    ok.
+
+t_bad_attributes(Config) ->
+    ServiceAccountJSON = ?config(service_account_json, Config),
+    LocalTopic = <<"t/topic">>,
+    ?check_trace(
+        begin
+            {ok, _} = create_bridge_http(
+                Config,
+                #{
+                    <<"local_topic">> => LocalTopic,
+                    <<"attributes_template">> =>
+                        [
+                            #{
+                                <<"key">> => <<"${.payload.key}">>,
+                                <<"value">> => <<"${.payload.value}">>
+                            }
+                        ],
+                    <<"ordering_key_template">> => <<"${.payload.ok}">>
+                }
+            ),
+            %% Ok: attribute value is a map or list
+            lists:foreach(
+                fun(OkValue) ->
+                    Payload0 =
+                        emqx_utils_json:encode(
+                            #{
+                                <<"ok">> => <<"ord_key">>,
+                                <<"value">> => OkValue,
+                                <<"key">> => <<"attr_key">>
+                            }
+                        ),
+                    Message0 = emqx_message:make(LocalTopic, Payload0),
+                    emqx:publish(Message0)
+                end,
+                [
+                    #{<<"some">> => <<"map">>},
+                    [1, <<"str">>, #{<<"deep">> => true}]
+                ]
+            ),
+            DecodedMessages0 = receive_http_requests(ServiceAccountJSON, #{n => 1}),
+            ?assertMatch(
+                [
+                    #{
+                        <<"attributes">> :=
+                            #{<<"attr_key">> := <<"{\"some\":\"map\"}">>},
+                        <<"orderingKey">> := <<"ord_key">>
+                    },
+                    #{
+                        <<"attributes">> :=
+                            #{<<"attr_key">> := <<"[1,\"str\",{\"deep\":true}]">>},
+                        <<"orderingKey">> := <<"ord_key">>
+                    }
+                ],
+                DecodedMessages0
+            ),
+            %% Bad: key is not a plain value
+            lists:foreach(
+                fun(BadKey) ->
+                    Payload1 =
+                        emqx_utils_json:encode(
+                            #{
+                                <<"value">> => <<"v">>,
+                                <<"key">> => BadKey,
+                                <<"ok">> => BadKey
+                            }
+                        ),
+                    Message1 = emqx_message:make(LocalTopic, Payload1),
+                    emqx:publish(Message1)
+                end,
+                [
+                    #{<<"some">> => <<"map">>},
+                    [1, <<"list">>, true],
+                    true,
+                    false
+                ]
+            ),
+            DecodedMessages1 = receive_http_request(ServiceAccountJSON),
+            lists:foreach(
+                fun(DMsg) ->
+                    ?assertNot(is_map_key(<<"orderingKey">>, DMsg), #{decoded_message => DMsg}),
+                    ?assertNot(is_map_key(<<"attributes">>, DMsg), #{decoded_message => DMsg}),
+                    ok
+                end,
+                DecodedMessages1
+            ),
+            ok
+        end,
+        fun(Trace) ->
+            ct:pal("trace:\n  ~p", [Trace]),
+            ?assertMatch(
+                [
+                    #{placeholder := [<<"payload">>, <<"ok">>], value := #{}},
+                    #{placeholder := [<<"payload">>, <<"key">>], value := #{}},
+                    #{placeholder := [<<"payload">>, <<"ok">>], value := [_ | _]},
+                    #{placeholder := [<<"payload">>, <<"key">>], value := [_ | _]},
+                    #{placeholder := [<<"payload">>, <<"ok">>], value := true},
+                    #{placeholder := [<<"payload">>, <<"key">>], value := true},
+                    #{placeholder := [<<"payload">>, <<"ok">>], value := false},
+                    #{placeholder := [<<"payload">>, <<"key">>], value := false}
+                ],
+                ?of_kind("gcp_pubsub_producer_bad_value_for_key", Trace)
+            ),
+            ok
+        end
+    ),
+    ok.

apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_tests.erl

@@ -0,0 +1,149 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_gcp_pubsub_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+%%===========================================================================
+%% Data section
+%%===========================================================================
+
+%% erlfmt-ignore
+gcp_pubsub_producer_hocon() ->
+"""
+bridges.gcp_pubsub.my_producer {
+  attributes_template = [
+    {key = \"${payload.key}\", value = fixed_value}
+    {key = \"${payload.key}2\", value = \"${.payload.value}\"}
+    {key = fixed_key, value = fixed_value}
+    {key = fixed_key2, value = \"${.payload.value}\"}
+  ]
+  connect_timeout = 15s
+  enable = false
+  local_topic = \"t/gcp/produ\"
+  max_retries = 2
+  ordering_key_template = \"${.payload.ok}\"
+  payload_template = \"${.}\"
+  pipelining = 100
+  pool_size = 8
+  pubsub_topic = my-topic
+  resource_opts {
+    batch_size = 1
+    batch_time = 0ms
+    health_check_interval = 15s
+    inflight_window = 100
+    max_buffer_bytes = 256MB
+    query_mode = async
+    request_ttl = 15s
+    start_after_created = true
+    start_timeout = 5s
+    worker_pool_size = 16
+  }
+  service_account_json {
+    auth_provider_x509_cert_url = \"https://www.googleapis.com/oauth2/v1/certs\"
+    auth_uri = \"https://accounts.google.com/o/oauth2/auth\"
+    client_email = \"test@myproject.iam.gserviceaccount.com\"
+    client_id = \"123812831923812319190\"
+    client_x509_cert_url = \"https://www.googleapis.com/robot/v1/metadata/x509/...\"
+    private_key = \"-----BEGIN PRIVATE KEY-----...\"
+    private_key_id = \"kid\"
+    project_id = myproject
+    token_uri = \"https://oauth2.googleapis.com/token\"
+    type = service_account
+  }
+}
+""".
+
+%%===========================================================================
+%% Helper functions
+%%===========================================================================
+
+parse(Hocon) ->
+    {ok, Conf} = hocon:binary(Hocon),
+    Conf.
+
+check(Conf) when is_map(Conf) ->
+    hocon_tconf:check_plain(emqx_bridge_schema, Conf).
+
+-define(validation_error(Reason, Value),
+    {emqx_bridge_schema, [
+        #{
+            kind := validation_error,
+            reason := Reason,
+            value := Value
+        }
+    ]}
+).
+
+-define(ok_config(Cfg), #{
+    <<"bridges">> :=
+        #{
+            <<"gcp_pubsub">> :=
+                #{
+                    <<"my_producer">> :=
+                        Cfg
+                }
+        }
+}).
+
+%%===========================================================================
+%% Test cases
+%%===========================================================================
+
+producer_attributes_validator_test_() ->
+    %% ensure this module is loaded when testing only this file
+    _ = emqx_bridge_enterprise:module_info(),
+    BaseConf = parse(gcp_pubsub_producer_hocon()),
+    Override = fun(Cfg) ->
+        emqx_utils_maps:deep_merge(
+            BaseConf,
+            #{
+                <<"bridges">> =>
+                    #{
+                        <<"gcp_pubsub">> =>
+                            #{<<"my_producer">> => Cfg}
+                    }
+            }
+        )
+    end,
+    [
+        {"base config",
+            ?_assertMatch(
+                ?ok_config(#{
+                    <<"attributes_template">> := [_, _, _, _]
+                }),
+                check(BaseConf)
+            )},
+        {"empty key template",
+            ?_assertThrow(
+                ?validation_error("Key templates must not be empty", _),
+                check(
+                    Override(#{
+                        <<"attributes_template">> => [
+                            #{
+                                <<"key">> => <<>>,
+                                <<"value">> => <<"some_value">>
+                            }
+                        ]
+                    })
+                )
+            )},
+        {"empty value template",
+            ?_assertMatch(
+                ?ok_config(#{
+                    <<"attributes_template">> := [_]
+                }),
+                check(
+                    Override(#{
+                        <<"attributes_template">> => [
+                            #{
+                                <<"key">> => <<"some_key">>,
+                                <<"value">> => <<>>
+                            }
+                        ]
+                    })
+                )
+            )}
+    ].

apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_influxdb, [
     {description, "EMQX Enterprise InfluxDB Bridge"},
-    {vsn, "0.1.3"},
+    {vsn, "0.1.4"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.erl

@@ -168,6 +168,9 @@ write_syntax(format) ->
 write_syntax(_) ->
     undefined.
 
+to_influx_lines(Lines = [#{} | _]) ->
+    %% already parsed/converted (e.g.: bridge_probe, after hocon_tconf:check_plain)
+    Lines;
 to_influx_lines(RawLines) ->
     try
         influx_lines(str(RawLines), [])

apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl

@@ -66,7 +66,9 @@ on_start(InstId, Config) ->
 on_stop(InstId, _State) ->
     case emqx_resource:get_allocated_resources(InstId) of
         #{?influx_client := Client} ->
-            influxdb:stop_client(Client);
+            Res = influxdb:stop_client(Client),
+            ?tp(influxdb_client_stopped, #{instance_id => InstId}),
+            Res;
         _ ->
             ok
     end.

apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl

@@ -124,6 +124,9 @@ init_per_group(InfluxDBType, Config0) when
                 {influxdb_config, InfluxDBConfig},
                 {influxdb_config_string, ConfigString},
                 {ehttpc_pool_name, EHttpcPoolName},
+                {bridge_type, influxdb_api_v1},
+                {bridge_name, Name},
+                {bridge_config, InfluxDBConfig},
                 {influxdb_name, Name}
                 | Config
             ];
@@ -193,6 +196,9 @@ init_per_group(InfluxDBType, Config0) when
                 {influxdb_config, InfluxDBConfig},
                 {influxdb_config_string, ConfigString},
                 {ehttpc_pool_name, EHttpcPoolName},
+                {bridge_type, influxdb_api_v2},
+                {bridge_name, Name},
+                {bridge_config, InfluxDBConfig},
                 {influxdb_name, Name}
                 | Config
             ];
@@ -570,6 +576,10 @@ t_start_ok(Config) ->
     ),
     ok.
 
+t_start_stop(Config) ->
+    ok = emqx_bridge_testlib:t_start_stop(Config, influxdb_client_stopped),
+    ok.
+
 t_start_already_started(Config) ->
     Type = influxdb_type_bin(?config(influxdb_type, Config)),
     Name = ?config(influxdb_name, Config),

apps/emqx_bridge_kafka/rebar.config

@@ -1,6 +1,6 @@
 %% -*- mode: erlang; -*-
 {erl_opts, [debug_info]}.
-{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.6"}}}
+{deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.7"}}}
        , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.3"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0"}}}
        , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}}

apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge_kafka, [
     {description, "EMQX Enterprise Kafka Bridge"},
-    {vsn, "0.1.7"},
+    {vsn, "0.1.8"},
     {registered, [emqx_bridge_kafka_consumer_sup]},
     {applications, [
         kernel,

apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl

@@ -268,7 +268,8 @@ fields(producer_opts) ->
                 required => true,
                 desc => ?DESC(producer_kafka_opts),
                 validator => fun producer_strategy_key_validator/1
-            })}
+            })},
+        {resource_opts, mk(ref(resource_opts), #{default => #{}})}
     ];
 fields(producer_kafka_opts) ->
     [
@@ -425,7 +426,8 @@ fields(consumer_opts) ->
         {value_encoding_mode,
             mk(enum([none, base64]), #{
                 default => none, desc => ?DESC(consumer_value_encoding_mode)
-            })}
+            })},
+        {resource_opts, mk(ref(resource_opts), #{default => #{}})}
     ];
 fields(consumer_topic_mapping) ->
     [
@@ -460,10 +462,16 @@ fields(consumer_kafka_opts) ->
                 emqx_schema:timeout_duration_s(),
                 #{default => <<"5s">>, desc => ?DESC(consumer_offset_commit_interval_seconds)}
             )}
-    ].
+    ];
+fields(resource_opts) ->
+    SupportedFields = [health_check_interval],
+    CreationOpts = emqx_resource_schema:create_opts(_Overrides = []),
+    lists:filter(fun({Field, _}) -> lists:member(Field, SupportedFields) end, CreationOpts).
 
 desc("config") ->
     ?DESC("desc_config");
+desc(resource_opts) ->
+    ?DESC(emqx_resource_schema, "resource_opts");
 desc("get_" ++ Type) when Type =:= "consumer"; Type =:= "producer" ->
     ["Configuration for Kafka using `GET` method."];
 desc("put_" ++ Type) when Type =:= "consumer"; Type =:= "producer" ->

apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl

@@ -596,7 +596,6 @@ t_send_message_with_headers(Config) ->
         },
         KafkaMsg
     ),
-    ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg),
     %% TODO: refactor those into init/end per testcase
     ok = ?PRODUCER:on_stop(ResourceId, State),
     ?assertEqual([], supervisor:which_children(wolff_client_sup)),

apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl

@@ -306,6 +306,9 @@ kafka_producer_new_hocon() ->
     "      sndbuf = \"1024KB\"\n"
     "    }\n"
     "    ssl {enable = false, verify = \"verify_peer\"}\n"
+    "    resource_opts {\n"
+    "      health_check_interval = 10s\n"
+    "    }\n"
     "  }\n"
     "}\n"
     "".
@@ -351,5 +354,8 @@ bridges.kafka_consumer.my_consumer {
     verify = verify_none
     server_name_indication = \"auto\"
   }
+  resource_opts {
+    health_check_interval = 10s
+  }
 }
 """.

apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_kinesis, [
     {description, "EMQX Enterprise Amazon Kinesis Bridge"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl

@@ -111,7 +111,14 @@ init(#{
     erlcloud_config:configure(
         to_str(AwsAccessKey), to_str(AwsSecretAccessKey), Host, Port, Scheme, New
     ),
-    {ok, State}.
+    % check the connection
+    case erlcloud_kinesis:list_streams() of
+        {ok, _} ->
+            {ok, State};
+        {error, Reason} ->
+            ?tp(kinesis_init_failed, #{instance_id => InstanceId, reason => Reason}),
+            {stop, Reason}
+    end.
 
 handle_call(connection_status, _From, #{stream_name := StreamName} = State) ->
     Status =

apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl

@@ -114,7 +114,12 @@ on_get_status(_InstanceId, #{pool_name := Pool} = State) ->
                         false -> disconnected
                     end
             end;
-        {error, _} ->
+        {error, Reason} ->
+            ?SLOG(error, #{
+                msg => "kinesis_producer_get_status_failed",
+                state => State,
+                reason => Reason
+            }),
             disconnected
     end.
 

apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl

@@ -796,7 +796,9 @@ t_publish_connection_down(Config0) ->
     ok.
 
 t_wrong_server(Config) ->
+    TypeBin = ?BRIDGE_TYPE_BIN,
     Name = ?config(kinesis_name, Config),
+    KinesisConfig0 = ?config(kinesis_config, Config),
     ResourceId = ?config(resource_id, Config),
     Overrides =
         #{
@@ -806,12 +808,57 @@ t_wrong_server(Config) ->
                 <<"health_check_interval">> => <<"60s">>
             }
         },
+    % probe
+    KinesisConfig = emqx_utils_maps:deep_merge(KinesisConfig0, Overrides),
+    Params = KinesisConfig#{<<"type">> => TypeBin, <<"name">> => Name},
+    ProbePath = emqx_mgmt_api_test_util:api_path(["bridges_probe"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    ?assertMatch(
+        {error, {_, 400, _}},
+        emqx_mgmt_api_test_util:request_api(post, ProbePath, "", AuthHeader, Params)
+    ),
+    % create
     ?wait_async_action(
         create_bridge(Config, Overrides),
-        #{?snk_kind := emqx_bridge_kinesis_impl_producer_start_ok},
+        #{?snk_kind := start_pool_failed},
         30_000
     ),
-    ?assertEqual({error, timeout}, emqx_resource_manager:health_check(ResourceId)),
+    ?assertMatch(
-    emqx_bridge_resource:stop(?BRIDGE_TYPE, Name),
+        {ok, _, #{error := {start_pool_failed, ResourceId, _}}},
-    emqx_bridge_resource:remove(?BRIDGE_TYPE, Name),
+        emqx_resource_manager:lookup_cached(ResourceId)
+    ),
+    ok.
+
+t_access_denied(Config) ->
+    TypeBin = ?BRIDGE_TYPE_BIN,
+    Name = ?config(kinesis_name, Config),
+    KinesisConfig = ?config(kinesis_config, Config),
+    ResourceId = ?config(resource_id, Config),
+    AccessError = {<<"AccessDeniedException">>, <<>>},
+    Params = KinesisConfig#{<<"type">> => TypeBin, <<"name">> => Name},
+    ProbePath = emqx_mgmt_api_test_util:api_path(["bridges_probe"]),
+    AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
+    emqx_common_test_helpers:with_mock(
+        erlcloud_kinesis,
+        list_streams,
+        fun() -> {error, AccessError} end,
+        fun() ->
+            % probe
+            ?assertMatch(
+                {error, {_, 400, _}},
+                emqx_mgmt_api_test_util:request_api(post, ProbePath, "", AuthHeader, Params)
+            ),
+            % create
+            ?wait_async_action(
+                create_bridge(Config),
+                #{?snk_kind := kinesis_init_failed},
+                30_000
+            ),
+            ?assertMatch(
+                {ok, _, #{error := {start_pool_failed, ResourceId, AccessError}}},
+                emqx_resource_manager:lookup_cached(ResourceId)
+            ),
+            ok
+        end
+    ),
     ok.

apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl

@@ -29,7 +29,8 @@ group_tests() ->
         t_payload_template,
         t_collection_template,
         t_mongo_date_rule_engine_functions,
-        t_get_status_server_selection_too_short
+        t_get_status_server_selection_too_short,
+        t_use_legacy_protocol_option
     ].
 
 groups() ->
@@ -180,6 +181,7 @@ mongo_config(MongoHost, MongoPort0, rs = Type, Config) ->
             "  replica_set_name = rs0\n"
             "  servers = [~p]\n"
             "  w_mode = safe\n"
+            "  use_legacy_protocol = auto\n"
             "  database = mqtt\n"
             "  resource_opts = {\n"
             "    query_mode = ~s\n"
@@ -205,6 +207,7 @@ mongo_config(MongoHost, MongoPort0, sharded = Type, Config) ->
             "  collection = mycol\n"
             "  servers = [~p]\n"
             "  w_mode = safe\n"
+            "  use_legacy_protocol = auto\n"
             "  database = mqtt\n"
             "  resource_opts = {\n"
             "    query_mode = ~s\n"
@@ -230,6 +233,7 @@ mongo_config(MongoHost, MongoPort0, single = Type, Config) ->
             "  collection = mycol\n"
             "  server = ~p\n"
             "  w_mode = safe\n"
+            "  use_legacy_protocol = auto\n"
             "  database = mqtt\n"
             "  resource_opts = {\n"
             "    query_mode = ~s\n"
@@ -286,10 +290,8 @@ clear_db(Config) ->
     mongo_api:disconnect(Client).
 
 find_all(Config) ->
-    Type = mongo_type_bin(?config(mongo_type, Config)),
-    Name = ?config(mongo_name, Config),
     #{<<"collection">> := Collection} = ?config(mongo_config, Config),
-    ResourceID = emqx_bridge_resource:resource_id(Type, Name),
+    ResourceID = resource_id(Config),
     emqx_resource:simple_sync_query(ResourceID, {find, Collection, #{}, #{}}).
 
 find_all_wait_until_non_empty(Config) ->
@@ -340,6 +342,27 @@ probe_bridge_api(Config, Overrides) ->
     ct:pal("bridge probe result: ~p", [Res]),
     Res.
 
+resource_id(Config) ->
+    Type0 = ?config(mongo_type, Config),
+    Name = ?config(mongo_name, Config),
+    Type = mongo_type_bin(Type0),
+    emqx_bridge_resource:resource_id(Type, Name).
+
+get_worker_pids(Config) ->
+    ResourceID = resource_id(Config),
+    %% abusing health check api a bit...
+    GetWorkerPid = fun(TopologyPid) ->
+        mongoc:transaction_query(TopologyPid, fun(#{pool := WorkerPid}) -> WorkerPid end)
+    end,
+    {ok, WorkerPids = [_ | _]} =
+        emqx_resource_pool:health_check_workers(
+            ResourceID,
+            GetWorkerPid,
+            5_000,
+            #{return_values => true}
+        ),
+    WorkerPids.
+
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
@@ -494,3 +517,30 @@ t_get_status_server_selection_too_short(Config) ->
         emqx_utils_json:decode(Body)
     ),
     ok.
+
+t_use_legacy_protocol_option(Config) ->
+    ResourceID = resource_id(Config),
+    {ok, _} = create_bridge(Config, #{<<"use_legacy_protocol">> => <<"true">>}),
+    ?retry(
+        _Interval0 = 200,
+        _NAttempts0 = 20,
+        ?assertMatch({ok, connected}, emqx_resource_manager:health_check(ResourceID))
+    ),
+    WorkerPids0 = get_worker_pids(Config),
+    Expected0 = maps:from_keys(WorkerPids0, true),
+    LegacyOptions0 = maps:from_list([{Pid, mc_utils:use_legacy_protocol(Pid)} || Pid <- WorkerPids0]),
+    ?assertEqual(Expected0, LegacyOptions0),
+    {ok, _} = delete_bridge(Config),
+
+    {ok, _} = create_bridge(Config, #{<<"use_legacy_protocol">> => <<"false">>}),
+    ?retry(
+        _Interval0 = 200,
+        _NAttempts0 = 20,
+        ?assertMatch({ok, connected}, emqx_resource_manager:health_check(ResourceID))
+    ),
+    WorkerPids1 = get_worker_pids(Config),
+    Expected1 = maps:from_keys(WorkerPids1, false),
+    LegacyOptions1 = maps:from_list([{Pid, mc_utils:use_legacy_protocol(Pid)} || Pid <- WorkerPids1]),
+    ?assertEqual(Expected1, LegacyOptions1),
+
+    ok.

apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl

@@ -1089,7 +1089,7 @@ t_strategy_key_validation(Config) ->
                     #{
                         <<"kind">> := <<"validation_error">>,
                         <<"reason">> := <<"Message key cannot be empty", _/binary>>
-                    } = Msg
+                    }
             }}},
         probe_bridge_api(
             Config,
@@ -1103,7 +1103,7 @@ t_strategy_key_validation(Config) ->
                     #{
                         <<"kind">> := <<"validation_error">>,
                         <<"reason">> := <<"Message key cannot be empty", _/binary>>
-                    } = Msg
+                    }
             }}},
         create_bridge_api(
             Config,

apps/emqx_conf/src/emqx_conf.app.src

@@ -1,6 +1,6 @@
 {application, emqx_conf, [
     {description, "EMQX configuration management"},
-    {vsn, "0.1.25"},
+    {vsn, "0.1.26"},
     {registered, []},
     {mod, {emqx_conf_app, []}},
     {applications, [kernel, stdlib, emqx_ctl]},

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -63,6 +63,7 @@
     emqx_psk_schema,
     emqx_limiter_schema,
     emqx_slow_subs_schema,
+    emqx_otel_schema,
     emqx_mgmt_api_key_schema
 ]).
 %% 1 million default ports counter

apps/emqx_connector/rebar.config

@@ -9,7 +9,6 @@
     {emqx, {path, "../emqx"}},
     {emqx_utils, {path, "../emqx_utils"}},
     {emqx_resource, {path, "../emqx_resource"}},
-    {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
     {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}}
 ]}.
 

apps/emqx_connector/src/emqx_connector.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "EMQX Data Integration Connectors"},
-    {vsn, "0.1.29"},
+    {vsn, "0.1.30"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [
@@ -12,7 +12,6 @@
         eredis_cluster,
         eredis,
         epgsql,
-        eldap2,
         ehttpc,
         jose,
         emqx,

apps/emqx_dashboard/src/emqx_dashboard.app.src

@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.25"},
+    {vsn, "5.0.26"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl, emqx_bridge_http]},
@@ -12,6 +12,6 @@
     {maintainers, ["EMQX Team <contact@emqx.io>"]},
     {links, [
         {"Homepage", "https://emqx.io/"},
-        {"Github", "https://github.com/emqx/emqx-dashboard"}
+        {"Github", "https://github.com/emqx/emqx-dashboard5"}
     ]}
 ]}.

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -856,8 +856,6 @@ typename_to_spec("timeout()", _Mod) ->
     };
 typename_to_spec("bytesize()", _Mod) ->
     #{type => string, example => <<"32MB">>};
-typename_to_spec("mqtt_max_packet_size()", _Mod) ->
-    #{type => string, example => <<"32MB">>};
 typename_to_spec("wordsize()", _Mod) ->
     #{type => string, example => <<"1024KB">>};
 typename_to_spec("map()", _Mod) ->

apps/emqx_ft/src/emqx_ft.app.src

@@ -1,6 +1,6 @@
 {application, emqx_ft, [
     {description, "EMQX file transfer over MQTT"},
-    {vsn, "0.1.4"},
+    {vsn, "0.1.5"},
     {registered, []},
     {mod, {emqx_ft_app, []}},
     {applications, [

apps/emqx_ft/src/emqx_ft.erl

@@ -71,7 +71,7 @@
     %% the resulting file is corrupted during transmission).
     size => _Bytes :: non_neg_integer(),
     checksum => checksum(),
-    expire_at := emqx_datetime:epoch_second(),
+    expire_at := emqx_utils_calendar:epoch_second(),
     %% TTL of individual segments
     %% Somewhat confusing that we won't know it on the nodes where the filemeta
     %% is missing.

apps/emqx_ft/src/emqx_ft_api.erl

@@ -278,7 +278,7 @@ format_file_info(
     end.
 
 format_timestamp(Timestamp) ->
-    iolist_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])).
+    emqx_utils_calendar:epoch_to_rfc3339(Timestamp, second).
 
 format_name(NameBin) when is_binary(NameBin) ->
     NameBin;

apps/emqx_ft/src/emqx_ft_storage.erl

@@ -68,7 +68,7 @@
     transfer := emqx_ft:transfer(),
     name := file:name(),
     size := _Bytes :: non_neg_integer(),
-    timestamp := emqx_datetime:epoch_second(),
+    timestamp := emqx_utils_calendar:epoch_second(),
     uri => uri_string:uri_string(),
     meta => emqx_ft:filemeta()
 }.

apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl

@@ -43,7 +43,7 @@
     transfer := transfer(),
     name := file:name(),
     uri := uri_string:uri_string(),
-    timestamp := emqx_datetime:epoch_second(),
+    timestamp := emqx_utils_calendar:epoch_second(),
     size := _Bytes :: non_neg_integer(),
     filemeta => filemeta()
 }.

apps/emqx_ft/src/emqx_ft_storage_fs.erl

@@ -76,7 +76,7 @@
 % TODO naming
 -type filefrag(T) :: #{
     path := file:name(),
-    timestamp := emqx_datetime:epoch_second(),
+    timestamp := emqx_utils_calendar:epoch_second(),
     size := _Bytes :: non_neg_integer(),
     fragment := T
 }.

apps/emqx_gateway/src/emqx_gateway.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_gateway, [
     {description, "The Gateway management application"},
-    {vsn, "0.1.22"},
+    {vsn, "0.1.23"},
     {registered, []},
     {mod, {emqx_gateway_app, []}},
     {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]},

apps/emqx_gateway/src/emqx_gateway_api_clients.erl

@@ -397,13 +397,13 @@ format_channel_info(WhichNode, {_, Infos, Stats} = R) ->
         {ip_address, {peername, ConnInfo, fun peer_to_binary_addr/1}},
         {port, {peername, ConnInfo, fun peer_to_port/1}},
         {is_bridge, ClientInfo, false},
-        {connected_at, {connected_at, ConnInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}},
+        {connected_at, {connected_at, ConnInfo, fun emqx_utils_calendar:epoch_to_rfc3339/1}},
-        {disconnected_at, {disconnected_at, ConnInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}},
+        {disconnected_at, {disconnected_at, ConnInfo, fun emqx_utils_calendar:epoch_to_rfc3339/1}},
         {connected, {conn_state, Infos, fun conn_state_to_connected/1}},
         {keepalive, ClientInfo, 0},
         {clean_start, ConnInfo, true},
         {expiry_interval, ConnInfo, 0},
-        {created_at, {created_at, SessInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}},
+        {created_at, {created_at, SessInfo, fun emqx_utils_calendar:epoch_to_rfc3339/1}},
         {subscriptions_cnt, Stats, 0},
         {subscriptions_max, Stats, infinity},
         {inflight_cnt, Stats, 0},
@@ -640,28 +640,28 @@ params_client_searching_in_qs() ->
             )},
         {gte_created_at,
             mk(
-                emqx_datetime:epoch_millisecond(),
+                emqx_utils_calendar:epoch_millisecond(),
                 M#{
                     desc => ?DESC(param_gte_created_at)
                 }
             )},
         {lte_created_at,
             mk(
-                emqx_datetime:epoch_millisecond(),
+                emqx_utils_calendar:epoch_millisecond(),
                 M#{
                     desc => ?DESC(param_lte_created_at)
                 }
             )},
         {gte_connected_at,
             mk(
-                emqx_datetime:epoch_millisecond(),
+                emqx_utils_calendar:epoch_millisecond(),
                 M#{
                     desc => ?DESC(param_gte_connected_at)
                 }
             )},
         {lte_connected_at,
             mk(
-                emqx_datetime:epoch_millisecond(),
+                emqx_utils_calendar:epoch_millisecond(),
                 M#{
                     desc => ?DESC(param_lte_connected_at)
                 }
@@ -888,12 +888,12 @@ common_client_props() ->
             )},
         {connected_at,
             mk(
-                emqx_datetime:epoch_millisecond(),
+                emqx_utils_calendar:epoch_millisecond(),
                 #{desc => ?DESC(connected_at)}
             )},
         {disconnected_at,
             mk(
-                emqx_datetime:epoch_millisecond(),
+                emqx_utils_calendar:epoch_millisecond(),
                 #{
                     desc => ?DESC(disconnected_at)
                 }
@@ -931,7 +931,7 @@ common_client_props() ->
             )},
         {created_at,
             mk(
-                emqx_datetime:epoch_millisecond(),
+                emqx_utils_calendar:epoch_millisecond(),
                 #{desc => ?DESC(created_at)}
             )},
         {subscriptions_cnt,