Merge remote-tracking branch 'origin/release-57' into sync-release-57-20240728-022010

feat(cluster link): refactor http api, add status and metrics

.ci/docker-compose-file/docker-compose-azurite.yaml

@@ -0,0 +1,24 @@
+version: '3.9'
+
+services:
+  azurite:
+    container_name: azurite
+    image: mcr.microsoft.com/azure-storage/azurite:3.30.0
+    restart: always
+    expose:
+      - "10000"
+    # ports:
+    #   - "10000:10000"
+    networks:
+      - emqx_bridge
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:10000"]
+      interval: 30s
+      timeout: 5s
+      retries: 4
+    command:
+      - azurite-blob
+      - "--blobHost"
+      - 0.0.0.0
+      - "-d"
+      - debug.log

.ci/docker-compose-file/docker-compose-couchbase.yaml

@@ -0,0 +1,30 @@
+version: '3.9'
+
+services:
+  couchbase:
+    container_name: couchbase
+    hostname: couchbase
+    image: ghcr.io/emqx/couchbase:1.0.0
+    restart: always
+    expose:
+      - 8091-8093
+    # ports:
+    #   - "8091-8093:8091-8093"
+    networks:
+      - emqx_bridge
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8093/admin/ping"]
+      interval: 30s
+      timeout: 5s
+      retries: 4
+    environment:
+      - CLUSTER=localhost
+      - USER=admin
+      - PASS=public
+      - PORT=8091
+      - RAMSIZEMB=2048
+      - RAMSIZEINDEXMB=512
+      - RAMSIZEFTSMB=512
+      - BUCKETS=mqtt
+      - BUCKETSIZES=100
+      - AUTOREBALANCE=true

.ci/docker-compose-file/toxiproxy.json

@@ -215,5 +215,17 @@
     "listen": "0.0.0.0:9200",
     "upstream": "elasticsearch:9200",
     "enabled": true
+  },
+  {
+    "name": "azurite_plain",
+    "listen": "0.0.0.0:10000",
+    "upstream": "azurite:10000",
+    "enabled": true
+  },
+  {
+    "name": "couchbase",
+    "listen": "0.0.0.0:8093",
+    "upstream": "couchbase:8093",
+    "enabled": true
   }
 ]

.ci/fvt_tests/http_server/src/http_server.app.src

@@ -1,18 +1,18 @@
 %% -*- mode: erlang -*-
-{application, http_server,
+{application, http_server, [
- [{description, "An HTTP server application"},
+    {description, "An HTTP server application"},
-  {vsn, "0.2.0"},
+    {vsn, "0.2.0"},
-  {registered, []},
+    {registered, []},
-  % {mod, {http_server_app, []}},
+    % {mod, {http_server_app, []}},
-  {modules, []},
+    {modules, []},
-  {applications,
+    {applications, [
-   [kernel,
+        kernel,
-    stdlib,
+        stdlib,
-    minirest
+        minirest
-   ]},
+    ]},
-  {env,[]},
+    {env, []},
-  {modules, []},
+    {modules, []},
 
-  {licenses, ["Apache 2.0"]},
+    {licenses, ["Apache 2.0"]},
-  {links, []}
+    {links, []}
- ]}.
+]}.

.github/workflows/.zipignore2

@@ -0,0 +1 @@
+*/.github/*

.github/workflows/_pr_entrypoint.yaml

@@ -27,7 +27,7 @@ jobs:
       ELIXIR_VSN: ${{ steps.env.outputs.ELIXIR_VSN }}
       BUILDER: ${{ steps.env.outputs.BUILDER }}
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ github.event.inputs.ref }}
       - name: Set up environment
@@ -52,7 +52,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ github.event.inputs.ref }}
           fetch-depth: 0
@@ -136,7 +136,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           fetch-depth: 0
       - name: Work around https://github.com/actions/checkout/issues/766
@@ -152,7 +152,7 @@ jobs:
           echo "PROFILE=${PROFILE}" | tee -a .env
           echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env
           zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip .
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: ${{ matrix.profile }}
           path: ${{ matrix.profile }}.zip

.github/workflows/_push-entrypoint.yaml

@@ -35,7 +35,7 @@ jobs:
       BUILD_FROM: ${{ steps.env.outputs.BUILD_FROM }}
       RUN_FROM: ${{ steps.env.outputs.BUILD_FROM }}
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ github.event.inputs.ref }}
       - name: Set up environment
@@ -65,7 +65,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ github.event.inputs.ref }}
           fetch-depth: 0
@@ -147,7 +147,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ github.event.inputs.ref }}
           fetch-depth: 0
@@ -163,7 +163,7 @@ jobs:
           echo "PROFILE=${PROFILE}" | tee -a .env
           echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env
           zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip .
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: ${{ matrix.profile }}
           path: ${{ matrix.profile }}.zip

.github/workflows/build_and_push_docker_images.yaml

@@ -75,7 +75,7 @@ jobs:
           - arm64
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ github.event.inputs.ref }}
       - run: git config --global --add safe.directory "$PWD"
@@ -83,7 +83,7 @@ jobs:
         id: build
         run: |
           make ${{ matrix.profile }}-tgz
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: "${{ matrix.profile }}-${{ matrix.arch }}.tar.gz"
           path: "_packages/emqx*/emqx-*.tar.gz"
@@ -107,10 +107,10 @@ jobs:
           - ["${{ inputs.profile }}-elixir", "${{ inputs.profile == 'emqx' && 'docker.io,public.ecr.aws' || 'docker.io' }}"]
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ github.event.inputs.ref }}
-      - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
           pattern: "${{ matrix.profile[0] }}-*.tar.gz"
           path: _packages
@@ -129,17 +129,17 @@ jobs:
           sudo systemctl restart docker
 
       - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
-      - uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 # v3.2.0
+      - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
 
       - name: Login to hub.docker.com
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
         if: inputs.publish && contains(matrix.profile[1], 'docker.io')
         with:
           username: ${{ secrets.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_HUB_TOKEN }}
 
       - name: Login to AWS ECR
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
         if: inputs.publish && contains(matrix.profile[1], 'public.ecr.aws')
         with:
           registry: public.ecr.aws

.github/workflows/build_docker_for_test.yaml

@@ -26,7 +26,7 @@ jobs:
           - emqx-enterprise-elixir
 
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Set up environment
         id: env
         run: |
@@ -51,7 +51,7 @@ jobs:
         if: always()
         run: |
           docker save $_EMQX_DOCKER_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: "${{ env.EMQX_NAME }}-docker"
           path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz"

.github/workflows/build_packages.yaml

@@ -82,7 +82,7 @@ jobs:
           - ${{ inputs.otp_vsn }}
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         ref: ${{ github.event.inputs.ref }}
         fetch-depth: 0
@@ -95,7 +95,7 @@ jobs:
         apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
         apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
         apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: success()
       with:
         name: ${{ matrix.profile }}-${{ matrix.os }}-${{ matrix.otp }}
@@ -113,13 +113,11 @@ jobs:
           - ubuntu24.04
           - ubuntu22.04
           - ubuntu20.04
-          - ubuntu18.04
           - debian12
           - debian11
           - debian10
           - el9
           - el8
-          - el7
           - amzn2
           - amzn2023
         arch:
@@ -147,7 +145,7 @@ jobs:
         shell: bash
 
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         ref: ${{ github.event.inputs.ref }}
         fetch-depth: 0
@@ -182,7 +180,7 @@ jobs:
           --builder $BUILDER \
           --elixir $IS_ELIXIR \
           --pkgtype pkg
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       with:
         name: ${{ matrix.profile }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.with_elixir == 'yes' && '-elixir' || '' }}-${{ matrix.builder }}-${{ matrix.otp }}-${{ matrix.elixir }}
         path: _packages/${{ matrix.profile }}/
@@ -200,7 +198,7 @@ jobs:
         profile:
           - ${{ inputs.profile }}
     steps:
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         pattern: "${{ matrix.profile }}-*"
         path: packages/${{ matrix.profile }}

.github/workflows/build_packages_cron.yaml

@@ -37,7 +37,7 @@ jobs:
         shell: bash
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ matrix.profile[1] }}
           fetch-depth: 0
@@ -56,11 +56,11 @@ jobs:
       - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: success()
         with:
-          name: ${{ matrix.profile[0] }}-${{ matrix.os }}-${{ github.ref_name }}
+          name: ${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.os }}
           path: _packages/${{ matrix.profile[0] }}/
           retention-days: 7
       - name: Send notification to Slack
-        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
@@ -83,7 +83,7 @@ jobs:
           - macos-14-arm64
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ matrix.branch }}
           fetch-depth: 0
@@ -101,14 +101,14 @@ jobs:
           apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
           apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
           apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: success()
         with:
           name: ${{ matrix.profile }}-${{ matrix.os }}
           path: _packages/${{ matrix.profile }}/
           retention-days: 7
       - name: Send notification to Slack
-        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/build_slim_packages.yaml

@@ -32,7 +32,7 @@ jobs:
           - ["emqx-enterprise", "erlang", "x64"]
 
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         fetch-depth: 0
     - name: build tgz
@@ -47,7 +47,7 @@ jobs:
         path: _packages/${{ matrix.profile[0] }}/*
         retention-days: 7
         compression-level: 0
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       with:
         name: "${{ matrix.profile[0] }}-schema-dump-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
         path: |
@@ -69,7 +69,7 @@ jobs:
       EMQX_NAME: ${{ matrix.profile }}
 
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Set up environment
       id: env
       run: |
@@ -84,7 +84,7 @@ jobs:
         apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
         apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
         apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       with:
         name: ${{ matrix.os }}
         path: _packages/**/*

.github/workflows/check_deps_integrity.yaml

@@ -22,7 +22,7 @@ jobs:
         profile:
           - emqx-enterprise
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - run: make ensure-rebar3
       - run: ./scripts/check-deps-integrity.escript
@@ -37,7 +37,7 @@ jobs:
       - run: ./scripts/check-elixir-deps-discrepancies.exs
       - run: ./scripts/check-elixir-applications.exs
       - name: Upload produced lock files
-        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: failure()
         with:
           name: ${{ matrix.profile }}_produced_lock_files

.github/workflows/codeql.yaml

@@ -30,7 +30,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         ref: ${{ matrix.branch }}
 

.github/workflows/green_master.yaml

@@ -7,9 +7,6 @@ on:
     # run hourly
     - cron: "0 * * * *"
   workflow_dispatch:
-    inputs:
-      ref:
-        required: false
 
 permissions:
   contents: read
@@ -17,14 +14,20 @@ permissions:
 jobs:
   rerun-failed-jobs:
     if: github.repository_owner == 'emqx'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     permissions:
       checks: read
       actions: write
+    strategy:
+      fail-fast: false
+      matrix:
+        ref:
+          - master
+          - release-57
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
-          ref: ${{ github.event.inputs.ref || 'master' }}
+          ref: ${{ matrix.ref }}
 
       - name: run script
         shell: bash

.github/workflows/performance_test.yaml

@@ -32,7 +32,7 @@ jobs:
       PACKAGE_FILE: ${{ steps.package_file.outputs.PACKAGE_FILE }}
 
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         fetch-depth: 0
         ref: ${{ github.event.inputs.ref }}
@@ -52,7 +52,7 @@ jobs:
       id: package_file
       run: |
         echo "PACKAGE_FILE=$(find _packages/emqx -name 'emqx-*.deb' | head -n 1 | xargs basename)" >> $GITHUB_OUTPUT
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       with:
         name: emqx-ubuntu20.04
         path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }}
@@ -72,17 +72,17 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
         aws-region: eu-west-1
     - name: Checkout tf-emqx-performance-test
-      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         repository: emqx/tf-emqx-performance-test
         path: tf-emqx-performance-test
         ref: v0.2.3
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: emqx-ubuntu20.04
         path: tf-emqx-performance-test/
     - name: Setup Terraform
-      uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0
+      uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
       with:
         terraform_wrapper: false
     - name: run scenario
@@ -105,7 +105,7 @@ jobs:
         terraform destroy -auto-approve
         aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
     - name: Send notification to Slack
-      uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+      uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
       with:
         payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
     - name: terraform destroy
@@ -113,13 +113,13 @@ jobs:
       working-directory: ./tf-emqx-performance-test
       run: |
         terraform destroy -auto-approve
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: success()
       with:
         name: metrics
         path: |
           "./tf-emqx-performance-test/*.tar.gz"
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: failure()
       with:
         name: terraform
@@ -143,17 +143,17 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
         aws-region: eu-west-1
     - name: Checkout tf-emqx-performance-test
-      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         repository: emqx/tf-emqx-performance-test
         path: tf-emqx-performance-test
         ref: v0.2.3
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: emqx-ubuntu20.04
         path: tf-emqx-performance-test/
     - name: Setup Terraform
-      uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0
+      uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
       with:
         terraform_wrapper: false
     - name: run scenario
@@ -176,7 +176,7 @@ jobs:
         terraform destroy -auto-approve
         aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
     - name: Send notification to Slack
-      uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+      uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
       with:
         payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
     - name: terraform destroy
@@ -184,13 +184,13 @@ jobs:
       working-directory: ./tf-emqx-performance-test
       run: |
         terraform destroy -auto-approve
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: success()
       with:
         name: metrics
         path: |
           "./tf-emqx-performance-test/*.tar.gz"
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: failure()
       with:
         name: terraform
@@ -215,17 +215,17 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
         aws-region: eu-west-1
     - name: Checkout tf-emqx-performance-test
-      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         repository: emqx/tf-emqx-performance-test
         path: tf-emqx-performance-test
         ref: v0.2.3
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: emqx-ubuntu20.04
         path: tf-emqx-performance-test/
     - name: Setup Terraform
-      uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0
+      uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
       with:
         terraform_wrapper: false
     - name: run scenario
@@ -249,7 +249,7 @@ jobs:
         terraform destroy -auto-approve
         aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
     - name: Send notification to Slack
-      uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+      uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
       with:
         payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
     - name: terraform destroy
@@ -257,13 +257,13 @@ jobs:
       working-directory: ./tf-emqx-performance-test
       run: |
         terraform destroy -auto-approve
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: success()
       with:
         name: metrics
         path: |
           "./tf-emqx-performance-test/*.tar.gz"
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: failure()
       with:
         name: terraform
@@ -289,17 +289,17 @@ jobs:
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
         aws-region: eu-west-1
     - name: Checkout tf-emqx-performance-test
-      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         repository: emqx/tf-emqx-performance-test
         path: tf-emqx-performance-test
         ref: v0.2.3
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: emqx-ubuntu20.04
         path: tf-emqx-performance-test/
     - name: Setup Terraform
-      uses: hashicorp/setup-terraform@a1502cd9e758c50496cc9ac5308c4843bcd56d36 # v3.0.0
+      uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
       with:
         terraform_wrapper: false
     - name: run scenario
@@ -322,7 +322,7 @@ jobs:
         terraform destroy -auto-approve
         aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
     - name: Send notification to Slack
-      uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+      uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
       with:
         payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
     - name: terraform destroy
@@ -330,13 +330,13 @@ jobs:
       working-directory: ./tf-emqx-performance-test
       run: |
         terraform destroy -auto-approve
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: success()
       with:
         name: metrics
         path: |
           "./tf-emqx-performance-test/*.tar.gz"
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: failure()
       with:
         name: terraform

.github/workflows/release.yaml

@@ -36,7 +36,7 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ github.event.inputs.tag }}
       - name: Detect profile
@@ -106,16 +106,12 @@ jobs:
           push "debian/bullseye" "packages/$PROFILE-$VERSION-debian11-arm64.deb"
           push "debian/bookworm" "packages/$PROFILE-$VERSION-debian12-amd64.deb"
           push "debian/bookworm" "packages/$PROFILE-$VERSION-debian12-arm64.deb"
-          push "ubuntu/bionic" "packages/$PROFILE-$VERSION-ubuntu18.04-amd64.deb"
-          push "ubuntu/bionic" "packages/$PROFILE-$VERSION-ubuntu18.04-arm64.deb"
           push "ubuntu/focal" "packages/$PROFILE-$VERSION-ubuntu20.04-amd64.deb"
           push "ubuntu/focal" "packages/$PROFILE-$VERSION-ubuntu20.04-arm64.deb"
           push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-amd64.deb"
           push "ubuntu/jammy" "packages/$PROFILE-$VERSION-ubuntu22.04-arm64.deb"
           push "ubuntu/noble" "packages/$PROFILE-$VERSION-ubuntu24.04-amd64.deb"
           push "ubuntu/noble" "packages/$PROFILE-$VERSION-ubuntu24.04-arm64.deb"
-          push "el/7" "packages/$PROFILE-$VERSION-el7-amd64.rpm"
-          push "el/7" "packages/$PROFILE-$VERSION-el7-arm64.rpm"
           push "el/8" "packages/$PROFILE-$VERSION-el8-amd64.rpm"
           push "el/8" "packages/$PROFILE-$VERSION-el8-arm64.rpm"
           push "el/9" "packages/$PROFILE-$VERSION-el9-amd64.rpm"
@@ -135,7 +131,7 @@ jobs:
       checks: write
       actions: write
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: trigger re-run of app versions check on open PRs
         shell: bash
         env:

.github/workflows/run_conf_tests.yaml

@@ -25,7 +25,7 @@ jobs:
           - emqx
           - emqx-enterprise
     steps:
-      - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
           name: ${{ matrix.profile }}
       - name: extract artifact
@@ -39,10 +39,10 @@ jobs:
       - name: print erlang log
         if: failure()
         run: |
-          cat _build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.*
+          cat _build/${{ matrix.profile }}/rel/emqx/log/erlang.log.*
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: failure()
         with:
           name: conftest-logs-${{ matrix.profile }}
-          path: _build/${{ matrix.profile }}/rel/emqx/logs
+          path: _build/${{ matrix.profile }}/rel/emqx/log
           retention-days: 7

.github/workflows/run_docker_tests.yaml

@@ -28,7 +28,7 @@ jobs:
       EMQX_IMAGE_OLD_VERSION_TAG: ${{ matrix.profile[1] }}
 
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Set up environment
         id: env
         run: |
@@ -82,7 +82,7 @@ jobs:
           - mnesia
           - rlog
     steps:
-      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       - name: Set up environment
         id: env
         run: |

.github/workflows/run_emqx_app_tests.yaml

@@ -37,7 +37,7 @@ jobs:
       matrix: ${{ steps.matrix.outputs.matrix }}
       skip: ${{ steps.matrix.outputs.skip }}
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         fetch-depth: 0
     - name: prepare test matrix
@@ -72,7 +72,7 @@ jobs:
       run:
         shell: bash
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         fetch-depth: 0
     - name: run
@@ -95,7 +95,7 @@ jobs:
             echo "Suites: $SUITES"
             ./rebar3 as standalone_test ct --name 'test@127.0.0.1' -v --readable=true --suite="$SUITES"
         fi
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: failure()
       with:
         name: logs-emqx-app-tests-${{ matrix.type }}

.github/workflows/run_helm_tests.yaml

@@ -34,7 +34,7 @@ jobs:
         - ssl1.3
         - ssl1.2
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         path: source
     - name: Set up environment
@@ -164,7 +164,7 @@ jobs:
           fi
           sleep 1;
         done
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         repository: emqx/paho.mqtt.testing
         ref: develop-5.0

.github/workflows/run_jmeter_tests.yaml

@@ -12,7 +12,7 @@ jobs:
     steps:
     - name: Cache Jmeter
       id: cache-jmeter
-      uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1
+      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
       with:
         path: /tmp/apache-jmeter.tgz
         key: apache-jmeter-5.4.3.tgz
@@ -31,7 +31,7 @@ jobs:
         else
           wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz $ARCHIVE_URL
         fi
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       with:
         name: apache-jmeter.tgz
         path: /tmp/apache-jmeter.tgz
@@ -51,14 +51,14 @@ jobs:
 
     needs: jmeter_artifact
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Set up environment
       id: env
       run: |
         source env.sh
         PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
         echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: emqx-docker
         path: /tmp
@@ -95,7 +95,7 @@ jobs:
           echo "check logs failed"
           exit 1
         fi
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: always()
       with:
         name: jmeter_logs-advanced_feat-${{ matrix.scripts_type }}
@@ -120,14 +120,14 @@ jobs:
 
     needs: jmeter_artifact
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Set up environment
       id: env
       run: |
         source env.sh
         PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
         echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: emqx-docker
         path: /tmp
@@ -175,7 +175,7 @@ jobs:
       if: failure()
       run: |
         docker compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: always()
       with:
         name: jmeter_logs-pgsql_authn_authz-${{ matrix.scripts_type }}_${{ matrix.pgsql_tag }}
@@ -197,14 +197,14 @@ jobs:
 
     needs: jmeter_artifact
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Set up environment
       id: env
       run: |
         source env.sh
         PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
         echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: emqx-docker
         path: /tmp
@@ -248,7 +248,7 @@ jobs:
           echo "check logs failed"
           exit 1
         fi
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: always()
       with:
         name: jmeter_logs-mysql_authn_authz-${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}
@@ -266,14 +266,14 @@ jobs:
 
     needs: jmeter_artifact
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Set up environment
       id: env
       run: |
         source env.sh
         PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
         echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: emqx-docker
         path: /tmp
@@ -313,7 +313,7 @@ jobs:
           echo "check logs failed"
           exit 1
         fi
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: always()
       with:
         name: jmeter_logs-JWT_authn-${{ matrix.scripts_type }}
@@ -332,14 +332,14 @@ jobs:
 
     needs: jmeter_artifact
     steps:
-    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - name: Set up environment
       id: env
       run: |
         source env.sh
         PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
         echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: emqx-docker
         path: /tmp
@@ -370,7 +370,7 @@ jobs:
           echo "check logs failed"
           exit 1
         fi
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       if: always()
       with:
         name: jmeter_logs-built_in_database_authn_authz-${{ matrix.scripts_type }}

.github/workflows/run_relup_tests.yaml

@@ -25,7 +25,7 @@ jobs:
       run:
         shell: bash
     steps:
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: emqx-enterprise
     - name: extract artifact
@@ -45,7 +45,7 @@ jobs:
       run: |
         export PROFILE='emqx-enterprise'
         make emqx-enterprise-tgz
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       name: Upload built emqx and test scenario
       with:
         name: relup_tests_emqx_built
@@ -72,10 +72,10 @@ jobs:
       run:
         shell: bash
     steps:
-    - uses: erlef/setup-beam@2f0cc07b4b9bea248ae098aba9e1a8a1de5ec24c # v1.17.5
+    - uses: erlef/setup-beam@a6e26b22319003294c58386b6f25edbc7336819a # v1.18.0
       with:
         otp-version: 26.2.5
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       with:
         repository: hawk/lux
         ref: lux-2.8.1
@@ -88,7 +88,7 @@ jobs:
         ./configure
         make
         echo "$(pwd)/bin" >> $GITHUB_PATH
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       name: Download built emqx and test scenario
       with:
         name: relup_tests_emqx_built
@@ -111,7 +111,7 @@ jobs:
           docker logs node2.emqx.io | tee lux_logs/emqx2.log
           exit 1
         fi
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       name: Save debug data
       if: failure()
       with:

.github/workflows/run_test_cases.yaml

@@ -46,7 +46,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
           name: ${{ matrix.profile }}
 
@@ -90,7 +90,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
           name: ${{ matrix.profile }}
       - name: extract artifact
@@ -133,7 +133,7 @@ jobs:
         if: failure()
         run: tar -czf logs.tar.gz _build/test/logs
 
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: failure()
         with:
           name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-sg${{ matrix.suitegroup }}
@@ -164,7 +164,7 @@ jobs:
       CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-sg${{ matrix.suitegroup }}
 
     steps:
-      - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
           name: ${{ matrix.profile }}
       - name: extract artifact
@@ -193,7 +193,7 @@ jobs:
         if: failure()
         run: tar -czf logs.tar.gz _build/test/logs
 
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: failure()
         with:
           name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-sg${{ matrix.suitegroup }}
@@ -216,7 +216,7 @@ jobs:
     steps:
       - name: Coveralls finished
         if: github.repository == 'emqx/emqx'
-        uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # v2.2.3
+        uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0
         with:
           parallel-finished: true
           git-branch: ${{ github.ref }}

.github/workflows/scorecard.yaml

@@ -25,12 +25,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
+        uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3
         with:
           results_file: results.sarif
           results_format: sarif
@@ -40,7 +40,7 @@ jobs:
           publish_results: true
 
       - name: "Upload artifact"
-        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: SARIF file
           path: results.sarif

.github/workflows/spellcheck.yaml

@@ -19,7 +19,7 @@ jobs:
         - emqx-enterprise
     runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
     steps:
-      - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
           pattern: "${{ matrix.profile }}-schema-dump-*-x64"
           merge-multiple: true

.github/workflows/static_checks.yaml

@@ -30,14 +30,14 @@ jobs:
         include: ${{ fromJson(inputs.ct-matrix) }}
     container: "${{ inputs.builder }}"
     steps:
-      - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
           name: ${{ matrix.profile }}
       - name: extract artifact
         run: |
           unzip -o -q ${{ matrix.profile }}.zip
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
-      - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
         with:
           path: "emqx_dialyzer_${{ matrix.profile }}_plt"
           key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*') }}

.github/workflows/sync-release-branch.yaml

@@ -0,0 +1,88 @@
+name: Sync release branch
+
+concurrency:
+  group: sync-release-branch-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  schedule:
+    - cron:  '0 2 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  create-pr:
+    runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        branch:
+          - release-57
+
+    env:
+      SYNC_BRANCH: ${{ matrix.branch }}
+
+    defaults:
+      run:
+        shell: bash
+
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        with:
+          fetch-depth: 0
+
+      - name: create new branch
+        run: |
+          set -euxo pipefail
+          NEW_BRANCH_NAME=sync-${SYNC_BRANCH}-$(date +"%Y%m%d-%H%M%S")
+          echo "NEW_BRANCH_NAME=${NEW_BRANCH_NAME}" >> $GITHUB_ENV
+          git config --global user.name "${GITHUB_ACTOR}"
+          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git checkout -b ${NEW_BRANCH_NAME}
+          git merge origin/${SYNC_BRANCH} 2>&1 | tee merge.log
+          git push origin ${NEW_BRANCH_NAME}:${NEW_BRANCH_NAME}
+
+      - name: create pull request
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euxo pipefail
+          for pr in $(gh pr list --state open --base master --label sync-release-branch --search "Sync ${SYNC_BRANCH} in:title" --repo ${{ github.repository }} --json number --jq '.[] | .number'); do
+            gh pr close $pr --repo ${{ github.repository }} --delete-branch || true
+          done
+          gh pr create --title "Sync ${SYNC_BRANCH}" --body "Sync ${SYNC_BRANCH}" --base master --head ${NEW_BRANCH_NAME} --label sync-release-branch --repo ${{ github.repository }}
+
+      - name: Send notification to Slack
+        if: failure()
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        run: |
+          awk '{printf "%s\\n", $0}' merge.log > merge.log.1
+          cat <<EOF > payload.json
+          {
+            "blocks": [
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Automatic sync of ${SYNC_BRANCH} branch failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                }
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "\`\`\`$(cat merge.log.1)\`\`\`"
+                }
+              }
+            ]
+          }
+          EOF
+          curl -X POST -H 'Content-type: application/json' --data @payload.json "$SLACK_WEBHOOK_URL"

.github/workflows/upload-helm-charts.yaml

@@ -23,7 +23,7 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:
           ref: ${{ github.event.inputs.tag }}
       - name: Detect profile

Makefile

@@ -28,6 +28,8 @@ CT_COVER_EXPORT_PREFIX ?= $(PROFILE)
 
 export REBAR_GIT_CLONE_OPTIONS += --depth=1
 
+ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar
+
 .PHONY: default
 default: $(REBAR) $(PROFILE)
 
@@ -58,8 +60,12 @@ ensure-mix-rebar3: $(REBAR)
 ensure-mix-rebar: $(REBAR)
 	@mix local.rebar --if-missing --force
 
+
+.PHONY: elixir-common-deps
+elixir-common-deps: $(ELIXIR_COMMON_DEPS)
+
 .PHONY: mix-deps-get
-mix-deps-get: $(ELIXIR_COMMON_DEPS)
+mix-deps-get: elixir-common-deps
 	@mix deps.get
 
 .PHONY: eunit
@@ -238,7 +244,7 @@ $(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt))))
 ## relup target is to create relup instructions
 .PHONY: $(REL_PROFILES:%=%-relup)
 define gen-relup-target
-$1-relup: $1-relup-downloads $(COMMON_DEPS)
+$1-relup: $(COMMON_DEPS)
 	@$(BUILD) $1 relup
 endef
 ALL_TGZS = $(REL_PROFILES)
@@ -247,7 +253,7 @@ $(foreach zt,$(ALL_TGZS),$(eval $(call gen-relup-target,$(zt))))
 ## tgz target is to create a release package .tar.gz with relup
 .PHONY: $(REL_PROFILES:%=%-tgz)
 define gen-tgz-target
-$1-tgz: $1-relup
+$1-tgz: $(COMMON_DEPS)
 	@$(BUILD) $1 tgz
 endef
 ALL_TGZS = $(REL_PROFILES)
@@ -310,10 +316,20 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt))))
 
 .PHONY: fmt
 fmt: $(REBAR)
-	@$(SCRIPTS)/erlfmt -w 'apps/*/{src,include,priv,test,integration_test}/**/*.{erl,hrl,app.src,eterm}'
+	@find . \( -name '*.app.src' -o \
-	@$(SCRIPTS)/erlfmt -w 'apps/*/rebar.config' 'apps/emqx/rebar.config.script' '.ci/fvt_tests/http_server/rebar.config'
+						 -name '*.erl' -o \
-	@$(SCRIPTS)/erlfmt -w 'rebar.config' 'rebar.config.erl'
+					   -name '*.hrl' -o \
-	@$(SCRIPTS)/erlfmt -w 'scripts/*.escript' 'bin/*.escript' 'bin/nodetool'
+			  		 -name 'rebar.config' -o \
+			  		 -name '*.eterm' -o \
+			  		 -name '*.escript' \) \
+	                          -not -path '*/_build/*' \
+	                          -not -path '*/deps/*' \
+	                          -not -path '*/_checkouts/*' \
+	                          -type f \
+		| xargs $(SCRIPTS)/erlfmt -w
+	@$(SCRIPTS)/erlfmt -w 'apps/emqx/rebar.config.script'
+	@$(SCRIPTS)/erlfmt -w 'elvis.config'
+	@$(SCRIPTS)/erlfmt -w 'bin/nodetool'
 	@mix format
 
 .PHONY: clean-test-cluster-config

apps/emqx/include/asserts.hrl

@@ -45,6 +45,10 @@
 ).
 
 -define(assertReceive(PATTERN, TIMEOUT),
+    ?assertReceive(PATTERN, TIMEOUT, #{})
+).
+
+-define(assertReceive(PATTERN, TIMEOUT, EXTRA),
     (fun() ->
         receive
             X__V = PATTERN -> X__V
@@ -54,7 +58,8 @@
                     {module, ?MODULE},
                     {line, ?LINE},
                     {expression, (??PATTERN)},
-                    {mailbox, ?drainMailbox()}
+                    {mailbox, ?drainMailbox()},
+                    {extra_info, EXTRA}
                 ]}
             )
         end

apps/emqx/include/emqx.hrl

@@ -67,7 +67,7 @@
 
 -record(route, {
     topic :: binary(),
-    dest :: node() | {binary(), node()} | emqx_session:session_id()
+    dest :: node() | {binary(), node()} | emqx_session:session_id() | emqx_external_broker:dest()
 }).
 
 %%--------------------------------------------------------------------

apps/emqx/include/emqx_session.hrl

@@ -20,4 +20,11 @@
 -define(IS_SESSION_IMPL_MEM(S), (is_tuple(S) andalso element(1, S) =:= session)).
 -define(IS_SESSION_IMPL_DS(S), (is_map_key(id, S))).
 
+%% (Erlang) messages that a connection process should forward to the
+%% session handler.
+-record(session_message, {
+    message :: term()
+}).
+-define(session_message(MSG), #session_message{message = MSG}).
+
 -endif.

apps/emqx/include/http_api.hrl

@@ -86,5 +86,6 @@
     {'SOURCE_ERROR', <<"Source error">>},
     {'UPDATE_FAILED', <<"Update failed">>},
     {'REST_FAILED', <<"Reset source or config failed">>},
-    {'CLIENT_NOT_RESPONSE', <<"Client not responding">>}
+    {'CLIENT_NOT_RESPONSE', <<"Client not responding">>},
+    {'UNSUPPORTED_MEDIA_TYPE', <<"Unsupported media type">>}
 ]).

apps/emqx/include/logger.hrl

@@ -30,7 +30,10 @@
             logger:log(
                 Level,
                 (Data),
-                Meta
+                maps:merge(Meta, #{
+                    mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY},
+                    line => ?LINE
+                })
             );
         false ->
             ok

apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl

@@ -25,11 +25,16 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    TCApps = emqx_cth_suite:start(
+    case emqx_ds_test_helpers:skip_if_norepl() of
-        app_specs(),
+        false ->
-        #{work_dir => emqx_cth_suite:work_dir(Config)}
+            TCApps = emqx_cth_suite:start(
-    ),
+                app_specs(),
-    [{tc_apps, TCApps} | Config].
+                #{work_dir => emqx_cth_suite:work_dir(Config)}
+            ),
+            [{tc_apps, TCApps} | Config];
+        Yes ->
+            Yes
+    end.
 
 end_per_suite(Config) ->
     TCApps = ?config(tc_apps, Config),
@@ -158,7 +163,7 @@ mk_clientid(Prefix, ID) ->
 
 restart_node(Node, NodeSpec) ->
     ?tp(will_restart_node, #{}),
-    emqx_cth_cluster:restart(Node, NodeSpec),
+    emqx_cth_cluster:restart(NodeSpec),
     wait_nodeup(Node),
     ?tp(restarted_node, #{}),
     ok.
@@ -253,7 +258,7 @@ t_session_subscription_idempotency(Config) ->
 
             ok
         end,
-        fun(Trace) ->
+        fun(_Trace) ->
             Session = session_open(Node1, ClientId),
             ?assertMatch(
                 #{SubTopicFilter := #{}},
@@ -326,7 +331,7 @@ t_session_unsubscription_idempotency(Config) ->
 
             ok
         end,
-        fun(Trace) ->
+        fun(_Trace) ->
             Session = session_open(Node1, ClientId),
             ?assertEqual(
                 #{},

apps/emqx/mix.exs

@@ -8,7 +8,7 @@ defmodule EMQX.MixProject do
       app: :emqx,
       version: "0.1.0",
       build_path: "../../_build",
-      erlc_paths: UMP.erlc_paths(),
+      erlc_paths: erlc_paths(),
       erlc_options: [
         {:i, "src"}
         | UMP.erlc_options()
@@ -36,8 +36,9 @@ defmodule EMQX.MixProject do
   def deps() do
     ## FIXME!!! go though emqx.app.src and add missing stuff...
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx_utils, in_umbrella: true},
-      # {:emqx_ds_backends, in_umbrella: true},
+      {:emqx_ds_backends, in_umbrella: true},
 
       UMP.common_dep(:gproc),
       UMP.common_dep(:gen_rpc),
@@ -53,6 +54,15 @@ defmodule EMQX.MixProject do
     ] ++ UMP.quicer_dep()
   end
 
+  defp erlc_paths() do
+    paths = UMP.erlc_paths()
+    if UMP.test_env?() do
+      ["integration_test" | paths]
+    else
+      paths
+    end
+  end
+
   defp extra_dirs() do
     dirs = ["src", "etc"]
     if UMP.test_env?() do

apps/emqx/priv/bpapi.versions

@@ -10,12 +10,14 @@
 {emqx_bridge,5}.
 {emqx_bridge,6}.
 {emqx_broker,1}.
+{emqx_cluster_link,1}.
 {emqx_cm,1}.
 {emqx_cm,2}.
 {emqx_cm,3}.
 {emqx_conf,1}.
 {emqx_conf,2}.
 {emqx_conf,3}.
+{emqx_conf,4}.
 {emqx_connector,1}.
 {emqx_dashboard,1}.
 {emqx_delayed,1}.
@@ -47,6 +49,7 @@
 {emqx_mgmt_api_plugins,1}.
 {emqx_mgmt_api_plugins,2}.
 {emqx_mgmt_api_plugins,3}.
+{emqx_mgmt_api_relup,1}.
 {emqx_mgmt_cluster,1}.
 {emqx_mgmt_cluster,2}.
 {emqx_mgmt_cluster,3}.

apps/emqx/rebar.config

@@ -24,6 +24,7 @@
 {deps, [
     {emqx_utils, {path, "../emqx_utils"}},
     {emqx_durable_storage, {path, "../emqx_durable_storage"}},
+    {emqx_ds_backends, {path, "../emqx_ds_backends"}},
     {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}},
     {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},

apps/emqx/rebar.config.script

@@ -24,7 +24,8 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.313"}}}.
+Quicer =
+    {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.500"}}}.
 
 Dialyzer = fun(Config) ->
     {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),

apps/emqx/src/config/emqx_config_logger.erl

@@ -237,27 +237,29 @@ log_formatter(HandlerName, Conf) ->
             _ ->
                 conf_get("formatter", Conf)
         end,
-    TsFormat = timstamp_format(Conf),
+    TsFormat = timestamp_format(Conf),
+    WithMfa = conf_get("with_mfa", Conf),
     PayloadEncode = conf_get("payload_encode", Conf, text),
     do_formatter(
-        Format, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, PayloadEncode
+        Format, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa, PayloadEncode
     ).
 
 %% auto | epoch | rfc3339
-timstamp_format(Conf) ->
+timestamp_format(Conf) ->
     conf_get("timestamp_format", Conf).
 
 %% helpers
-do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, PayloadEncode) ->
+do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa, PayloadEncode) ->
     {emqx_logger_jsonfmt, #{
         chars_limit => CharsLimit,
         single_line => SingleLine,
         time_offset => TimeOffSet,
         depth => Depth,
         timestamp_format => TsFormat,
+        with_mfa => WithMfa,
         payload_encode => PayloadEncode
     }};
-do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, PayloadEncode) ->
+do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, WithMfa, PayloadEncode) ->
     {emqx_logger_textfmt, #{
         template => ["[", level, "] ", msg, "\n"],
         chars_limit => CharsLimit,
@@ -265,6 +267,7 @@ do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth, TsFormat, PayloadE
         time_offset => TimeOffSet,
         depth => Depth,
         timestamp_format => TsFormat,
+        with_mfa => WithMfa,
         payload_encode => PayloadEncode
     }}.
 

apps/emqx/src/emqx.app.src

@@ -18,7 +18,7 @@
         sasl,
         lc,
         hocon,
-        emqx_durable_storage,
+        emqx_ds_backends,
         bcrypt,
         pbkdf2,
         emqx_http_lib,

apps/emqx/src/emqx_banned.erl

@@ -33,6 +33,7 @@
 
 -export([
     check/1,
+    check_clientid/1,
     create/1,
     look_up/1,
     delete/1,
@@ -117,6 +118,10 @@ check(ClientInfo) ->
         do_check({peerhost, maps:get(peerhost, ClientInfo, undefined)}) orelse
         do_check_rules(ClientInfo).
 
+-spec check_clientid(emqx_types:clientid()) -> boolean().
+check_clientid(ClientId) ->
+    do_check({clientid, ClientId}) orelse do_check_rules(#{clientid => ClientId}).
+
 -spec format(emqx_types:banned()) -> map().
 format(#banned{
     who = Who0,

apps/emqx/src/emqx_broker.erl

@@ -244,11 +244,24 @@ publish(Msg) when is_record(Msg, message) ->
                 topic => Topic
             }),
             [];
-        Msg1 = #message{topic = Topic} ->
+        Msg1 = #message{} ->
-            PersistRes = persist_publish(Msg1),
+            do_publish(Msg1);
-            route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1), PersistRes)
+        Msgs when is_list(Msgs) ->
+            do_publish_many(Msgs)
     end.
 
+do_publish_many([]) ->
+    [];
+do_publish_many([Msg | T]) ->
+    do_publish(Msg) ++ do_publish_many(T).
+
+do_publish(#message{topic = Topic} = Msg) ->
+    PersistRes = persist_publish(Msg),
+    Routes = aggre(emqx_router:match_routes(Topic)),
+    Delivery = delivery(Msg),
+    RouteRes = route(Routes, Delivery, PersistRes),
+    do_forward_external(Delivery, RouteRes).
+
 persist_publish(Msg) ->
     case emqx_persistent_message:persist(Msg) of
         ok ->
@@ -332,6 +345,9 @@ aggre([], false, Acc) ->
 aggre([], true, Acc) ->
     lists:usort(Acc).
 
+do_forward_external(Delivery, RouteRes) ->
+    emqx_external_broker:forward(Delivery) ++ RouteRes.
+
 %% @doc Forward message to another node.
 -spec forward(
     node(), emqx_types:topic() | emqx_types:share(), emqx_types:delivery(), RpcMode :: sync | async
@@ -643,19 +659,27 @@ maybe_delete_route(Topic) ->
 
 sync_route(Action, Topic, ReplyTo) ->
     EnabledOn = emqx_config:get([broker, routing, batch_sync, enable_on]),
-    case EnabledOn of
+    Res =
-        all ->
+        case EnabledOn of
-            push_sync_route(Action, Topic, ReplyTo);
+            all ->
-        none ->
+                push_sync_route(Action, Topic, ReplyTo);
-            regular_sync_route(Action, Topic);
+            none ->
-        Role ->
+                regular_sync_route(Action, Topic);
-            case Role =:= mria_config:whoami() of
+            Role ->
-                true ->
+                case Role =:= mria_config:whoami() of
-                    push_sync_route(Action, Topic, ReplyTo);
+                    true ->
-                false ->
+                        push_sync_route(Action, Topic, ReplyTo);
-                    regular_sync_route(Action, Topic)
+                    false ->
-            end
+                        regular_sync_route(Action, Topic)
-    end.
+                end
+        end,
+    _ = external_sync_route(Action, Topic),
+    Res.
+
+external_sync_route(add, Topic) ->
+    emqx_external_broker:add_route(Topic);
+external_sync_route(delete, Topic) ->
+    emqx_external_broker:delete_route(Topic).
 
 push_sync_route(Action, Topic, Opts) ->
     emqx_router_syncer:push(Action, Topic, node(), Opts).

apps/emqx/src/emqx_broker_sup.erl

@@ -47,7 +47,7 @@ init([]) ->
         router_syncer_pool,
         hash,
         PoolSize,
-        {emqx_router_syncer, start_link, []}
+        {emqx_router_syncer, start_link_pooled, []}
     ]),
 
     %% Shared subscription

apps/emqx/src/emqx_channel.erl

@@ -19,6 +19,7 @@
 
 -include("emqx.hrl").
 -include("emqx_channel.hrl").
+-include("emqx_session.hrl").
 -include("emqx_mqtt.hrl").
 -include("emqx_access_control.hrl").
 -include("logger.hrl").
@@ -234,7 +235,7 @@ caps(#channel{clientinfo = #{zone := Zone}}) ->
 -spec init(emqx_types:conninfo(), opts()) -> channel().
 init(
     ConnInfo = #{
-        peername := {PeerHost, PeerPort},
+        peername := {PeerHost, PeerPort} = PeerName,
         sockname := {_Host, SockPort}
     },
     #{
@@ -258,6 +259,9 @@ init(
             listener => ListenerId,
             protocol => Protocol,
             peerhost => PeerHost,
+            %% We copy peername to clientinfo because some event contexts only have access
+            %% to client info (e.g.: authn/authz).
+            peername => PeerName,
             peerport => PeerPort,
             sockport => SockPort,
             clientid => undefined,
@@ -1315,6 +1319,9 @@ handle_info({'DOWN', Ref, process, Pid, Reason}, Channel) ->
         [] -> {ok, Channel};
         Msgs -> {ok, Msgs, Channel}
     end;
+handle_info(?session_message(Message), #channel{session = Session} = Channel) ->
+    NSession = emqx_session:handle_info(Message, Session),
+    {ok, Channel#channel{session = NSession}};
 handle_info(Info, Channel) ->
     ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {ok, Channel}.
@@ -1749,7 +1756,7 @@ maybe_add_cert(Map, #channel{conninfo = ConnInfo}) ->
     maybe_add_cert(Map, ConnInfo);
 maybe_add_cert(Map, #{peercert := PeerCert}) when is_binary(PeerCert) ->
     %% NOTE: it's raw binary at this point,
-    %% encoding to PEM (base64) is done lazy in emqx_authn_utils:render_var
+    %% encoding to PEM (base64) is done lazy in emqx_auth_utils:render_var
     Map#{cert_pem => PeerCert};
 maybe_add_cert(Map, _) ->
     Map.

apps/emqx/src/emqx_config.erl

@@ -499,15 +499,14 @@ fill_defaults(RawConf, Opts) ->
     ).
 
 -spec fill_defaults(module(), raw_config(), hocon_tconf:opts()) -> map().
-fill_defaults(_SchemaMod, RawConf = #{<<"durable_storage">> := _}, _) ->
+fill_defaults(SchemaMod, RawConf = #{<<"durable_storage">> := Ds}, Opts) ->
     %% FIXME: kludge to prevent `emqx_config' module from filling in
     %% the default values for backends and layouts. These records are
     %% inside unions, and adding default values there will add
     %% incompatible fields.
-    %%
+    RawConf1 = maps:remove(<<"durable_storage">>, RawConf),
-    %% Note: this function is called for each individual conf root, so
+    Conf = fill_defaults(SchemaMod, RawConf1, Opts),
-    %% this clause only affects this particular subtree.
+    Conf#{<<"durable_storage">> => Ds};
-    RawConf;
 fill_defaults(SchemaMod, RawConf, Opts0) ->
     Opts = maps:merge(#{required => false, make_serializable => true}, Opts0),
     hocon_tconf:check_plain(

apps/emqx/src/emqx_connection.erl

@@ -173,7 +173,9 @@
         system_code_change/4
     ]}
 ).
+-dialyzer({no_missing_calls, [handle_msg/2]}).
 
+-ifndef(BUILD_WITHOUT_QUIC).
 -spec start_link
     (esockd:transport(), esockd:socket(), emqx_channel:opts()) ->
         {ok, pid()};
@@ -183,6 +185,9 @@
         emqx_quic_connection:cb_state()
     ) ->
         {ok, pid()}.
+-else.
+-spec start_link(esockd:transport(), esockd:socket(), emqx_channel:opts()) -> {ok, pid()}.
+-endif.
 
 start_link(Transport, Socket, Options) ->
     Args = [self(), Transport, Socket, Options],
@@ -323,7 +328,7 @@ init_state(
         max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size])
     },
     ParseState = emqx_frame:initial_parse_state(FrameOpts),
-    Serialize = emqx_frame:serialize_opts(),
+    Serialize = emqx_frame:initial_serialize_opts(FrameOpts),
     %% Init Channel
     Channel = emqx_channel:init(ConnInfo, Opts),
     GcState =
@@ -468,19 +473,17 @@ cancel_stats_timer(State) ->
 process_msg([], State) ->
     {ok, State};
 process_msg([Msg | More], State) ->
-    try
+    try handle_msg(Msg, State) of
-        case handle_msg(Msg, State) of
+        ok ->
-            ok ->
+            process_msg(More, State);
-                process_msg(More, State);
+        {ok, NState} ->
-            {ok, NState} ->
+            process_msg(More, NState);
-                process_msg(More, NState);
+        {ok, Msgs, NState} ->
-            {ok, Msgs, NState} ->
+            process_msg(append_msg(More, Msgs), NState);
-                process_msg(append_msg(More, Msgs), NState);
+        {stop, Reason, NState} ->
-            {stop, Reason, NState} ->
+            {stop, Reason, NState};
-                {stop, Reason, NState};
+        {stop, Reason} ->
-            {stop, Reason} ->
+            {stop, Reason, State}
-                {stop, Reason, State}
-        end
     catch
         exit:normal ->
             {stop, normal, State};
@@ -566,12 +569,10 @@ handle_msg({Closed, _Sock}, State) when
 handle_msg({Passive, _Sock}, State) when
     Passive == tcp_passive; Passive == ssl_passive; Passive =:= quic_passive
 ->
-    %% In Stats
     Pubs = emqx_pd:reset_counter(incoming_pubs),
     Bytes = emqx_pd:reset_counter(incoming_bytes),
-    InStats = #{cnt => Pubs, oct => Bytes},
     %% Run GC and Check OOM
-    NState1 = check_oom(run_gc(InStats, State)),
+    NState1 = check_oom(Pubs, Bytes, run_gc(Pubs, Bytes, State)),
     handle_info(activate_socket, NState1);
 handle_msg(
     Deliver = {deliver, _Topic, _Msg},
@@ -899,8 +900,7 @@ sent(#state{listener = {Type, Listener}} = State) ->
         true ->
             Pubs = emqx_pd:reset_counter(outgoing_pubs),
             Bytes = emqx_pd:reset_counter(outgoing_bytes),
-            OutStats = #{cnt => Pubs, oct => Bytes},
+            {ok, check_oom(Pubs, Bytes, run_gc(Pubs, Bytes, State))};
-            {ok, check_oom(run_gc(OutStats, State))};
         false ->
             {ok, State}
     end.
@@ -993,17 +993,23 @@ check_limiter(
     Data,
     WhenOk,
     Msgs,
-    #state{limiter_timer = undefined, limiter = Limiter} = State
+    #state{channel = Channel, limiter_timer = undefined, limiter = Limiter} = State
 ) ->
     case emqx_limiter_container:check_list(Needs, Limiter) of
         {ok, Limiter2} ->
             WhenOk(Data, Msgs, State#state{limiter = Limiter2});
         {pause, Time, Limiter2} ->
-            ?SLOG(debug, #{
+            ?SLOG_THROTTLE(
-                msg => "pause_time_due_to_rate_limit",
+                warning,
-                needs => Needs,
+                #{
-                time_in_ms => Time
+                    msg => socket_receive_paused_by_rate_limit,
-            }),
+                    paused_ms => Time
+                },
+                #{
+                    tag => "RATE",
+                    clientid => emqx_channel:info(clientid, Channel)
+                }
+            ),
 
             Retry = #retry{
                 types = [Type || {_, Type} <- Needs],
@@ -1037,7 +1043,7 @@ check_limiter(
 
 %% try to perform a retry
 -spec retry_limiter(state()) -> _.
-retry_limiter(#state{limiter = Limiter} = State) ->
+retry_limiter(#state{channel = Channel, limiter = Limiter} = State) ->
     #retry{types = Types, data = Data, next = Next} =
         emqx_limiter_container:get_retry_context(Limiter),
     case emqx_limiter_container:retry_list(Types, Limiter) of
@@ -1051,11 +1057,17 @@ retry_limiter(#state{limiter = Limiter} = State) ->
                 }
             );
         {pause, Time, Limiter2} ->
-            ?SLOG(debug, #{
+            ?SLOG_THROTTLE(
-                msg => "pause_time_due_to_rate_limit",
+                warning,
-                types => Types,
+                #{
-                time_in_ms => Time
+                    msg => socket_receive_paused_by_rate_limit,
-            }),
+                    paused_ms => Time
+                },
+                #{
+                    tag => "RATE",
+                    clientid => emqx_channel:info(clientid, Channel)
+                }
+            ),
 
             TRef = start_timer(Time, limit_timeout),
 
@@ -1068,25 +1080,36 @@ retry_limiter(#state{limiter = Limiter} = State) ->
 %%--------------------------------------------------------------------
 %% Run GC and Check OOM
 
-run_gc(Stats, State = #state{gc_state = GcSt, zone = Zone}) ->
+run_gc(Pubs, Bytes, State = #state{gc_state = GcSt, zone = Zone}) ->
     case
         ?ENABLED(GcSt) andalso not emqx_olp:backoff_gc(Zone) andalso
-            emqx_gc:run(Stats, GcSt)
+            emqx_gc:run(Pubs, Bytes, GcSt)
     of
         false -> State;
         {_IsGC, GcSt1} -> State#state{gc_state = GcSt1}
     end.
 
-check_oom(State = #state{channel = Channel}) ->
+check_oom(Pubs, Bytes, State = #state{channel = Channel}) ->
     ShutdownPolicy = emqx_config:get_zone_conf(
         emqx_channel:info(zone, Channel), [force_shutdown]
     ),
-    ?tp(debug, check_oom, #{policy => ShutdownPolicy}),
     case emqx_utils:check_oom(ShutdownPolicy) of
         {shutdown, Reason} ->
             %% triggers terminate/2 callback immediately
+            ?tp(warning, check_oom_shutdown, #{
+                policy => ShutdownPolicy,
+                incoming_pubs => Pubs,
+                incoming_bytes => Bytes,
+                shutdown => Reason
+            }),
             erlang:exit({shutdown, Reason});
-        _ ->
+        Result ->
+            ?tp(debug, check_oom_ok, #{
+                policy => ShutdownPolicy,
+                incoming_pubs => Pubs,
+                incoming_bytes => Bytes,
+                result => Result
+            }),
             ok
     end,
     State.

apps/emqx/src/emqx_ds_schema.erl

@@ -18,7 +18,7 @@
 -module(emqx_ds_schema).
 
 %% API:
--export([schema/0, translate_builtin/1]).
+-export([schema/0, translate_builtin_raft/1, translate_builtin_local/1]).
 
 %% Behavior callbacks:
 -export([fields/1, desc/1, namespace/0]).
@@ -32,42 +32,51 @@
 %% Type declarations
 %%================================================================================
 
+-ifndef(EMQX_RELEASE_EDITION).
+-define(EMQX_RELEASE_EDITION, ce).
+-endif.
+
+-if(?EMQX_RELEASE_EDITION == ee).
+-define(DEFAULT_BACKEND, builtin_raft).
+-define(BUILTIN_BACKENDS, [ref(builtin_raft), ref(builtin_local)]).
+-else.
+-define(DEFAULT_BACKEND, builtin_local).
+-define(BUILTIN_BACKENDS, [ref(builtin_local)]).
+-endif.
+
 %%================================================================================
 %% API
 %%================================================================================
 
-translate_builtin(
+translate_builtin_raft(
     Backend = #{
-        backend := builtin,
+        backend := builtin_raft,
         n_shards := NShards,
         n_sites := NSites,
         replication_factor := ReplFactor,
         layout := Layout
     }
 ) ->
-    Storage =
-        case Layout of
-            #{
-                type := wildcard_optimized,
-                bits_per_topic_level := BitsPerTopicLevel,
-                epoch_bits := EpochBits,
-                topic_index_bytes := TIBytes
-            } ->
-                {emqx_ds_storage_bitfield_lts, #{
-                    bits_per_topic_level => BitsPerTopicLevel,
-                    topic_index_bytes => TIBytes,
-                    epoch_bits => EpochBits
-                }};
-            #{type := reference} ->
-                {emqx_ds_storage_reference, #{}}
-        end,
     #{
-        backend => builtin,
+        backend => builtin_raft,
         n_shards => NShards,
         n_sites => NSites,
         replication_factor => ReplFactor,
         replication_options => maps:get(replication_options, Backend, #{}),
-        storage => Storage
+        storage => translate_layout(Layout)
+    }.
+
+translate_builtin_local(
+    #{
+        backend := builtin_local,
+        n_shards := NShards,
+        layout := Layout
+    }
+) ->
+    #{
+        backend => builtin_local,
+        n_shards => NShards,
+        storage => translate_layout(Layout)
     }.
 
 %%================================================================================
@@ -83,24 +92,24 @@ schema() ->
             ds_schema(#{
                 default =>
                     #{
-                        <<"backend">> => builtin
+                        <<"backend">> => ?DEFAULT_BACKEND
                     },
                 importance => ?IMPORTANCE_MEDIUM,
                 desc => ?DESC(messages)
             })}
     ].
 
-fields(builtin) ->
+fields(builtin_local) ->
-    %% Schema for the builtin backend:
+    %% Schema for the builtin_raft backend:
     [
         {backend,
             sc(
-                builtin,
+                builtin_local,
                 #{
                     'readOnly' => true,
-                    default => builtin,
+                    default => builtin_local,
                     importance => ?IMPORTANCE_MEDIUM,
-                    desc => ?DESC(builtin_backend)
+                    desc => ?DESC(backend_type)
                 }
             )},
         {'_config_handler',
@@ -108,27 +117,32 @@ fields(builtin) ->
                 {module(), atom()},
                 #{
                     'readOnly' => true,
-                    default => {?MODULE, translate_builtin},
+                    default => {?MODULE, translate_builtin_local},
                     importance => ?IMPORTANCE_HIDDEN
                 }
-            )},
+            )}
-        {data_dir,
+        | common_builtin_fields()
+    ];
+fields(builtin_raft) ->
+    %% Schema for the builtin_raft backend:
+    [
+        {backend,
             sc(
-                string(),
+                builtin_raft,
                 #{
-                    mapping => "emqx_durable_storage.db_data_dir",
+                    'readOnly' => true,
-                    required => false,
+                    default => builtin_raft,
                     importance => ?IMPORTANCE_MEDIUM,
-                    desc => ?DESC(builtin_data_dir)
+                    desc => ?DESC(backend_type)
                 }
             )},
-        {n_shards,
+        {'_config_handler',
             sc(
-                pos_integer(),
+                {module(), atom()},
                 #{
-                    default => 12,
+                    'readOnly' => true,
-                    importance => ?IMPORTANCE_MEDIUM,
+                    default => {?MODULE, translate_builtin_raft},
-                    desc => ?DESC(builtin_n_shards)
+                    importance => ?IMPORTANCE_HIDDEN
                 }
             )},
         %% TODO: Deprecate once cluster management and rebalancing is implemented.
@@ -157,29 +171,10 @@ fields(builtin) ->
                     default => #{},
                     importance => ?IMPORTANCE_HIDDEN
                 }
-            )},
-        {local_write_buffer,
-            sc(
-                ref(builtin_local_write_buffer),
-                #{
-                    importance => ?IMPORTANCE_HIDDEN,
-                    desc => ?DESC(builtin_local_write_buffer)
-                }
-            )},
-        {layout,
-            sc(
-                hoconsc:union(builtin_layouts()),
-                #{
-                    desc => ?DESC(builtin_layout),
-                    importance => ?IMPORTANCE_MEDIUM,
-                    default =>
-                        #{
-                            <<"type">> => wildcard_optimized
-                        }
-                }
             )}
+        | common_builtin_fields()
     ];
-fields(builtin_local_write_buffer) ->
+fields(builtin_write_buffer) ->
     [
         {max_items,
             sc(
@@ -188,7 +183,7 @@ fields(builtin_local_write_buffer) ->
                     default => 1000,
                     mapping => "emqx_durable_storage.egress_batch_size",
                     importance => ?IMPORTANCE_HIDDEN,
-                    desc => ?DESC(builtin_local_write_buffer_max_items)
+                    desc => ?DESC(builtin_write_buffer_max_items)
                 }
             )},
         {flush_interval,
@@ -198,7 +193,7 @@ fields(builtin_local_write_buffer) ->
                     default => 100,
                     mapping => "emqx_durable_storage.egress_flush_interval",
                     importance => ?IMPORTANCE_HIDDEN,
-                    desc => ?DESC(builtin_local_write_buffer_flush_interval)
+                    desc => ?DESC(builtin_write_buffer_flush_interval)
                 }
             )}
     ];
@@ -239,6 +234,42 @@ fields(layout_builtin_wildcard_optimized) ->
                 }
             )}
     ];
+fields(layout_builtin_wildcard_optimized_v2) ->
+    [
+        {type,
+            sc(
+                wildcard_optimized_v2,
+                #{
+                    'readOnly' => true,
+                    default => wildcard_optimized_v2,
+                    desc => ?DESC(layout_builtin_wildcard_optimized_type)
+                }
+            )},
+        {bytes_per_topic_level,
+            sc(
+                range(1, 16),
+                #{
+                    default => 8,
+                    importance => ?IMPORTANCE_HIDDEN
+                }
+            )},
+        {topic_index_bytes,
+            sc(
+                pos_integer(),
+                #{
+                    default => 8,
+                    importance => ?IMPORTANCE_HIDDEN
+                }
+            )},
+        {serialization_schema,
+            sc(
+                emqx_ds_msg_serializer:schema(),
+                #{
+                    default => v1,
+                    importance => ?IMPORTANCE_HIDDEN
+                }
+            )}
+    ];
 fields(layout_builtin_reference) ->
     [
         {type,
@@ -247,17 +278,65 @@ fields(layout_builtin_reference) ->
                 #{
                     'readOnly' => true,
                     importance => ?IMPORTANCE_LOW,
+                    default => reference,
                     desc => ?DESC(layout_builtin_reference_type)
                 }
             )}
     ].
 
-desc(builtin) ->
+common_builtin_fields() ->
-    ?DESC(builtin);
+    [
-desc(builtin_local_write_buffer) ->
+        {data_dir,
-    ?DESC(builtin_local_write_buffer);
+            sc(
+                string(),
+                #{
+                    mapping => "emqx_durable_storage.db_data_dir",
+                    required => false,
+                    importance => ?IMPORTANCE_MEDIUM,
+                    desc => ?DESC(builtin_data_dir)
+                }
+            )},
+        {n_shards,
+            sc(
+                pos_integer(),
+                #{
+                    default => 16,
+                    importance => ?IMPORTANCE_MEDIUM,
+                    desc => ?DESC(builtin_n_shards)
+                }
+            )},
+        {local_write_buffer,
+            sc(
+                ref(builtin_write_buffer),
+                #{
+                    importance => ?IMPORTANCE_HIDDEN,
+                    desc => ?DESC(builtin_write_buffer)
+                }
+            )},
+        {layout,
+            sc(
+                hoconsc:union(builtin_layouts()),
+                #{
+                    desc => ?DESC(builtin_layout),
+                    importance => ?IMPORTANCE_MEDIUM,
+                    default =>
+                        #{
+                            <<"type">> => wildcard_optimized_v2
+                        }
+                }
+            )}
+    ].
+
+desc(builtin_raft) ->
+    ?DESC(builtin_raft);
+desc(builtin_local) ->
+    ?DESC(builtin_local);
+desc(builtin_write_buffer) ->
+    ?DESC(builtin_write_buffer);
 desc(layout_builtin_wildcard_optimized) ->
     ?DESC(layout_builtin_wildcard_optimized);
+desc(layout_builtin_wildcard_optimized_v2) ->
+    ?DESC(layout_builtin_wildcard_optimized);
 desc(layout_builtin_reference) ->
     ?DESC(layout_builtin_reference);
 desc(_) ->
@@ -267,12 +346,40 @@ desc(_) ->
 %% Internal functions
 %%================================================================================
 
+translate_layout(
+    #{
+        type := wildcard_optimized_v2,
+        bytes_per_topic_level := BytesPerTopicLevel,
+        topic_index_bytes := TopicIndexBytes,
+        serialization_schema := SSchema
+    }
+) ->
+    {emqx_ds_storage_skipstream_lts, #{
+        wildcard_hash_bytes => BytesPerTopicLevel,
+        topic_index_bytes => TopicIndexBytes,
+        serialization_schema => SSchema
+    }};
+translate_layout(
+    #{
+        type := wildcard_optimized,
+        bits_per_topic_level := BitsPerTopicLevel,
+        epoch_bits := EpochBits,
+        topic_index_bytes := TIBytes
+    }
+) ->
+    {emqx_ds_storage_bitfield_lts, #{
+        bits_per_topic_level => BitsPerTopicLevel,
+        topic_index_bytes => TIBytes,
+        epoch_bits => EpochBits
+    }};
+translate_layout(#{type := reference}) ->
+    {emqx_ds_storage_reference, #{}}.
+
 ds_schema(Options) ->
     sc(
-        hoconsc:union([
+        hoconsc:union(
-            ref(builtin)
+            ?BUILTIN_BACKENDS ++ emqx_schema_hooks:injection_point('durable_storage.backends', [])
-            | emqx_schema_hooks:injection_point('durable_storage.backends', [])
+        ),
-        ]),
         Options
     ).
 
@@ -281,7 +388,11 @@ builtin_layouts() ->
     %% suitable for production use. However, it's very simple and
     %% produces a very predictabale replay order, which can be useful
     %% for testing and debugging:
-    [ref(layout_builtin_wildcard_optimized), ref(layout_builtin_reference)].
+    [
+        ref(layout_builtin_wildcard_optimized_v2),
+        ref(layout_builtin_wildcard_optimized),
+        ref(layout_builtin_reference)
+    ].
 
 sc(Type, Meta) -> hoconsc:mk(Type, Meta).
 

apps/emqx/src/emqx_external_broker.erl

@@ -0,0 +1,140 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_external_broker).
+
+-callback forward(emqx_types:delivery()) ->
+    emqx_types:publish_result().
+
+-callback add_route(emqx_types:topic()) -> ok.
+-callback delete_route(emqx_types:topic()) -> ok.
+
+-callback add_shared_route(emqx_types:topic(), emqx_types:group()) -> ok.
+-callback delete_shared_route(emqx_types:topic(), emqx_types:group()) -> ok.
+
+-callback add_persistent_route(emqx_types:topic(), emqx_persistent_session_ds:id()) -> ok.
+-callback delete_persistent_route(emqx_types:topic(), emqx_persistent_session_ds:id()) -> ok.
+
+-type dest() :: term().
+
+-export([
+    %% Registration
+    provider/0,
+    register_provider/1,
+    unregister_provider/1,
+    %% Forwarding
+    forward/1,
+    %% Routing updates
+    add_route/1,
+    delete_route/1,
+    add_shared_route/2,
+    delete_shared_route/2,
+    add_persistent_route/2,
+    delete_persistent_route/2
+]).
+
+-export_type([dest/0]).
+
+-include("logger.hrl").
+
+-define(PROVIDER, {?MODULE, external_broker}).
+
+-define(safe_with_provider(IfRegistered, IfNotRegistered),
+    case persistent_term:get(?PROVIDER, undefined) of
+        undefined ->
+            IfNotRegistered;
+        Provider ->
+            try
+                Provider:IfRegistered
+            catch
+                Err:Reason:St ->
+                    ?SLOG_THROTTLE(error, #{
+                        msg => external_broker_crashed,
+                        provider => Provider,
+                        callback => ?FUNCTION_NAME,
+                        stacktrace => St,
+                        error => Err,
+                        reason => Reason
+                    }),
+                    {error, Reason}
+            end
+    end
+).
+
+%% TODO: provider API copied from emqx_external_traces,
+%% but it can be moved to a common module.
+
+%%--------------------------------------------------------------------
+%% Provider API
+%%--------------------------------------------------------------------
+
+-spec register_provider(module()) -> ok | {error, term()}.
+register_provider(Module) when is_atom(Module) ->
+    case is_valid_provider(Module) of
+        true ->
+            persistent_term:put(?PROVIDER, Module);
+        false ->
+            {error, invalid_provider}
+    end.
+
+-spec unregister_provider(module()) -> ok | {error, term()}.
+unregister_provider(Module) ->
+    case persistent_term:get(?PROVIDER, undefined) of
+        Module ->
+            persistent_term:erase(?PROVIDER),
+            ok;
+        _ ->
+            {error, not_registered}
+    end.
+
+-spec provider() -> module() | undefined.
+provider() ->
+    persistent_term:get(?PROVIDER, undefined).
+
+%%--------------------------------------------------------------------
+%% Broker API
+%%--------------------------------------------------------------------
+
+forward(Delivery) ->
+    ?safe_with_provider(?FUNCTION_NAME(Delivery), []).
+
+add_route(Topic) ->
+    ?safe_with_provider(?FUNCTION_NAME(Topic), ok).
+
+delete_route(Topic) ->
+    ?safe_with_provider(?FUNCTION_NAME(Topic), ok).
+
+add_shared_route(Topic, Group) ->
+    ?safe_with_provider(?FUNCTION_NAME(Topic, Group), ok).
+
+delete_shared_route(Topic, Group) ->
+    ?safe_with_provider(?FUNCTION_NAME(Topic, Group), ok).
+
+add_persistent_route(Topic, ID) ->
+    ?safe_with_provider(?FUNCTION_NAME(Topic, ID), ok).
+
+delete_persistent_route(Topic, ID) ->
+    ?safe_with_provider(?FUNCTION_NAME(Topic, ID), ok).
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+is_valid_provider(Module) ->
+    lists:all(
+        fun({F, A}) -> erlang:function_exported(Module, F, A) end,
+        ?MODULE:behaviour_info(callbacks)
+    ).

apps/emqx/src/emqx_frame.erl

@@ -29,11 +29,12 @@
     parse/2,
     serialize_fun/0,
     serialize_fun/1,
-    serialize_opts/0,
+    initial_serialize_opts/1,
     serialize_opts/1,
     serialize_pkt/2,
     serialize/1,
-    serialize/2
+    serialize/2,
+    serialize/3
 ]).
 
 -export([describe_state/1]).
@@ -84,7 +85,7 @@
 
 -define(MULTIPLIER_MAX, 16#200000).
 
--dialyzer({no_match, [serialize_utf8_string/2]}).
+-dialyzer({no_match, [serialize_utf8_string/3]}).
 
 %% @doc Describe state for logging.
 describe_state(?NONE(_Opts)) ->
@@ -725,43 +726,51 @@ serialize_fun() -> serialize_fun(?DEFAULT_OPTIONS).
 
 serialize_fun(#mqtt_packet_connect{proto_ver = ProtoVer, properties = ConnProps}) ->
     MaxSize = get_property('Maximum-Packet-Size', ConnProps, ?MAX_PACKET_SIZE),
-    serialize_fun(#{version => ProtoVer, max_size => MaxSize});
+    serialize_fun(#{version => ProtoVer, max_size => MaxSize, strict_mode => false});
-serialize_fun(#{version := Ver, max_size := MaxSize}) ->
+serialize_fun(#{version := Ver, max_size := MaxSize, strict_mode := StrictMode}) ->
     fun(Packet) ->
-        IoData = serialize(Packet, Ver),
+        IoData = serialize(Packet, Ver, StrictMode),
         case is_too_large(IoData, MaxSize) of
             true -> <<>>;
             false -> IoData
         end
     end.
 
-serialize_opts() ->
+initial_serialize_opts(Opts) ->
-    ?DEFAULT_OPTIONS.
+    maps:merge(?DEFAULT_OPTIONS, Opts).
 
 serialize_opts(#mqtt_packet_connect{proto_ver = ProtoVer, properties = ConnProps}) ->
     MaxSize = get_property('Maximum-Packet-Size', ConnProps, ?MAX_PACKET_SIZE),
-    #{version => ProtoVer, max_size => MaxSize}.
+    #{version => ProtoVer, max_size => MaxSize, strict_mode => false}.
 
-serialize_pkt(Packet, #{version := Ver, max_size := MaxSize}) ->
+serialize_pkt(Packet, #{version := Ver, max_size := MaxSize, strict_mode := StrictMode}) ->
-    IoData = serialize(Packet, Ver),
+    IoData = serialize(Packet, Ver, StrictMode),
     case is_too_large(IoData, MaxSize) of
         true -> <<>>;
         false -> IoData
     end.
 
 -spec serialize(emqx_types:packet()) -> iodata().
-serialize(Packet) -> serialize(Packet, ?MQTT_PROTO_V4).
+serialize(Packet) -> serialize(Packet, ?MQTT_PROTO_V4, false).
 
--spec serialize(emqx_types:packet(), emqx_types:proto_ver()) -> iodata().
+serialize(Packet, Ver) -> serialize(Packet, Ver, false).
+
+-spec serialize(emqx_types:packet(), emqx_types:proto_ver(), boolean()) -> iodata().
 serialize(
     #mqtt_packet{
         header = Header,
         variable = Variable,
         payload = Payload
     },
-    Ver
+    Ver,
+    StrictMode
 ) ->
-    serialize(Header, serialize_variable(Variable, Ver), serialize_payload(Payload)).
+    serialize(
+        Header,
+        serialize_variable(Variable, Ver, StrictMode),
+        serialize_payload(Payload),
+        StrictMode
+    ).
 
 serialize(
     #mqtt_packet_header{
@@ -771,7 +780,8 @@ serialize(
         retain = Retain
     },
     VariableBin,
-    PayloadBin
+    PayloadBin,
+    _StrictMode
 ) when
     ?CONNECT =< Type andalso Type =< ?AUTH
 ->
@@ -803,7 +813,8 @@ serialize_variable(
         username = Username,
         password = Password
     },
-    _Ver
+    _Ver,
+    StrictMode
 ) ->
     [
         serialize_binary_data(ProtoName),
@@ -821,20 +832,20 @@ serialize_variable(
             0:1,
             KeepAlive:16/big-unsigned-integer
         >>,
-        serialize_properties(Properties, ProtoVer),
+        serialize_properties(Properties, ProtoVer, StrictMode),
-        serialize_utf8_string(ClientId),
+        serialize_utf8_string(ClientId, StrictMode),
         case WillFlag of
             true ->
                 [
-                    serialize_properties(WillProps, ProtoVer),
+                    serialize_properties(WillProps, ProtoVer, StrictMode),
-                    serialize_utf8_string(WillTopic),
+                    serialize_utf8_string(WillTopic, StrictMode),
                     serialize_binary_data(WillPayload)
                 ];
             false ->
                 <<>>
         end,
-        serialize_utf8_string(Username, true),
+        serialize_utf8_string(Username, true, StrictMode),
-        serialize_utf8_string(Password, true)
+        serialize_utf8_string(Password, true, StrictMode)
     ];
 serialize_variable(
     #mqtt_packet_connack{
@@ -842,26 +853,28 @@ serialize_variable(
         reason_code = ReasonCode,
         properties = Properties
     },
-    Ver
+    Ver,
+    StrictMode
 ) ->
-    [AckFlags, ReasonCode, serialize_properties(Properties, Ver)];
+    [AckFlags, ReasonCode, serialize_properties(Properties, Ver, StrictMode)];
 serialize_variable(
     #mqtt_packet_publish{
         topic_name = TopicName,
         packet_id = PacketId,
         properties = Properties
     },
-    Ver
+    Ver,
+    StrictMode
 ) ->
     [
-        serialize_utf8_string(TopicName),
+        serialize_utf8_string(TopicName, StrictMode),
         case PacketId of
             undefined -> <<>>;
             _ -> <<PacketId:16/big-unsigned-integer>>
         end,
-        serialize_properties(Properties, Ver)
+        serialize_properties(Properties, Ver, StrictMode)
     ];
-serialize_variable(#mqtt_packet_puback{packet_id = PacketId}, Ver) when
+serialize_variable(#mqtt_packet_puback{packet_id = PacketId}, Ver, _StrictMode) when
     Ver == ?MQTT_PROTO_V3; Ver == ?MQTT_PROTO_V4
 ->
     <<PacketId:16/big-unsigned-integer>>;
@@ -871,12 +884,13 @@ serialize_variable(
         reason_code = ReasonCode,
         properties = Properties
     },
-    Ver = ?MQTT_PROTO_V5
+    Ver = ?MQTT_PROTO_V5,
+    StrictMode
 ) ->
     [
         <<PacketId:16/big-unsigned-integer>>,
         ReasonCode,
-        serialize_properties(Properties, Ver)
+        serialize_properties(Properties, Ver, StrictMode)
     ];
 serialize_variable(
     #mqtt_packet_subscribe{
@@ -884,12 +898,13 @@ serialize_variable(
         properties = Properties,
         topic_filters = TopicFilters
     },
-    Ver
+    Ver,
+    StrictMode
 ) ->
     [
         <<PacketId:16/big-unsigned-integer>>,
-        serialize_properties(Properties, Ver),
+        serialize_properties(Properties, Ver, StrictMode),
-        serialize_topic_filters(subscribe, TopicFilters, Ver)
+        serialize_topic_filters(subscribe, TopicFilters, Ver, StrictMode)
     ];
 serialize_variable(
     #mqtt_packet_suback{
@@ -897,11 +912,12 @@ serialize_variable(
         properties = Properties,
         reason_codes = ReasonCodes
     },
-    Ver
+    Ver,
+    StrictMode
 ) ->
     [
         <<PacketId:16/big-unsigned-integer>>,
-        serialize_properties(Properties, Ver),
+        serialize_properties(Properties, Ver, StrictMode),
         serialize_reason_codes(ReasonCodes)
     ];
 serialize_variable(
@@ -910,12 +926,13 @@ serialize_variable(
         properties = Properties,
         topic_filters = TopicFilters
     },
-    Ver
+    Ver,
+    StrictMode
 ) ->
     [
         <<PacketId:16/big-unsigned-integer>>,
-        serialize_properties(Properties, Ver),
+        serialize_properties(Properties, Ver, StrictMode),
-        serialize_topic_filters(unsubscribe, TopicFilters, Ver)
+        serialize_topic_filters(unsubscribe, TopicFilters, Ver, StrictMode)
     ];
 serialize_variable(
     #mqtt_packet_unsuback{
@@ -923,14 +940,15 @@ serialize_variable(
         properties = Properties,
         reason_codes = ReasonCodes
     },
-    Ver
+    Ver,
+    StrictMode
 ) ->
     [
         <<PacketId:16/big-unsigned-integer>>,
-        serialize_properties(Properties, Ver),
+        serialize_properties(Properties, Ver, StrictMode),
         serialize_reason_codes(ReasonCodes)
     ];
-serialize_variable(#mqtt_packet_disconnect{}, Ver) when
+serialize_variable(#mqtt_packet_disconnect{}, Ver, _StrictMode) when
     Ver == ?MQTT_PROTO_V3; Ver == ?MQTT_PROTO_V4
 ->
     <<>>;
@@ -939,110 +957,115 @@ serialize_variable(
         reason_code = ReasonCode,
         properties = Properties
     },
-    Ver = ?MQTT_PROTO_V5
+    Ver = ?MQTT_PROTO_V5,
+    StrictMode
 ) ->
-    [ReasonCode, serialize_properties(Properties, Ver)];
+    [ReasonCode, serialize_properties(Properties, Ver, StrictMode)];
-serialize_variable(#mqtt_packet_disconnect{}, _Ver) ->
+serialize_variable(#mqtt_packet_disconnect{}, _Ver, _StrictMode) ->
     <<>>;
 serialize_variable(
     #mqtt_packet_auth{
         reason_code = ReasonCode,
         properties = Properties
     },
-    Ver = ?MQTT_PROTO_V5
+    Ver = ?MQTT_PROTO_V5,
+    StrictMode
 ) ->
-    [ReasonCode, serialize_properties(Properties, Ver)];
+    [ReasonCode, serialize_properties(Properties, Ver, StrictMode)];
-serialize_variable(PacketId, ?MQTT_PROTO_V3) when is_integer(PacketId) ->
+serialize_variable(PacketId, ?MQTT_PROTO_V3, _StrictMode) when is_integer(PacketId) ->
     <<PacketId:16/big-unsigned-integer>>;
-serialize_variable(PacketId, ?MQTT_PROTO_V4) when is_integer(PacketId) ->
+serialize_variable(PacketId, ?MQTT_PROTO_V4, _StrictMode) when is_integer(PacketId) ->
     <<PacketId:16/big-unsigned-integer>>;
-serialize_variable(undefined, _Ver) ->
+serialize_variable(undefined, _Ver, _StrictMode) ->
     <<>>.
 
 serialize_payload(undefined) -> <<>>;
 serialize_payload(Bin) -> Bin.
 
-serialize_properties(_Props, Ver) when Ver =/= ?MQTT_PROTO_V5 ->
+serialize_properties(_Props, Ver, _StrictMode) when Ver =/= ?MQTT_PROTO_V5 ->
     <<>>;
-serialize_properties(Props, ?MQTT_PROTO_V5) ->
+serialize_properties(Props, ?MQTT_PROTO_V5, StrictMode) ->
-    serialize_properties(Props).
+    serialize_properties(Props, StrictMode).
 
-serialize_properties(undefined) ->
+serialize_properties(undefined, _StrictMode) ->
     <<0>>;
-serialize_properties(Props) when map_size(Props) == 0 ->
+serialize_properties(Props, _StrictMode) when map_size(Props) == 0 ->
     <<0>>;
-serialize_properties(Props) when is_map(Props) ->
+serialize_properties(Props, StrictMode) when is_map(Props) ->
-    Bin = <<<<(serialize_property(Prop, Val))/binary>> || {Prop, Val} <- maps:to_list(Props)>>,
+    Bin = <<
+        <<(serialize_property(Prop, Val, StrictMode))/binary>>
+     || {Prop, Val} <- maps:to_list(Props)
+    >>,
     [serialize_variable_byte_integer(byte_size(Bin)), Bin].
 
-serialize_property(_, Disabled) when Disabled =:= disabled; Disabled =:= undefined ->
+serialize_property(_, Disabled, _StrictMode) when Disabled =:= disabled; Disabled =:= undefined ->
     <<>>;
-serialize_property(internal_extra, _) ->
+serialize_property(internal_extra, _, _StrictMode) ->
     <<>>;
-serialize_property('Payload-Format-Indicator', Val) ->
+serialize_property('Payload-Format-Indicator', Val, _StrictMode) ->
     <<16#01, Val>>;
-serialize_property('Message-Expiry-Interval', Val) ->
+serialize_property('Message-Expiry-Interval', Val, _StrictMode) ->
     <<16#02, Val:32/big>>;
-serialize_property('Content-Type', Val) ->
+serialize_property('Content-Type', Val, StrictMode) ->
-    <<16#03, (serialize_utf8_string(Val))/binary>>;
+    <<16#03, (serialize_utf8_string(Val, StrictMode))/binary>>;
-serialize_property('Response-Topic', Val) ->
+serialize_property('Response-Topic', Val, StrictMode) ->
-    <<16#08, (serialize_utf8_string(Val))/binary>>;
+    <<16#08, (serialize_utf8_string(Val, StrictMode))/binary>>;
-serialize_property('Correlation-Data', Val) ->
+serialize_property('Correlation-Data', Val, _StrictMode) ->
     <<16#09, (byte_size(Val)):16, Val/binary>>;
-serialize_property('Subscription-Identifier', Val) ->
+serialize_property('Subscription-Identifier', Val, _StrictMode) ->
     <<16#0B, (serialize_variable_byte_integer(Val))/binary>>;
-serialize_property('Session-Expiry-Interval', Val) ->
+serialize_property('Session-Expiry-Interval', Val, _StrictMode) ->
     <<16#11, Val:32/big>>;
-serialize_property('Assigned-Client-Identifier', Val) ->
+serialize_property('Assigned-Client-Identifier', Val, StrictMode) ->
-    <<16#12, (serialize_utf8_string(Val))/binary>>;
+    <<16#12, (serialize_utf8_string(Val, StrictMode))/binary>>;
-serialize_property('Server-Keep-Alive', Val) ->
+serialize_property('Server-Keep-Alive', Val, _StrictMode) ->
     <<16#13, Val:16/big>>;
-serialize_property('Authentication-Method', Val) ->
+serialize_property('Authentication-Method', Val, StrictMode) ->
-    <<16#15, (serialize_utf8_string(Val))/binary>>;
+    <<16#15, (serialize_utf8_string(Val, StrictMode))/binary>>;
-serialize_property('Authentication-Data', Val) ->
+serialize_property('Authentication-Data', Val, _StrictMode) ->
     <<16#16, (iolist_size(Val)):16, Val/binary>>;
-serialize_property('Request-Problem-Information', Val) ->
+serialize_property('Request-Problem-Information', Val, _StrictMode) ->
     <<16#17, Val>>;
-serialize_property('Will-Delay-Interval', Val) ->
+serialize_property('Will-Delay-Interval', Val, _StrictMode) ->
     <<16#18, Val:32/big>>;
-serialize_property('Request-Response-Information', Val) ->
+serialize_property('Request-Response-Information', Val, _StrictMode) ->
     <<16#19, Val>>;
-serialize_property('Response-Information', Val) ->
+serialize_property('Response-Information', Val, StrictMode) ->
-    <<16#1A, (serialize_utf8_string(Val))/binary>>;
+    <<16#1A, (serialize_utf8_string(Val, StrictMode))/binary>>;
-serialize_property('Server-Reference', Val) ->
+serialize_property('Server-Reference', Val, StrictMode) ->
-    <<16#1C, (serialize_utf8_string(Val))/binary>>;
+    <<16#1C, (serialize_utf8_string(Val, StrictMode))/binary>>;
-serialize_property('Reason-String', Val) ->
+serialize_property('Reason-String', Val, StrictMode) ->
-    <<16#1F, (serialize_utf8_string(Val))/binary>>;
+    <<16#1F, (serialize_utf8_string(Val, StrictMode))/binary>>;
-serialize_property('Receive-Maximum', Val) ->
+serialize_property('Receive-Maximum', Val, _StrictMode) ->
     <<16#21, Val:16/big>>;
-serialize_property('Topic-Alias-Maximum', Val) ->
+serialize_property('Topic-Alias-Maximum', Val, _StrictMode) ->
     <<16#22, Val:16/big>>;
-serialize_property('Topic-Alias', Val) ->
+serialize_property('Topic-Alias', Val, _StrictMode) ->
     <<16#23, Val:16/big>>;
-serialize_property('Maximum-QoS', Val) ->
+serialize_property('Maximum-QoS', Val, _StrictMode) ->
     <<16#24, Val>>;
-serialize_property('Retain-Available', Val) ->
+serialize_property('Retain-Available', Val, _StrictMode) ->
     <<16#25, Val>>;
-serialize_property('User-Property', {Key, Val}) ->
+serialize_property('User-Property', {Key, Val}, StrictMode) ->
-    <<16#26, (serialize_utf8_pair({Key, Val}))/binary>>;
+    <<16#26, (serialize_utf8_pair(Key, Val, StrictMode))/binary>>;
-serialize_property('User-Property', Props) when is_list(Props) ->
+serialize_property('User-Property', Props, StrictMode) when is_list(Props) ->
     <<
-        <<(serialize_property('User-Property', {Key, Val}))/binary>>
+        <<(serialize_property('User-Property', {Key, Val}, StrictMode))/binary>>
      || {Key, Val} <- Props
     >>;
-serialize_property('Maximum-Packet-Size', Val) ->
+serialize_property('Maximum-Packet-Size', Val, _StrictMode) ->
     <<16#27, Val:32/big>>;
-serialize_property('Wildcard-Subscription-Available', Val) ->
+serialize_property('Wildcard-Subscription-Available', Val, _StrictMode) ->
     <<16#28, Val>>;
-serialize_property('Subscription-Identifier-Available', Val) ->
+serialize_property('Subscription-Identifier-Available', Val, _StrictMode) ->
     <<16#29, Val>>;
-serialize_property('Shared-Subscription-Available', Val) ->
+serialize_property('Shared-Subscription-Available', Val, _StrictMode) ->
     <<16#2A, Val>>.
 
-serialize_topic_filters(subscribe, TopicFilters, ?MQTT_PROTO_V5) ->
+serialize_topic_filters(subscribe, TopicFilters, ?MQTT_PROTO_V5, StrictMode) ->
     <<
         <<
-            (serialize_utf8_string(Topic))/binary,
+            (serialize_utf8_string(Topic, StrictMode))/binary,
             ?RESERVED:2,
             Rh:2,
             (flag(Rap)):1,
@@ -1051,37 +1074,42 @@ serialize_topic_filters(subscribe, TopicFilters, ?MQTT_PROTO_V5) ->
         >>
      || {Topic, #{rh := Rh, rap := Rap, nl := Nl, qos := QoS}} <- TopicFilters
     >>;
-serialize_topic_filters(subscribe, TopicFilters, _Ver) ->
+serialize_topic_filters(subscribe, TopicFilters, _Ver, StrictMode) ->
     <<
-        <<(serialize_utf8_string(Topic))/binary, ?RESERVED:6, QoS:2>>
+        <<(serialize_utf8_string(Topic, StrictMode))/binary, ?RESERVED:6, QoS:2>>
      || {Topic, #{qos := QoS}} <- TopicFilters
     >>;
-serialize_topic_filters(unsubscribe, TopicFilters, _Ver) ->
+serialize_topic_filters(unsubscribe, TopicFilters, _Ver, StrictMode) ->
-    <<<<(serialize_utf8_string(Topic))/binary>> || Topic <- TopicFilters>>.
+    <<<<(serialize_utf8_string(Topic, StrictMode))/binary>> || Topic <- TopicFilters>>.
 
 serialize_reason_codes(undefined) ->
     <<>>;
 serialize_reason_codes(ReasonCodes) when is_list(ReasonCodes) ->
     <<<<Code>> || Code <- ReasonCodes>>.
 
-serialize_utf8_pair({Name, Value}) ->
+serialize_utf8_pair(Name, Value, StrictMode) ->
-    <<(serialize_utf8_string(Name))/binary, (serialize_utf8_string(Value))/binary>>.
+    <<
+        (serialize_utf8_string(Name, StrictMode))/binary,
+        (serialize_utf8_string(Value, StrictMode))/binary
+    >>.
 
 serialize_binary_data(Bin) ->
     [<<(byte_size(Bin)):16/big-unsigned-integer>>, Bin].
 
-serialize_utf8_string(undefined, false) ->
+serialize_utf8_string(undefined, false, _StrictMode) ->
     ?SERIALIZE_ERR(utf8_string_undefined);
-serialize_utf8_string(undefined, true) ->
+serialize_utf8_string(undefined, true, _StrictMode) ->
     <<>>;
-serialize_utf8_string(String, _AllowNull) ->
+serialize_utf8_string(String, _AllowNull, StrictMode) ->
-    serialize_utf8_string(String).
+    serialize_utf8_string(String, StrictMode).
 
-serialize_utf8_string(String) ->
+serialize_utf8_string(String, true) ->
     StringBin = unicode:characters_to_binary(String),
-    Len = byte_size(StringBin),
+    serialize_utf8_string(StringBin, false);
+serialize_utf8_string(String, false) ->
+    Len = byte_size(String),
     true = (Len =< 16#ffff),
-    <<Len:16/big, StringBin/binary>>.
+    <<Len:16/big, String/binary>>.
 
 serialize_remaining_len(I) ->
     serialize_variable_byte_integer(I).

apps/emqx/src/emqx_gc.erl

@@ -30,7 +30,6 @@
 
 -export([
     init/1,
-    run/2,
     run/3,
     info/1,
     reset/1
@@ -62,12 +61,7 @@ init(#{count := Count, bytes := Bytes}) ->
     Oct = [{oct, {Bytes, Bytes}} || ?ENABLED(Bytes)],
     ?GCS(maps:from_list(Cnt ++ Oct)).
 
-%% @doc Try to run GC based on reduntions of count or bytes.
+%% @doc Try to run GC based on reductions of count or bytes.
--spec run(#{cnt := pos_integer(), oct := pos_integer()}, gc_state()) ->
-    {boolean(), gc_state()}.
-run(#{cnt := Cnt, oct := Oct}, GcSt) ->
-    run(Cnt, Oct, GcSt).
-
 -spec run(pos_integer(), pos_integer(), gc_state()) ->
     {boolean(), gc_state()}.
 run(Cnt, Oct, ?GCS(St)) ->

apps/emqx/src/emqx_listeners.erl

@@ -64,6 +64,17 @@
 
 -export_type([listener_id/0]).
 
+-dialyzer(
+    {no_unknown, [
+        is_running/3,
+        current_conns/3,
+        do_stop_listener/3,
+        do_start_listener/4,
+        do_update_listener/4,
+        quic_listener_conf_rollback/3
+    ]}
+).
+
 -type listener_id() :: atom() | binary().
 -type listener_type() :: tcp | ssl | ws | wss | quic | dtls.
 
@@ -1018,7 +1029,6 @@ ensure_max_conns(<<"infinity">>) -> <<"infinity">>;
 ensure_max_conns(MaxConn) when is_binary(MaxConn) -> binary_to_integer(MaxConn);
 ensure_max_conns(MaxConn) -> MaxConn.
 
--spec quic_listen_on(X :: any()) -> quicer:listen_on().
 quic_listen_on(Bind) ->
     case Bind of
         {Addr, Port} when tuple_size(Addr) == 4 ->

apps/emqx/src/emqx_logger_jsonfmt.erl

@@ -294,6 +294,7 @@ json_obj_root(Data0, Config) ->
             _ ->
                 json(Msg1, Config)
         end,
+    MFA = emqx_utils:format_mfal(Data0, Config),
     Data =
         maps:fold(
             fun(K, V, D) ->
@@ -302,12 +303,12 @@ json_obj_root(Data0, Config) ->
             end,
             [],
             maps:without(
-                [time, gl, file, report_cb, msg, '$kind', level, is_trace], Data0
+                [time, gl, file, report_cb, msg, '$kind', level, mfa, is_trace], Data0
             )
         ),
     lists:filter(
         fun({_, V}) -> V =/= undefined end,
-        [{time, format_ts(Time, Config)}, {level, Level}, {msg, Msg}]
+        [{time, format_ts(Time, Config)}, {level, Level}, {msg, Msg}, {mfa, MFA}]
     ) ++ Data.
 
 format_ts(Ts, #{timestamp_format := rfc3339, time_offset := Offset}) when is_integer(Ts) ->

apps/emqx/src/emqx_logger_textfmt.erl

@@ -25,7 +25,7 @@
 -export([evaluate_lazy_values_if_dbg_level/1, evaluate_lazy_values/1]).
 
 check_config(X) ->
-    logger_formatter:check_config(maps:without([timestamp_format, payload_encode], X)).
+    logger_formatter:check_config(maps:without([timestamp_format, with_mfa, payload_encode], X)).
 
 %% Principle here is to delegate the formatting to logger_formatter:format/2
 %% as much as possible, and only enrich the report with clientid, peername, topic, username
@@ -41,18 +41,24 @@ format(#{msg := {report, ReportMap0}, meta := _Meta} = Event0, Config) when is_m
             false ->
                 maps:from_list(ReportList)
         end,
-    fmt(Event#{msg := {report, Report}}, Config);
+    fmt(Event#{msg := {report, Report}}, maps:remove(with_mfa, Config));
 format(#{msg := {string, String}} = Event, Config) ->
     %% copied from logger_formatter:format/2
     %% unsure how this case is triggered
-    format(Event#{msg => {"~ts ", [String]}}, Config);
+    format(Event#{msg => {"~ts ", [String]}}, maps:remove(with_mfa, Config));
 format(#{msg := _Msg, meta := _Meta} = Event0, Config) ->
     #{msg := Msg0, meta := Meta} = Event1 = evaluate_lazy_values_if_dbg_level(Event0),
     %% For format strings like logger:log(Level, "~p", [Var])
     %% and logger:log(Level, "message", #{key => value})
     Msg1 = enrich_client_info(Msg0, Meta),
-    Msg2 = enrich_topic(Msg1, Meta),
+    Msg2 = enrich_mfa(Msg1, Meta, Config),
-    fmt(Event1#{msg := Msg2}, Config).
+    Msg3 = enrich_topic(Msg2, Meta),
+    fmt(Event1#{msg := Msg3}, maps:remove(with_mfa, Config)).
+
+enrich_mfa({Fmt, Args}, Data, #{with_mfa := true} = Config) when is_list(Fmt) ->
+    {Fmt ++ " mfa: ~ts", Args ++ [emqx_utils:format_mfal(Data, Config)]};
+enrich_mfa(Msg, _, _) ->
+    Msg.
 
 %% Most log entries with lazy values are trace events with level debug. So to
 %% be more efficient we only search for lazy values to evaluate in the entries
@@ -119,6 +125,7 @@ enrich_report(ReportRaw0, Meta, Config) ->
     ClientId = maps:get(clientid, Meta, undefined),
     Peer = maps:get(peername, Meta, undefined),
     Msg = maps:get(msg, ReportRaw, undefined),
+    MFA = emqx_utils:format_mfal(Meta, Config),
     %% TODO: move all tags to Meta so we can filter traces
     %% based on tags (currently not supported)
     Tag = maps:get(tag, ReportRaw, maps:get(tag, Meta, undefined)),
@@ -133,6 +140,7 @@ enrich_report(ReportRaw0, Meta, Config) ->
             {topic, try_format_unicode(Topic)},
             {username, try_format_unicode(Username)},
             {peername, Peer},
+            {mfa, try_format_unicode(MFA)},
             {msg, Msg},
             {clientid, try_format_unicode(ClientId)},
             {tag, Tag}

apps/emqx/src/emqx_persistent_session_ds.erl

@@ -25,6 +25,7 @@
 
 -include("emqx_mqtt.hrl").
 
+-include("emqx_session.hrl").
 -include("emqx_persistent_session_ds/session_internals.hrl").
 
 -ifdef(TEST).
@@ -63,6 +64,7 @@
     deliver/3,
     replay/3,
     handle_timeout/3,
+    handle_info/2,
     disconnect/2,
     terminate/2
 ]).
@@ -106,6 +108,7 @@
     seqno/0,
     timestamp/0,
     topic_filter/0,
+    share_topic_filter/0,
     subscription_id/0,
     subscription/0,
     session/0,
@@ -117,7 +120,8 @@
 %% Currently, this is the clientid.  We avoid `emqx_types:clientid()' because that can be
 %% an atom, in theory (?).
 -type id() :: binary().
--type topic_filter() :: emqx_types:topic() | #share{}.
+-type share_topic_filter() :: #share{}.
+-type topic_filter() :: emqx_types:topic() | share_topic_filter().
 
 %% Subscription and subscription states:
 %%
@@ -155,6 +159,8 @@
     subopts := map()
 }.
 
+-type shared_sub_state() :: term().
+
 -define(TIMER_PULL, timer_pull).
 -define(TIMER_GET_STREAMS, timer_get_streams).
 -define(TIMER_BUMP_LAST_ALIVE_AT, timer_bump_last_alive_at).
@@ -172,8 +178,13 @@
     props := map(),
     %% Persistent state:
     s := emqx_persistent_session_ds_state:t(),
+    %% Shared subscription state:
+    shared_sub_s := shared_sub_state(),
     %% Buffer:
     inflight := emqx_persistent_session_ds_inflight:t(),
+    %% Last fetched stream:
+    %% Used as a continuation point for fair stream scheduling.
+    last_fetched_stream => emqx_persistent_session_ds_state:stream_key(),
     %% In-progress replay:
     %% List of stream replay states to be added to the inflight buffer.
     replay => [{_StreamKey, stream_state()}, ...],
@@ -277,8 +288,11 @@ info(created_at, #{s := S}) ->
     emqx_persistent_session_ds_state:get_created_at(S);
 info(is_persistent, #{}) ->
     true;
-info(subscriptions, #{s := S}) ->
+info(subscriptions, #{s := S, shared_sub_s := SharedSubS}) ->
-    emqx_persistent_session_ds_subs:to_map(S);
+    maps:merge(
+        emqx_persistent_session_ds_subs:to_map(S),
+        emqx_persistent_session_ds_shared_subs:to_map(S, SharedSubS)
+    );
 info(subscriptions_cnt, #{s := S}) ->
     emqx_persistent_session_ds_state:n_subscriptions(S);
 info(subscriptions_max, #{props := Conf}) ->
@@ -356,15 +370,23 @@ print_session(ClientId) ->
 %% Client -> Broker: SUBSCRIBE / UNSUBSCRIBE
 %%--------------------------------------------------------------------
 
+%% Suppress warnings about clauses handling unimplemented results
+%% of `emqx_persistent_session_ds_shared_subs:on_subscribe/3`
+-dialyzer({nowarn_function, subscribe/3}).
 -spec subscribe(topic_filter(), emqx_types:subopts(), session()) ->
     {ok, session()} | {error, emqx_types:reason_code()}.
 subscribe(
-    #share{},
+    #share{} = TopicFilter,
-    _SubOpts,
+    SubOpts,
-    _Session
+    Session
 ) ->
-    %% TODO: Shared subscriptions are not supported yet:
+    case emqx_persistent_session_ds_shared_subs:on_subscribe(TopicFilter, SubOpts, Session) of
-    {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED};
+        {ok, S0, SharedSubS} ->
+            S = emqx_persistent_session_ds_state:commit(S0),
+            {ok, Session#{s => S, shared_sub_s => SharedSubS}};
+        Error = {error, _} ->
+            Error
+    end;
 subscribe(
     TopicFilter,
     SubOpts,
@@ -378,8 +400,27 @@ subscribe(
             Error
     end.
 
+%% Suppress warnings about clauses handling unimplemented results
+%% of `emqx_persistent_session_ds_shared_subs:on_unsubscribe/4`
+-dialyzer({nowarn_function, unsubscribe/2}).
 -spec unsubscribe(topic_filter(), session()) ->
     {ok, session(), emqx_types:subopts()} | {error, emqx_types:reason_code()}.
+unsubscribe(
+    #share{} = TopicFilter,
+    Session = #{id := SessionId, s := S0, shared_sub_s := SharedSubS0}
+) ->
+    case
+        emqx_persistent_session_ds_shared_subs:on_unsubscribe(
+            SessionId, TopicFilter, S0, SharedSubS0
+        )
+    of
+        {ok, S1, SharedSubS1, #{id := SubId, subopts := SubOpts}} ->
+            S2 = emqx_persistent_session_ds_stream_scheduler:on_unsubscribe(SubId, S1),
+            S = emqx_persistent_session_ds_state:commit(S2),
+            {ok, Session#{s => S, shared_sub_s => SharedSubS1}, SubOpts};
+        Error = {error, _} ->
+            Error
+    end;
 unsubscribe(
     TopicFilter,
     Session = #{id := SessionId, s := S0}
@@ -540,6 +581,8 @@ pubcomp(_ClientInfo, PacketId, Session0) ->
     end.
 
 %%--------------------------------------------------------------------
+%% Delivers
+%%--------------------------------------------------------------------
 
 -spec deliver(clientinfo(), [emqx_types:deliver()], session()) ->
     {ok, replies(), session()}.
@@ -551,6 +594,10 @@ deliver(ClientInfo, Delivers, Session0) ->
     ),
     {ok, [], pull_now(Session)}.
 
+%%--------------------------------------------------------------------
+%% Timeouts
+%%--------------------------------------------------------------------
+
 -spec handle_timeout(clientinfo(), _Timeout, session()) ->
     {ok, replies(), session()} | {ok, replies(), timeout(), session()}.
 handle_timeout(ClientInfo, ?TIMER_PULL, Session0) ->
@@ -573,14 +620,15 @@ handle_timeout(ClientInfo, ?TIMER_PULL, Session0) ->
 handle_timeout(ClientInfo, ?TIMER_RETRY_REPLAY, Session0) ->
     Session = replay_streams(Session0, ClientInfo),
     {ok, [], Session};
-handle_timeout(ClientInfo, ?TIMER_GET_STREAMS, Session0 = #{s := S0}) ->
+handle_timeout(ClientInfo, ?TIMER_GET_STREAMS, Session0 = #{s := S0, shared_sub_s := SharedSubS0}) ->
     S1 = emqx_persistent_session_ds_subs:gc(S0),
-    S = emqx_persistent_session_ds_stream_scheduler:renew_streams(S1),
+    S2 = emqx_persistent_session_ds_stream_scheduler:renew_streams(S1),
+    {S, SharedSubS} = emqx_persistent_session_ds_shared_subs:renew_streams(S2, SharedSubS0),
     Interval = get_config(ClientInfo, [renew_streams_interval]),
     Session = emqx_session:ensure_timer(
         ?TIMER_GET_STREAMS,
         Interval,
-        Session0#{s => S}
+        Session0#{s => S, shared_sub_s => SharedSubS}
     ),
     {ok, [], Session};
 handle_timeout(_ClientInfo, ?TIMER_BUMP_LAST_ALIVE_AT, Session0 = #{s := S0}) ->
@@ -601,6 +649,22 @@ handle_timeout(_ClientInfo, Timeout, Session) ->
     ?SLOG(warning, #{msg => "unknown_ds_timeout", timeout => Timeout}),
     {ok, [], Session}.
 
+%%--------------------------------------------------------------------
+%% Generic messages
+%%--------------------------------------------------------------------
+
+-spec handle_info(term(), session()) -> session().
+handle_info(?shared_sub_message(Msg), Session = #{s := S0, shared_sub_s := SharedSubS0}) ->
+    {S, SharedSubS} = emqx_persistent_session_ds_shared_subs:on_info(S0, SharedSubS0, Msg),
+    Session#{s => S, shared_sub_s => SharedSubS}.
+
+%%--------------------------------------------------------------------
+%% Shared subscription outgoing messages
+%%--------------------------------------------------------------------
+
+shared_sub_opts(SessionId) ->
+    #{session_id => SessionId}.
+
 bump_last_alive(S0) ->
     %% Note: we take a pessimistic approach here and assume that the client will be alive
     %% until the next bump timeout.  With this, we avoid garbage collecting this session
@@ -814,13 +878,17 @@ session_open(
                     S4 = emqx_persistent_session_ds_state:set_will_message(MaybeWillMsg, S3),
                     S5 = set_clientinfo(ClientInfo, S4),
                     S6 = emqx_persistent_session_ds_state:set_protocol({ProtoName, ProtoVer}, S5),
-                    S = emqx_persistent_session_ds_state:commit(S6),
+                    {ok, S7, SharedSubS} = emqx_persistent_session_ds_shared_subs:open(
+                        S6, shared_sub_opts(SessionId)
+                    ),
+                    S = emqx_persistent_session_ds_state:commit(S7),
                     Inflight = emqx_persistent_session_ds_inflight:new(
                         receive_maximum(NewConnInfo)
                     ),
                     #{
                         id => SessionId,
                         s => S,
+                        shared_sub_s => SharedSubS,
                         inflight => Inflight,
                         props => #{}
                     }
@@ -869,6 +937,7 @@ session_ensure_new(
         id => Id,
         props => Conf,
         s => S,
+        shared_sub_s => emqx_persistent_session_ds_shared_subs:new(shared_sub_opts(Id)),
         inflight => emqx_persistent_session_ds_inflight:new(receive_maximum(ConnInfo))
     }.
 
@@ -879,8 +948,8 @@ session_drop(SessionId, Reason) ->
     case emqx_persistent_session_ds_state:open(SessionId) of
         {ok, S0} ->
             ?tp(debug, drop_persistent_session, #{client_id => SessionId, reason => Reason}),
-            emqx_persistent_session_ds_subs:on_session_drop(SessionId, S0),
+            ok = emqx_persistent_session_ds_subs:on_session_drop(SessionId, S0),
-            emqx_persistent_session_ds_state:delete(SessionId);
+            ok = emqx_persistent_session_ds_state:delete(SessionId);
         undefined ->
             ok
     end.
@@ -917,24 +986,33 @@ do_ensure_all_iterators_closed(_DSSessionID) ->
 %% Normal replay:
 %%--------------------------------------------------------------------
 
-fetch_new_messages(Session = #{s := S}, ClientInfo) ->
+fetch_new_messages(Session0 = #{s := S0}, ClientInfo) ->
-    Streams = emqx_persistent_session_ds_stream_scheduler:find_new_streams(S),
+    LFS = maps:get(last_fetched_stream, Session0, beginning),
-    fetch_new_messages(Streams, Session, ClientInfo).
+    ItStream = emqx_persistent_session_ds_stream_scheduler:iter_next_streams(LFS, S0),
-
-fetch_new_messages([], Session, _ClientInfo) ->
-    Session;
-fetch_new_messages([I | Streams], Session0 = #{inflight := Inflight}, ClientInfo) ->
     BatchSize = get_config(ClientInfo, [batch_size]),
+    Session1 = fetch_new_messages(ItStream, BatchSize, Session0, ClientInfo),
+    #{s := S1, shared_sub_s := SharedSubS0} = Session1,
+    {S2, SharedSubS1} = emqx_persistent_session_ds_shared_subs:on_streams_replayed(S1, SharedSubS0),
+    Session1#{s => S2, shared_sub_s => SharedSubS1}.
+
+fetch_new_messages(ItStream0, BatchSize, Session0, ClientInfo) ->
+    #{inflight := Inflight} = Session0,
     case emqx_persistent_session_ds_inflight:n_buffered(all, Inflight) >= BatchSize of
         true ->
             %% Buffer is full:
             Session0;
         false ->
-            Session = new_batch(I, BatchSize, Session0, ClientInfo),
+            case emqx_persistent_session_ds_stream_scheduler:next_stream(ItStream0) of
-            fetch_new_messages(Streams, Session, ClientInfo)
+                {StreamKey, Srs, ItStream} ->
+                    Session1 = new_batch(StreamKey, Srs, BatchSize, Session0, ClientInfo),
+                    Session = Session1#{last_fetched_stream => StreamKey},
+                    fetch_new_messages(ItStream, BatchSize, Session, ClientInfo);
+                none ->
+                    Session0
+            end
     end.
 
-new_batch({StreamKey, Srs0}, BatchSize, Session0 = #{s := S0}, ClientInfo) ->
+new_batch(StreamKey, Srs0, BatchSize, Session0 = #{s := S0}, ClientInfo) ->
     SN1 = emqx_persistent_session_ds_state:get_seqno(?next(?QOS_1), S0),
     SN2 = emqx_persistent_session_ds_state:get_seqno(?next(?QOS_2), S0),
     Srs1 = Srs0#srs{

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_inflight.erl

@@ -67,7 +67,7 @@
 -type t() :: #inflight{}.
 
 %%================================================================================
-%% API funcions
+%% API functions
 %%================================================================================
 
 -spec new(non_neg_integer()) -> t().

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl

@@ -46,9 +46,10 @@
 -export([has_route/2]).
 -endif.
 
+-type route() :: #ps_route{}.
 -type dest() :: emqx_persistent_session_ds:id().
 
--export_type([dest/0]).
+-export_type([dest/0, route/0]).
 
 %%--------------------------------------------------------------------
 %% Table Initialization
@@ -123,19 +124,19 @@ has_any_route(Topic) ->
 
 %% @doc Take a real topic (not filter) as input, return the matching topics and topic
 %% filters associated with route destination.
--spec match_routes(emqx_types:topic()) -> [emqx_types:route()].
+-spec match_routes(emqx_types:topic()) -> [route()].
 match_routes(Topic) when is_binary(Topic) ->
     lookup_route_tab(Topic) ++
         [match_to_route(M) || M <- match_filters(Topic)].
 
 %% @doc Take a topic or filter as input, and return the existing routes with exactly
 %% this topic or filter.
--spec lookup_routes(emqx_types:topic()) -> [emqx_types:route()].
+-spec lookup_routes(emqx_types:topic()) -> [route()].
 lookup_routes(Topic) ->
     case emqx_topic:wildcard(Topic) of
         true ->
             Pat = #ps_routeidx{entry = emqx_topic_index:make_key(Topic, '$1')},
-            [Dest || [Dest] <- ets:match(?PS_FILTERS_TAB, Pat)];
+            [#ps_route{topic = Topic, dest = Dest} || [Dest] <- ets:match(?PS_FILTERS_TAB, Pat)];
         false ->
             lookup_route_tab(Topic)
     end.
@@ -194,11 +195,11 @@ cleanup_routes(DSSessionId) ->
         ?PS_ROUTER_TAB
     ).
 
--spec foldl_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc.
+-spec foldl_routes(fun((route(), Acc) -> Acc), Acc) -> Acc.
 foldl_routes(FoldFun, AccIn) ->
     fold_routes(foldl, FoldFun, AccIn).
 
--spec foldr_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc.
+-spec foldr_routes(fun((route(), Acc) -> Acc), Acc) -> Acc.
 foldr_routes(FoldFun, AccIn) ->
     fold_routes(foldr, FoldFun, AccIn).
 

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl

@@ -0,0 +1,338 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_persistent_session_ds_shared_subs).
+
+-include("emqx_mqtt.hrl").
+-include("logger.hrl").
+-include("session_internals.hrl").
+-include_lib("snabbkaffe/include/trace.hrl").
+
+-export([
+    new/1,
+    open/2,
+
+    on_subscribe/3,
+    on_unsubscribe/4,
+
+    on_streams_replayed/2,
+    on_info/3,
+
+    renew_streams/2,
+    to_map/2
+]).
+
+-type t() :: #{
+    agent := emqx_persistent_session_ds_shared_subs_agent:t()
+}.
+-type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter().
+-type opts() :: #{
+    session_id := emqx_persistent_session_ds:id()
+}.
+
+-define(rank_x, rank_shared).
+-define(rank_y, 0).
+
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
+
+-spec new(opts()) -> t().
+new(Opts) ->
+    #{
+        agent => emqx_persistent_session_ds_shared_subs_agent:new(
+            agent_opts(Opts)
+        )
+    }.
+
+-spec open(emqx_persistent_session_ds_state:t(), opts()) ->
+    {ok, emqx_persistent_session_ds_state:t(), t()}.
+open(S, Opts) ->
+    SharedSubscriptions = fold_shared_subs(
+        fun(#share{} = TopicFilter, Sub, Acc) ->
+            [{TopicFilter, to_agent_subscription(S, Sub)} | Acc]
+        end,
+        [],
+        S
+    ),
+    Agent = emqx_persistent_session_ds_shared_subs_agent:open(
+        SharedSubscriptions, agent_opts(Opts)
+    ),
+    SharedSubS = #{agent => Agent},
+    {ok, S, SharedSubS}.
+
+-spec on_subscribe(
+    share_topic_filter(),
+    emqx_types:subopts(),
+    emqx_persistent_session_ds:session()
+) -> {ok, emqx_persistent_session_ds_state:t(), t()} | {error, emqx_types:reason_code()}.
+on_subscribe(TopicFilter, SubOpts, #{s := S} = Session) ->
+    Subscription = emqx_persistent_session_ds_state:get_subscription(TopicFilter, S),
+    on_subscribe(Subscription, TopicFilter, SubOpts, Session).
+
+-spec on_unsubscribe(
+    emqx_persistent_session_ds:id(),
+    emqx_persistent_session_ds:topic_filter(),
+    emqx_persistent_session_ds_state:t(),
+    t()
+) ->
+    {ok, emqx_persistent_session_ds_state:t(), t(), emqx_persistent_session_ds:subscription()}
+    | {error, emqx_types:reason_code()}.
+on_unsubscribe(SessionId, TopicFilter, S0, #{agent := Agent0} = SharedSubS0) ->
+    case lookup(TopicFilter, S0) of
+        undefined ->
+            {error, ?RC_NO_SUBSCRIPTION_EXISTED};
+        Subscription ->
+            ?tp(persistent_session_ds_subscription_delete, #{
+                session_id => SessionId, topic_filter => TopicFilter
+            }),
+            Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_unsubscribe(
+                Agent0, TopicFilter
+            ),
+            SharedSubS = SharedSubS0#{agent => Agent1},
+            S = emqx_persistent_session_ds_state:del_subscription(TopicFilter, S0),
+            {ok, S, SharedSubS, Subscription}
+    end.
+
+-spec renew_streams(emqx_persistent_session_ds_state:t(), t()) ->
+    {emqx_persistent_session_ds_state:t(), t()}.
+renew_streams(S0, #{agent := Agent0} = SharedSubS0) ->
+    {StreamLeaseEvents, Agent1} = emqx_persistent_session_ds_shared_subs_agent:renew_streams(
+        Agent0
+    ),
+    ?tp(info, shared_subs_new_stream_lease_events, #{stream_lease_events => StreamLeaseEvents}),
+    S1 = lists:foldl(
+        fun
+            (#{type := lease} = Event, S) -> accept_stream(Event, S);
+            (#{type := revoke} = Event, S) -> revoke_stream(Event, S)
+        end,
+        S0,
+        StreamLeaseEvents
+    ),
+    SharedSubS1 = SharedSubS0#{agent => Agent1},
+    {S1, SharedSubS1}.
+
+-spec on_streams_replayed(
+    emqx_persistent_session_ds_state:t(),
+    t()
+) -> {emqx_persistent_session_ds_state:t(), t()}.
+on_streams_replayed(S, #{agent := Agent0} = SharedSubS0) ->
+    %% TODO
+    %% Is it sufficient for a report?
+    Progress = fold_shared_stream_states(
+        fun(TopicFilter, Stream, SRS, Acc) ->
+            #srs{it_begin = BeginIt} = SRS,
+            StreamProgress = #{
+                topic_filter => TopicFilter,
+                stream => Stream,
+                iterator => BeginIt
+            },
+            [StreamProgress | Acc]
+        end,
+        [],
+        S
+    ),
+    Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress(
+        Agent0, Progress
+    ),
+    SharedSubS1 = SharedSubS0#{agent => Agent1},
+    {S, SharedSubS1}.
+
+-spec on_info(emqx_persistent_session_ds_state:t(), t(), term()) ->
+    {emqx_persistent_session_ds_state:t(), t()}.
+on_info(S, #{agent := Agent0} = SharedSubS0, Info) ->
+    Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_info(Agent0, Info),
+    SharedSubS1 = SharedSubS0#{agent => Agent1},
+    {S, SharedSubS1}.
+
+-spec to_map(emqx_persistent_session_ds_state:t(), t()) -> map().
+to_map(_S, _SharedSubS) ->
+    %% TODO
+    #{}.
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+fold_shared_subs(Fun, Acc, S) ->
+    emqx_persistent_session_ds_state:fold_subscriptions(
+        fun
+            (#share{} = TopicFilter, Sub, Acc0) -> Fun(TopicFilter, Sub, Acc0);
+            (_, _Sub, Acc0) -> Acc0
+        end,
+        Acc,
+        S
+    ).
+
+fold_shared_stream_states(Fun, Acc, S) ->
+    %% TODO
+    %% Optimize or cache
+    TopicFilters = fold_shared_subs(
+        fun
+            (#share{} = TopicFilter, #{id := Id} = _Sub, Acc0) ->
+                Acc0#{Id => TopicFilter};
+            (_, _, Acc0) ->
+                Acc0
+        end,
+        #{},
+        S
+    ),
+    emqx_persistent_session_ds_state:fold_streams(
+        fun({SubId, Stream}, SRS, Acc0) ->
+            case TopicFilters of
+                #{SubId := TopicFilter} ->
+                    Fun(TopicFilter, Stream, SRS, Acc0);
+                _ ->
+                    Acc0
+            end
+        end,
+        Acc,
+        S
+    ).
+
+on_subscribe(undefined, TopicFilter, SubOpts, #{props := Props, s := S} = Session) ->
+    #{max_subscriptions := MaxSubscriptions} = Props,
+    case emqx_persistent_session_ds_state:n_subscriptions(S) < MaxSubscriptions of
+        true ->
+            create_new_subscription(TopicFilter, SubOpts, Session);
+        false ->
+            {error, ?RC_QUOTA_EXCEEDED}
+    end;
+on_subscribe(Subscription, TopicFilter, SubOpts, Session) ->
+    update_subscription(Subscription, TopicFilter, SubOpts, Session).
+
+-dialyzer({nowarn_function, create_new_subscription/3}).
+create_new_subscription(TopicFilter, SubOpts, #{
+    id := SessionId, s := S0, shared_sub_s := #{agent := Agent0} = SharedSubS0, props := Props
+}) ->
+    case
+        emqx_persistent_session_ds_shared_subs_agent:on_subscribe(
+            Agent0, TopicFilter, SubOpts
+        )
+    of
+        {ok, Agent1} ->
+            #{upgrade_qos := UpgradeQoS} = Props,
+            {SubId, S1} = emqx_persistent_session_ds_state:new_id(S0),
+            {SStateId, S2} = emqx_persistent_session_ds_state:new_id(S1),
+            SState = #{
+                parent_subscription => SubId, upgrade_qos => UpgradeQoS, subopts => SubOpts
+            },
+            S3 = emqx_persistent_session_ds_state:put_subscription_state(
+                SStateId, SState, S2
+            ),
+            Subscription = #{
+                id => SubId,
+                current_state => SStateId,
+                start_time => now_ms()
+            },
+            S = emqx_persistent_session_ds_state:put_subscription(
+                TopicFilter, Subscription, S3
+            ),
+            SharedSubS = SharedSubS0#{agent => Agent1},
+            ?tp(persistent_session_ds_shared_subscription_added, #{
+                topic_filter => TopicFilter, session => SessionId
+            }),
+            {ok, S, SharedSubS};
+        {error, _} = Error ->
+            Error
+    end.
+
+update_subscription(#{current_state := SStateId0, id := SubId} = Sub0, TopicFilter, SubOpts, #{
+    s := S0, shared_sub_s := SharedSubS, props := Props
+}) ->
+    #{upgrade_qos := UpgradeQoS} = Props,
+    SState = #{parent_subscription => SubId, upgrade_qos => UpgradeQoS, subopts => SubOpts},
+    case emqx_persistent_session_ds_state:get_subscription_state(SStateId0, S0) of
+        SState ->
+            %% Client resubscribed with the same parameters:
+            {ok, S0, SharedSubS};
+        _ ->
+            %% Subsription parameters changed:
+            {SStateId, S1} = emqx_persistent_session_ds_state:new_id(S0),
+            S2 = emqx_persistent_session_ds_state:put_subscription_state(
+                SStateId, SState, S1
+            ),
+            Sub = Sub0#{current_state => SStateId},
+            S = emqx_persistent_session_ds_state:put_subscription(TopicFilter, Sub, S2),
+            {ok, S, SharedSubS}
+    end.
+
+lookup(TopicFilter, S) ->
+    case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S) of
+        Sub = #{current_state := SStateId} ->
+            case emqx_persistent_session_ds_state:get_subscription_state(SStateId, S) of
+                #{subopts := SubOpts} ->
+                    Sub#{subopts => SubOpts};
+                undefined ->
+                    undefined
+            end;
+        undefined ->
+            undefined
+    end.
+
+accept_stream(
+    #{topic_filter := TopicFilter, stream := Stream, iterator := Iterator}, S0
+) ->
+    case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S0) of
+        undefined ->
+            %% This should not happen.
+            %% Agent should have received unsubscribe callback
+            %% and should not have passed this stream as a new one
+            error(new_stream_without_sub);
+        #{id := SubId, current_state := SStateId} ->
+            Key = {SubId, Stream},
+            case emqx_persistent_session_ds_state:get_stream(Key, S0) of
+                undefined ->
+                    NewSRS =
+                        #srs{
+                            rank_x = ?rank_x,
+                            rank_y = ?rank_y,
+                            it_begin = Iterator,
+                            it_end = Iterator,
+                            sub_state_id = SStateId
+                        },
+                    S1 = emqx_persistent_session_ds_state:put_stream(Key, NewSRS, S0),
+                    S1;
+                _SRS ->
+                    S0
+            end
+    end.
+
+revoke_stream(
+    #{topic_filter := TopicFilter, stream := Stream}, S0
+) ->
+    case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S0) of
+        undefined ->
+            %% This should not happen.
+            %% Agent should have received unsubscribe callback
+            %% and should not have revoked this stream
+            S0;
+        #{id := SubId} ->
+            Key = {SubId, Stream},
+            case emqx_persistent_session_ds_state:get_stream(Key, S0) of
+                undefined ->
+                    S0;
+                SRS0 ->
+                    SRS1 = SRS0#srs{unsubscribed = true},
+                    S1 = emqx_persistent_session_ds_state:put_stream(Key, SRS1, S0),
+                    S1
+            end
+    end.
+
+-spec to_agent_subscription(
+    emqx_persistent_session_ds_state:t(), emqx_persistent_session_ds:subscription()
+) ->
+    emqx_persistent_session_ds_shared_subs_agent:subscription().
+to_agent_subscription(_S, Subscription) ->
+    %% TODO
+    %% do we need anything from sub state?
+    maps:with([start_time], Subscription).
+
+-spec agent_opts(opts()) -> emqx_persistent_session_ds_shared_subs_agent:opts().
+agent_opts(#{session_id := SessionId}) ->
+    #{session_id => SessionId}.
+
+-dialyzer({nowarn_function, now_ms/0}).
+now_ms() ->
+    erlang:system_time(millisecond).

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl

@@ -0,0 +1,127 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_persistent_session_ds_shared_subs_agent).
+
+-include("shared_subs_agent.hrl").
+-include("emqx_session.hrl").
+-include("session_internals.hrl").
+
+-type session_id() :: emqx_persistent_session_ds:id().
+
+-type subscription() :: #{
+    start_time := emqx_ds:time()
+}.
+
+-type t() :: term().
+-type topic_filter() :: emqx_persistent_session_ds:share_topic_filter().
+
+-type opts() :: #{
+    session_id := session_id()
+}.
+
+%% TODO
+%% This records go through network, we better shrink them
+%% * use integer keys
+%% * somehow avoid passing stream and topic_filter — they both are part of the iterator
+-type stream_lease() :: #{
+    type => lease,
+    %% Used as "external" subscription_id
+    topic_filter := topic_filter(),
+    stream := emqx_ds:stream(),
+    iterator := emqx_ds:iterator()
+}.
+
+-type stream_revoke() :: #{
+    type => revoke,
+    topic_filter := topic_filter(),
+    stream := emqx_ds:stream()
+}.
+
+-type stream_lease_event() :: stream_lease() | stream_revoke().
+
+-type stream_progress() :: #{
+    topic_filter := topic_filter(),
+    stream := emqx_ds:stream(),
+    iterator := emqx_ds:iterator()
+}.
+
+-export_type([
+    t/0,
+    subscription/0,
+    session_id/0,
+    stream_lease/0,
+    opts/0
+]).
+
+-export([
+    new/1,
+    open/2,
+
+    on_subscribe/3,
+    on_unsubscribe/2,
+    on_stream_progress/2,
+    on_info/2,
+
+    renew_streams/1
+]).
+
+-export([
+    send/2,
+    send_after/3
+]).
+
+%%--------------------------------------------------------------------
+%% Behaviour
+%%--------------------------------------------------------------------
+
+-callback new(opts()) -> t().
+-callback open([{topic_filter(), subscription()}], opts()) -> t().
+-callback on_subscribe(t(), topic_filter(), emqx_types:subopts()) ->
+    {ok, t()} | {error, term()}.
+-callback on_unsubscribe(t(), topic_filter()) -> t().
+-callback renew_streams(t()) -> {[stream_lease_event()], t()}.
+-callback on_stream_progress(t(), [stream_progress()]) -> t().
+-callback on_info(t(), term()) -> t().
+
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
+
+-spec new(opts()) -> t().
+new(Opts) ->
+    ?shared_subs_agent:new(Opts).
+
+-spec open([{topic_filter(), subscription()}], opts()) -> t().
+open(Topics, Opts) ->
+    ?shared_subs_agent:open(Topics, Opts).
+
+-spec on_subscribe(t(), topic_filter(), emqx_types:subopts()) ->
+    {ok, t()} | {error, emqx_types:reason_code()}.
+on_subscribe(Agent, TopicFilter, SubOpts) ->
+    ?shared_subs_agent:on_subscribe(Agent, TopicFilter, SubOpts).
+
+-spec on_unsubscribe(t(), topic_filter()) -> t().
+on_unsubscribe(Agent, TopicFilter) ->
+    ?shared_subs_agent:on_unsubscribe(Agent, TopicFilter).
+
+-spec renew_streams(t()) -> {[stream_lease_event()], t()}.
+renew_streams(Agent) ->
+    ?shared_subs_agent:renew_streams(Agent).
+
+-spec on_stream_progress(t(), [stream_progress()]) -> t().
+on_stream_progress(Agent, StreamProgress) ->
+    ?shared_subs_agent:on_stream_progress(Agent, StreamProgress).
+
+-spec on_info(t(), term()) -> t().
+on_info(Agent, Info) ->
+    ?shared_subs_agent:on_info(Agent, Info).
+
+-spec send(pid(), term()) -> term().
+send(Dest, Msg) ->
+    erlang:send(Dest, ?session_message(?shared_sub_message(Msg))).
+
+-spec send_after(non_neg_integer(), pid(), term()) -> reference().
+send_after(Time, Dest, Msg) ->
+    erlang:send_after(Time, Dest, ?session_message(?shared_sub_message(Msg))).

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl

@@ -0,0 +1,46 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_persistent_session_ds_shared_subs_null_agent).
+
+-include("emqx_mqtt.hrl").
+
+-export([
+    new/1,
+    open/2,
+
+    on_subscribe/3,
+    on_unsubscribe/2,
+    on_stream_progress/2,
+    on_info/2,
+
+    renew_streams/1
+]).
+
+-behaviour(emqx_persistent_session_ds_shared_subs_agent).
+
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
+
+new(_Opts) ->
+    undefined.
+
+open(_Topics, _Opts) ->
+    undefined.
+
+on_subscribe(_Agent, _TopicFilter, _SubOpts) ->
+    {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}.
+
+on_unsubscribe(Agent, _TopicFilter) ->
+    Agent.
+
+renew_streams(Agent) ->
+    {[], Agent}.
+
+on_stream_progress(Agent, _StreamProgress) ->
+    Agent.
+
+on_info(Agent, _Info) ->
+    Agent.

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl

@@ -39,7 +39,7 @@
 -export([get_peername/1, set_peername/2]).
 -export([get_protocol/1, set_protocol/2]).
 -export([new_id/1]).
--export([get_stream/2, put_stream/3, del_stream/2, fold_streams/3, n_streams/1]).
+-export([get_stream/2, put_stream/3, del_stream/2, fold_streams/3, iter_streams/2, n_streams/1]).
 -export([get_seqno/2, put_seqno/3]).
 -export([get_rank/2, put_rank/3, del_rank/2, fold_ranks/3]).
 -export([
@@ -66,11 +66,14 @@
     n_awaiting_rel/1
 ]).
 
+-export([iter_next/1]).
+
 -export([make_session_iterator/0, session_iterator_next/2]).
 
 -export_type([
     t/0,
     metadata/0,
+    iter/2,
     seqno_type/0,
     stream_key/0,
     rank_key/0,
@@ -89,6 +92,8 @@
 
 -type message() :: emqx_types:message().
 
+-opaque iter(K, V) :: gb_trees:iter(K, V).
+
 -opaque session_iterator() :: emqx_persistent_session_ds:id() | '$end_of_table'.
 
 %% Generic key-value wrapper that is used for exporting arbitrary
@@ -113,7 +118,7 @@
 -type pmap(K, V) ::
     #pmap{
         table :: atom(),
-        cache :: #{K => V},
+        cache :: #{K => V} | gb_trees:tree(K, V),
         dirty :: #{K => dirty | del}
     }.
 
@@ -192,7 +197,7 @@
 -endif.
 
 %%================================================================================
-%% API funcions
+%% API functions
 %%================================================================================
 
 -spec create_tables() -> ok.
@@ -476,6 +481,14 @@ del_stream(Key, Rec) ->
 fold_streams(Fun, Acc, Rec) ->
     gen_fold(?streams, Fun, Acc, Rec).
 
+-spec iter_streams(_StartAfter :: stream_key() | beginning, t()) ->
+    iter(stream_key(), emqx_persistent_session_ds:stream_state()).
+iter_streams(After, Rec) ->
+    %% NOTE
+    %% No special handling for `beginning', as it always compares less
+    %% than any `stream_key()'.
+    gen_iter_after(?streams, After, Rec).
+
 -spec n_streams(t()) -> non_neg_integer().
 n_streams(Rec) ->
     gen_size(?streams, Rec).
@@ -534,6 +547,12 @@ n_awaiting_rel(Rec) ->
 
 %%
 
+-spec iter_next(iter(K, V)) -> {K, V, iter(K, V)} | none.
+iter_next(It0) ->
+    gen_iter_next(It0).
+
+%%
+
 -spec make_session_iterator() -> session_iterator().
 make_session_iterator() ->
     mnesia:dirty_first(?session_tab).
@@ -601,6 +620,14 @@ gen_size(Field, Rec) ->
     check_sequence(Rec),
     pmap_size(maps:get(Field, Rec)).
 
+gen_iter_after(Field, After, Rec) ->
+    check_sequence(Rec),
+    pmap_iter_after(After, maps:get(Field, Rec)).
+
+gen_iter_next(It) ->
+    %% NOTE: Currently, gbt iterators is the only type of iterators.
+    gbt_iter_next(It).
+
 -spec update_pmaps(fun((pmap(_K, _V) | undefined, atom()) -> term()), map()) -> map().
 update_pmaps(Fun, Map) ->
     lists:foldl(
@@ -619,7 +646,7 @@ update_pmaps(Fun, Map) ->
 %% This functtion should be ran in a transaction.
 -spec pmap_open(atom(), emqx_persistent_session_ds:id()) -> pmap(_K, _V).
 pmap_open(Table, SessionId) ->
-    Clean = maps:from_list(kv_pmap_restore(Table, SessionId)),
+    Clean = cache_from_list(Table, kv_pmap_restore(Table, SessionId)),
     #pmap{
         table = Table,
         cache = Clean,
@@ -627,29 +654,29 @@ pmap_open(Table, SessionId) ->
     }.
 
 -spec pmap_get(K, pmap(K, V)) -> V | undefined.
-pmap_get(K, #pmap{cache = Cache}) ->
+pmap_get(K, #pmap{table = Table, cache = Cache}) ->
-    maps:get(K, Cache, undefined).
+    cache_get(Table, K, Cache).
 
 -spec pmap_put(K, V, pmap(K, V)) -> pmap(K, V).
-pmap_put(K, V, Pmap = #pmap{dirty = Dirty, cache = Cache}) ->
+pmap_put(K, V, Pmap = #pmap{table = Table, dirty = Dirty, cache = Cache}) ->
     Pmap#pmap{
-        cache = maps:put(K, V, Cache),
+        cache = cache_put(Table, K, V, Cache),
         dirty = Dirty#{K => dirty}
     }.
 
 -spec pmap_del(K, pmap(K, V)) -> pmap(K, V).
 pmap_del(
     Key,
-    Pmap = #pmap{dirty = Dirty, cache = Cache}
+    Pmap = #pmap{table = Table, dirty = Dirty, cache = Cache}
 ) ->
     Pmap#pmap{
-        cache = maps:remove(Key, Cache),
+        cache = cache_remove(Table, Key, Cache),
         dirty = Dirty#{Key => del}
     }.
 
 -spec pmap_fold(fun((K, V, A) -> A), A, pmap(K, V)) -> A.
-pmap_fold(Fun, Acc, #pmap{cache = Cache}) ->
+pmap_fold(Fun, Acc, #pmap{table = Table, cache = Cache}) ->
-    maps:fold(Fun, Acc, Cache).
+    cache_fold(Table, Fun, Acc, Cache).
 
 -spec pmap_commit(emqx_persistent_session_ds:id(), pmap(K, V)) -> pmap(K, V).
 pmap_commit(
@@ -660,7 +687,7 @@ pmap_commit(
             (K, del) ->
                 kv_pmap_delete(Tab, SessionId, K);
             (K, dirty) ->
-                V = maps:get(K, Cache),
+                V = cache_get(Tab, K, Cache),
                 kv_pmap_persist(Tab, SessionId, K, V)
         end,
         Dirty
@@ -670,13 +697,110 @@ pmap_commit(
     }.
 
 -spec pmap_format(pmap(_K, _V)) -> map().
-pmap_format(#pmap{cache = Cache}) ->
+pmap_format(#pmap{table = Table, cache = Cache}) ->
-    Cache.
+    cache_format(Table, Cache).
 
 -spec pmap_size(pmap(_K, _V)) -> non_neg_integer().
-pmap_size(#pmap{cache = Cache}) ->
+pmap_size(#pmap{table = Table, cache = Cache}) ->
+    cache_size(Table, Cache).
+
+pmap_iter_after(After, #pmap{table = Table, cache = Cache}) ->
+    %% NOTE: Only valid for gbt-backed PMAPs.
+    gbt = cache_data_type(Table),
+    gbt_iter_after(After, Cache).
+
+%%
+
+cache_data_type(?stream_tab) -> gbt;
+cache_data_type(_Table) -> map.
+
+cache_from_list(?stream_tab, L) ->
+    gbt_from_list(L);
+cache_from_list(_Table, L) ->
+    maps:from_list(L).
+
+cache_get(?stream_tab, K, Cache) ->
+    gbt_get(K, Cache, undefined);
+cache_get(_Table, K, Cache) ->
+    maps:get(K, Cache, undefined).
+
+cache_put(?stream_tab, K, V, Cache) ->
+    gbt_put(K, V, Cache);
+cache_put(_Table, K, V, Cache) ->
+    maps:put(K, V, Cache).
+
+cache_remove(?stream_tab, K, Cache) ->
+    gbt_remove(K, Cache);
+cache_remove(_Table, K, Cache) ->
+    maps:remove(K, Cache).
+
+cache_fold(?stream_tab, Fun, Acc, Cache) ->
+    gbt_fold(Fun, Acc, Cache);
+cache_fold(_Table, Fun, Acc, Cache) ->
+    maps:fold(Fun, Acc, Cache).
+
+cache_format(?stream_tab, Cache) ->
+    gbt_format(Cache);
+cache_format(_Table, Cache) ->
+    Cache.
+
+cache_size(?stream_tab, Cache) ->
+    gbt_size(Cache);
+cache_size(_Table, Cache) ->
     maps:size(Cache).
 
+%% PMAP Cache implementation backed by `gb_trees'.
+%% Supports iteration starting from specific key.
+
+gbt_from_list(L) ->
+    lists:foldl(
+        fun({K, V}, Acc) -> gb_trees:insert(K, V, Acc) end,
+        gb_trees:empty(),
+        L
+    ).
+
+gbt_get(K, Cache, undefined) ->
+    case gb_trees:lookup(K, Cache) of
+        none -> undefined;
+        {_, V} -> V
+    end.
+
+gbt_put(K, V, Cache) ->
+    gb_trees:enter(K, V, Cache).
+
+gbt_remove(K, Cache) ->
+    gb_trees:delete_any(K, Cache).
+
+gbt_format(Cache) ->
+    gb_trees:to_list(Cache).
+
+gbt_fold(Fun, Acc, Cache) ->
+    It = gb_trees:iterator(Cache),
+    gbt_fold_iter(Fun, Acc, It).
+
+gbt_fold_iter(Fun, Acc, It0) ->
+    case gb_trees:next(It0) of
+        {K, V, It} ->
+            gbt_fold_iter(Fun, Fun(K, V, Acc), It);
+        _ ->
+            Acc
+    end.
+
+gbt_size(Cache) ->
+    gb_trees:size(Cache).
+
+gbt_iter_after(After, Cache) ->
+    It0 = gb_trees:iterator_from(After, Cache),
+    case gb_trees:next(It0) of
+        {After, _, It} ->
+            It;
+        _ ->
+            It0
+    end.
+
+gbt_iter_next(It) ->
+    gb_trees:next(It).
+
 %% Functions dealing with set tables:
 
 kv_persist(Tab, SessionId, Val0) ->

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_stream_scheduler.erl

@@ -16,7 +16,8 @@
 -module(emqx_persistent_session_ds_stream_scheduler).
 
 %% API:
--export([find_new_streams/1, find_replay_streams/1, is_fully_acked/2]).
+-export([iter_next_streams/2, next_stream/1]).
+-export([find_replay_streams/1, is_fully_acked/2]).
 -export([renew_streams/1, on_unsubscribe/2]).
 
 %% behavior callbacks:
@@ -35,6 +36,29 @@
 %% Type declarations
 %%================================================================================
 
+-type stream_key() :: emqx_persistent_session_ds_state:stream_key().
+-type stream_state() :: emqx_persistent_session_ds:stream_state().
+
+%% Restartable iterator with a filter and an iteration limit.
+-record(iter, {
+    limit :: non_neg_integer(),
+    filter,
+    it,
+    it_cont
+}).
+
+-type iter(K, V, IterInner) :: #iter{
+    filter :: fun((K, V) -> boolean()),
+    it :: IterInner,
+    it_cont :: IterInner
+}.
+
+-type iter_stream() :: iter(
+    stream_key(),
+    stream_state(),
+    emqx_persistent_session_ds_state:iter(stream_key(), stream_state())
+).
+
 %%================================================================================
 %% API functions
 %%================================================================================
@@ -70,9 +94,9 @@ find_replay_streams(S) ->
 %%
 %% This function is non-detereministic: it randomizes the order of
 %% streams to ensure fair replay of different topics.
--spec find_new_streams(emqx_persistent_session_ds_state:t()) ->
+-spec iter_next_streams(_LastVisited :: stream_key(), emqx_persistent_session_ds_state:t()) ->
-    [{emqx_persistent_session_ds_state:stream_key(), emqx_persistent_session_ds:stream_state()}].
+    iter_stream().
-find_new_streams(S) ->
+iter_next_streams(LastVisited, S) ->
     %% FIXME: this function is currently very sensitive to the
     %% consistency of the packet IDs on both broker and client side.
     %%
@@ -87,23 +111,44 @@ find_new_streams(S) ->
     %% after timeout?)
     Comm1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S),
     Comm2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S),
-    shuffle(
+    Filter = fun(_Key, Stream) -> is_fetchable(Comm1, Comm2, Stream) end,
-        emqx_persistent_session_ds_state:fold_streams(
+    #iter{
-            fun
+        %% Limit the iteration to one round over all streams:
-                (_Key, #srs{it_end = end_of_stream}, Acc) ->
+        limit = emqx_persistent_session_ds_state:n_streams(S),
-                    Acc;
+        %% Filter out the streams not eligible for fetching:
-                (Key, Stream, Acc) ->
+        filter = Filter,
-                    case is_fully_acked(Comm1, Comm2, Stream) andalso not Stream#srs.unsubscribed of
+        %% Start the iteration right after the last visited stream:
-                        true ->
+        it = emqx_persistent_session_ds_state:iter_streams(LastVisited, S),
-                            [{Key, Stream} | Acc];
+        %% Restart the iteration from the beginning:
-                        false ->
+        it_cont = emqx_persistent_session_ds_state:iter_streams(beginning, S)
-                            Acc
+    }.
-                    end
+
-            end,
+-spec next_stream(iter_stream()) -> {stream_key(), stream_state(), iter_stream()} | none.
-            [],
+next_stream(#iter{limit = 0}) ->
-            S
+    none;
-        )
+next_stream(ItStream0 = #iter{limit = N, filter = Filter, it = It0, it_cont = ItCont}) ->
-    ).
+    case emqx_persistent_session_ds_state:iter_next(It0) of
+        {Key, Stream, It} ->
+            ItStream = ItStream0#iter{it = It, limit = N - 1},
+            case Filter(Key, Stream) of
+                true ->
+                    {Key, Stream, ItStream};
+                false ->
+                    next_stream(ItStream)
+            end;
+        none when It0 =/= ItCont ->
+            %% Restart the iteration from the beginning:
+            ItStream = ItStream0#iter{it = ItCont},
+            next_stream(ItStream);
+        none ->
+            %% No point in restarting the iteration, `ItCont` is empty:
+            none
+    end.
+
+is_fetchable(_Comm1, _Comm2, #srs{it_end = end_of_stream}) ->
+    false;
+is_fetchable(Comm1, Comm2, #srs{unsubscribed = Unsubscribed} = Stream) ->
+    is_fully_acked(Comm1, Comm2, Stream) andalso not Unsubscribed.
 
 %% @doc This function makes the session aware of the new streams.
 %%
@@ -127,7 +172,12 @@ renew_streams(S0) ->
     S1 = remove_unsubscribed_streams(S0),
     S2 = remove_fully_replayed_streams(S1),
     S3 = update_stream_subscription_state_ids(S2),
-    emqx_persistent_session_ds_subs:fold(
+    %% For shared subscriptions, the streams are populated by
+    %% `emqx_persistent_session_ds_shared_subs`.
+    %% TODO
+    %% Move discovery of proper streams
+    %% out of the scheduler for complete symmetry?
+    fold_proper_subscriptions(
         fun
             (Key, #{start_time := StartTime, id := SubId, current_state := SStateId}, Acc) ->
                 TopicFilter = emqx_topic:words(Key),
@@ -206,9 +256,6 @@ ensure_iterator(TopicFilter, StartTime, SubId, SStateId, {{RankX, RankY}, Stream
     Key = {SubId, Stream},
     case emqx_persistent_session_ds_state:get_stream(Key, S) of
         undefined ->
-            ?SLOG(debug, #{
-                msg => new_stream, key => Key, stream => Stream
-            }),
             case emqx_ds:make_iterator(?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime) of
                 {ok, Iterator} ->
                     NewStreamState = #srs{
@@ -408,15 +455,12 @@ is_fully_acked(_, _, #srs{
 is_fully_acked(Comm1, Comm2, #srs{last_seqno_qos1 = S1, last_seqno_qos2 = S2}) ->
     (Comm1 >= S1) andalso (Comm2 >= S2).
 
--spec shuffle([A]) -> [A].
+fold_proper_subscriptions(Fun, Acc, S) ->
-shuffle(L0) ->
+    emqx_persistent_session_ds_state:fold_subscriptions(
-    L1 = lists:map(
+        fun
-        fun(A) ->
+            (#share{}, _Sub, Acc0) -> Acc0;
-            %% maybe topic/stream prioritization could be introduced here?
+            (TopicFilter, Sub, Acc0) -> Fun(TopicFilter, Sub, Acc0)
-            {rand:uniform(), A}
         end,
-        L0
+        Acc,
-    ),
+        S
-    L2 = lists:sort(L1),
+    ).
-    {_, L} = lists:unzip(L2),
-    L.

apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_subs.erl

@@ -30,8 +30,7 @@
     on_session_drop/2,
     gc/1,
     lookup/2,
-    to_map/1,
+    to_map/1
-    fold/3
 ]).
 
 %% Management API:
@@ -93,6 +92,7 @@ on_subscribe(TopicFilter, SubOpts, #{id := SessionId, s := S0, props := Props})
             case emqx_persistent_session_ds_state:n_subscriptions(S0) < MaxSubscriptions of
                 true ->
                     ok = emqx_persistent_session_ds_router:do_add_route(TopicFilter, SessionId),
+                    _ = emqx_external_broker:add_persistent_route(TopicFilter, SessionId),
                     {SubId, S1} = emqx_persistent_session_ds_state:new_id(S0),
                     {SStateId, S2} = emqx_persistent_session_ds_state:new_id(S1),
                     SState = #{
@@ -155,12 +155,13 @@ on_unsubscribe(SessionId, TopicFilter, S0) ->
                 #{session_id => SessionId, topic_filter => TopicFilter},
                 ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilter, SessionId)
             ),
+            _ = emqx_external_broker:delete_persistent_route(TopicFilter, SessionId),
             {ok, emqx_persistent_session_ds_state:del_subscription(TopicFilter, S0), Subscription}
     end.
 
 -spec on_session_drop(emqx_persistent_session_ds:id(), emqx_persistent_session_ds_state:t()) -> ok.
 on_session_drop(SessionId, S0) ->
-    fold(
+    _ = fold_proper_subscriptions(
         fun(TopicFilter, _Subscription, S) ->
             case on_unsubscribe(SessionId, TopicFilter, S) of
                 {ok, S1, _} -> S1;
@@ -169,10 +170,14 @@ on_session_drop(SessionId, S0) ->
         end,
         S0,
         S0
-    ).
+    ),
+    ok.
 
 %% @doc Remove subscription states that don't have a parent, and that
-%% don't have any unacked messages:
+%% don't have any unacked messages.
+%% TODO
+%% This function collects shared subs as well
+%% Move to a separate module to keep symmetry?
 -spec gc(emqx_persistent_session_ds_state:t()) -> emqx_persistent_session_ds_state:t().
 gc(S0) ->
     %% Create a set of subscription states IDs referenced either by a
@@ -210,7 +215,7 @@ gc(S0) ->
         S0
     ).
 
-%% @doc Fold over active subscriptions:
+%% @doc Lookup a subscription and merge it with its current state:
 -spec lookup(emqx_persistent_session_ds:topic_filter(), emqx_persistent_session_ds_state:t()) ->
     emqx_persistent_session_ds:subscription() | undefined.
 lookup(TopicFilter, S) ->
@@ -230,22 +235,12 @@ lookup(TopicFilter, S) ->
 %% purpose:
 -spec to_map(emqx_persistent_session_ds_state:t()) -> map().
 to_map(S) ->
-    fold(
+    fold_proper_subscriptions(
         fun(TopicFilter, _, Acc) -> Acc#{TopicFilter => lookup(TopicFilter, S)} end,
         #{},
         S
     ).
 
-%% @doc Fold over active subscriptions:
--spec fold(
-    fun((emqx_types:topic(), emqx_persistent_session_ds:subscription(), Acc) -> Acc),
-    Acc,
-    emqx_persistent_session_ds_state:t()
-) ->
-    Acc.
-fold(Fun, Acc, S) ->
-    emqx_persistent_session_ds_state:fold_subscriptions(Fun, Acc, S).
-
 -spec cold_get_subscription(emqx_persistent_session_ds:id(), emqx_types:topic()) ->
     emqx_persistent_session_ds:subscription() | undefined.
 cold_get_subscription(SessionId, Topic) ->
@@ -267,5 +262,15 @@ cold_get_subscription(SessionId, Topic) ->
 %% Internal functions
 %%================================================================================
 
+fold_proper_subscriptions(Fun, Acc, S) ->
+    emqx_persistent_session_ds_state:fold_subscriptions(
+        fun
+            (#share{}, _Sub, Acc0) -> Acc0;
+            (TopicFilter, Sub, Acc0) -> Fun(TopicFilter, Sub, Acc0)
+        end,
+        Acc,
+        S
+    ).
+
 now_ms() ->
     erlang:system_time(millisecond).

apps/emqx/src/emqx_persistent_session_ds/session_internals.hrl

@@ -71,4 +71,11 @@
     sub_state_id :: emqx_persistent_session_ds_subs:subscription_state_id()
 }).
 
+%% (Erlang) messages that session should forward to the
+%% shared subscription handler.
+-record(shared_sub_message, {
+    message :: term()
+}).
+-define(shared_sub_message(MSG), #shared_sub_message{message = MSG}).
+
 -endif.

apps/emqx/src/emqx_persistent_session_ds/shared_subs_agent.hrl

@@ -0,0 +1,44 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-ifndef(SHARED_SUBS_AGENT_HRL).
+-define(SHARED_SUBS_AGENT_HRL, true).
+
+-ifdef(EMQX_RELEASE_EDITION).
+
+-if(?EMQX_RELEASE_EDITION == ee).
+
+%% agent from BSL app
+
+-ifdef(TEST).
+
+-define(shared_subs_agent, emqx_ds_shared_sub_agent).
+
+%% clause of -ifdef(TEST).
+-else.
+
+%% Till full implementation we need to dispach to the null agent.
+%% It will report "not implemented" error for attempts to use shared subscriptions.
+-define(shared_subs_agent, emqx_persistent_session_ds_shared_subs_null_agent).
+
+%% end of -ifdef(TEST).
+-endif.
+
+%% clause of -if(?EMQX_RELEASE_EDITION == ee).
+-else.
+
+-define(shared_subs_agent, emqx_persistent_session_ds_shared_subs_null_agent).
+
+%% end of -if(?EMQX_RELEASE_EDITION == ee).
+-endif.
+
+%% clause of -ifdef(EMQX_RELEASE_EDITION).
+-else.
+
+-define(shared_subs_agent, emqx_persistent_session_ds_shared_subs_null_agent).
+
+%% end of -ifdef(EMQX_RELEASE_EDITION).
+-endif.
+
+-endif.

apps/emqx/src/emqx_post_upgrade.erl

@@ -0,0 +1,40 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_post_upgrade).
+
+%% Example of a hot upgrade callback function.
+%% PR#12765
+% -export([
+%     pr12765_update_stats_timer/1,
+%     pr20000_ensure_sup_started/3
+% ]).
+
+%% Please ensure that every callback function is reentrant.
+%% This way, users can attempt upgrade multiple times if an issue arises.
+%%
+% pr12765_update_stats_timer(_FromVsn) ->
+%     emqx_stats:update_interval(broker_stats, fun emqx_broker_helper:stats_fun/0).
+%
+% pr20000_ensure_sup_started(_FromVsn, "5.6.1" ++ _, ChildSpec) ->
+%     ChildId = maps:get(id, ChildSpec),
+%     case supervisor:terminate_child(emqx_sup, ChildId) of
+%         ok -> supervisor:delete_child(emqx_sup, ChildId);
+%         Error -> Error
+%     end,
+%     supervisor:start_child(emqx_sup, ChildSpec);
+% pr20000_ensure_sup_started(_FromVsn, _TargetVsn, _) ->
+%     ok.

apps/emqx/src/emqx_relup.erl

@@ -1,42 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_relup).
-
-%% NOTE: DO NOT remove this `-include`.
-%% We use this to force this module to be upgraded every release.
--include("emqx_release.hrl").
-
--export([
-    post_release_upgrade/2,
-    post_release_downgrade/2
-]).
-
--define(INFO(FORMAT), io:format("[emqx_relup] " ++ FORMAT ++ "~n")).
--define(INFO(FORMAT, ARGS), io:format("[emqx_relup] " ++ FORMAT ++ "~n", ARGS)).
-
-%% What to do after upgraded from an old release vsn.
-post_release_upgrade(FromRelVsn, _) ->
-    ?INFO("emqx has been upgraded from ~s to ~s!", [FromRelVsn, emqx_release:version()]),
-    reload_components().
-
-%% What to do after downgraded to an old release vsn.
-post_release_downgrade(ToRelVsn, _) ->
-    ?INFO("emqx has been downgraded from ~s to ~s!", [emqx_release:version(), ToRelVsn]),
-    reload_components().
-
-reload_components() ->
-    ok.

apps/emqx/src/emqx_router.erl

@@ -107,7 +107,14 @@
     unused = [] :: nil()
 }).
 
--define(node_patterns(Node), [Node, {'_', Node}]).
+-define(dest_patterns(NodeOrExtDest),
+    case is_atom(NodeOrExtDest) of
+        %% node
+        true -> [NodeOrExtDest, {'_', NodeOrExtDest}];
+        %% external destination
+        false -> [NodeOrExtDest]
+    end
+).
 
 -define(UNSUPPORTED, unsupported).
 
@@ -307,13 +314,13 @@ print_routes(Topic) ->
     ).
 
 -spec cleanup_routes(node()) -> ok.
-cleanup_routes(Node) ->
+cleanup_routes(NodeOrExtDest) ->
-    cleanup_routes(get_schema_vsn(), Node).
+    cleanup_routes(get_schema_vsn(), NodeOrExtDest).
 
-cleanup_routes(v2, Node) ->
+cleanup_routes(v2, NodeOrExtDest) ->
-    cleanup_routes_v2(Node);
+    cleanup_routes_v2(NodeOrExtDest);
-cleanup_routes(v1, Node) ->
+cleanup_routes(v1, NodeOrExtDest) ->
-    cleanup_routes_v1(Node).
+    cleanup_routes_v1(NodeOrExtDest).
 
 -spec foldl_routes(fun((emqx_types:route(), Acc) -> Acc), Acc) -> Acc.
 foldl_routes(FoldFun, AccIn) ->
@@ -430,19 +437,19 @@ has_route_v1(Topic, Dest) ->
 has_route_tab_entry(Topic, Dest) ->
     [] =/= ets:match(?ROUTE_TAB, #route{topic = Topic, dest = Dest}).
 
-cleanup_routes_v1(Node) ->
+cleanup_routes_v1(NodeOrExtDest) ->
     ?with_fallback(
         lists:foreach(
             fun(Pattern) ->
                 throw_unsupported(mria:match_delete(?ROUTE_TAB, make_route_rec_pat(Pattern)))
             end,
-            ?node_patterns(Node)
+            ?dest_patterns(NodeOrExtDest)
         ),
-        cleanup_routes_v1_fallback(Node)
+        cleanup_routes_v1_fallback(NodeOrExtDest)
     ).
 
-cleanup_routes_v1_fallback(Node) ->
+cleanup_routes_v1_fallback(NodeOrExtDest) ->
-    Patterns = [make_route_rec_pat(P) || P <- ?node_patterns(Node)],
+    Patterns = [make_route_rec_pat(P) || P <- ?dest_patterns(NodeOrExtDest)],
     mria:transaction(?ROUTE_SHARD, fun() ->
         [
             mnesia:delete_object(?ROUTE_TAB, Route, write)
@@ -512,7 +519,7 @@ lookup_routes_v2(Topic) ->
     case emqx_topic:wildcard(Topic) of
         true ->
             Pat = #routeidx{entry = emqx_topic_index:make_key(Topic, '$1')},
-            [Dest || [Dest] <- ets:match(?ROUTE_TAB_FILTERS, Pat)];
+            [#route{topic = Topic, dest = Dest} || [Dest] <- ets:match(?ROUTE_TAB_FILTERS, Pat)];
         false ->
             lookup_route_tab(Topic)
     end.
@@ -525,7 +532,7 @@ has_route_v2(Topic, Dest) ->
             has_route_tab_entry(Topic, Dest)
     end.
 
-cleanup_routes_v2(Node) ->
+cleanup_routes_v2(NodeOrExtDest) ->
     ?with_fallback(
         lists:foreach(
             fun(Pattern) ->
@@ -537,18 +544,18 @@ cleanup_routes_v2(Node) ->
                 ),
                 throw_unsupported(mria:match_delete(?ROUTE_TAB, make_route_rec_pat(Pattern)))
             end,
-            ?node_patterns(Node)
+            ?dest_patterns(NodeOrExtDest)
         ),
-        cleanup_routes_v2_fallback(Node)
+        cleanup_routes_v2_fallback(NodeOrExtDest)
     ).
 
-cleanup_routes_v2_fallback(Node) ->
+cleanup_routes_v2_fallback(NodeOrExtDest) ->
     %% NOTE
     %% No point in transaction here because all the operations on filters table are dirty.
     ok = ets:foldl(
         fun(#routeidx{entry = K}, ok) ->
             case get_dest_node(emqx_topic_index:get_id(K)) of
-                Node ->
+                NodeOrExtDest ->
                     mria:dirty_delete(?ROUTE_TAB_FILTERS, K);
                 _ ->
                     ok
@@ -560,7 +567,7 @@ cleanup_routes_v2_fallback(Node) ->
     ok = ets:foldl(
         fun(#route{dest = Dest} = Route, ok) ->
             case get_dest_node(Dest) of
-                Node ->
+                NodeOrExtDest ->
                     mria:dirty_delete_object(?ROUTE_TAB, Route);
                 _ ->
                     ok
@@ -570,6 +577,8 @@ cleanup_routes_v2_fallback(Node) ->
         ?ROUTE_TAB
     ).
 
+get_dest_node({external, _} = ExtDest) ->
+    ExtDest;
 get_dest_node({_, Node}) ->
     Node;
 get_dest_node(Node) ->

apps/emqx/src/emqx_router_syncer.erl

@@ -21,11 +21,17 @@
 
 -behaviour(gen_server).
 
+-export([start_link/1]).
 -export([start_link/2]).
+-export([start_link_pooled/2]).
 
 -export([push/4]).
+-export([push/5]).
 -export([wait/1]).
 
+-export([suspend/1]).
+-export([activate/1]).
+
 -export([stats/0]).
 
 -export([
@@ -38,6 +44,15 @@
 
 -type action() :: add | delete.
 
+-type options() :: #{
+    max_batch_size => pos_integer(),
+    min_sync_interval => non_neg_integer(),
+    error_delay => non_neg_integer(),
+    error_retry_interval => non_neg_integer(),
+    initial_state => activated | suspended,
+    batch_handler => {module(), _Function :: atom(), _Args :: list()}
+}.
+
 -define(POOL, router_syncer_pool).
 
 -define(MAX_BATCH_SIZE, 1000).
@@ -77,13 +92,23 @@
 
 %%
 
--spec start_link(atom(), pos_integer()) ->
+-spec start_link(options()) ->
+    {ok, pid()} | {error, _Reason}.
+start_link(Options) ->
+    gen_server:start_link(?MODULE, mk_state(Options), []).
+
+-spec start_link(_Name, options()) ->
+    {ok, pid()} | {error, _Reason}.
+start_link(Name, Options) ->
+    gen_server:start_link(Name, ?MODULE, mk_state(Options), []).
+
+-spec start_link_pooled(atom(), pos_integer()) ->
     {ok, pid()}.
-start_link(Pool, Id) ->
+start_link_pooled(Pool, Id) ->
     gen_server:start_link(
         {local, emqx_utils:proc_name(?MODULE, Id)},
         ?MODULE,
-        [Pool, Id],
+        {Pool, Id, mk_state(#{})},
         []
     ).
 
@@ -93,9 +118,16 @@ when
     Opts :: #{reply => pid()}.
 push(Action, Topic, Dest, Opts) ->
     Worker = gproc_pool:pick_worker(?POOL, Topic),
+    push(Worker, Action, Topic, Dest, Opts).
+
+-spec push(_Ref, action(), emqx_types:topic(), emqx_router:dest(), Opts) ->
+    ok | _WaitRef :: reference()
+when
+    Opts :: #{reply => pid()}.
+push(Ref, Action, Topic, Dest, Opts) ->
     Prio = designate_prio(Action, Opts),
     Context = mk_push_context(Opts),
-    _ = erlang:send(Worker, ?PUSH(Prio, {Action, Topic, Dest, Context})),
+    _ = gproc:send(Ref, ?PUSH(Prio, {Action, Topic, Dest, Context})),
     case Context of
         [{MRef, _}] ->
             MRef;
@@ -134,6 +166,16 @@ mk_push_context(_) ->
 
 %%
 
+%% Suspended syncer receives and accumulates route ops but doesn't apply them
+%% until it is activated.
+suspend(Ref) ->
+    gen_server:call(Ref, suspend, infinity).
+
+activate(Ref) ->
+    gen_server:call(Ref, activate, infinity).
+
+%%
+
 -type stats() :: #{
     size := non_neg_integer(),
     n_add := non_neg_integer(),
@@ -149,10 +191,34 @@ stats() ->
 
 %%
 
-init([Pool, Id]) ->
+mk_state(Options) ->
-    true = gproc_pool:connect_worker(Pool, {Pool, Id}),
+    #{
-    {ok, #{stash => stash_new()}}.
+        state => maps:get(initial_state, Options, active),
+        stash => stash_new(),
+        retry_timer => undefined,
+        max_batch_size => maps:get(max_batch_size, Options, ?MAX_BATCH_SIZE),
+        min_sync_interval => maps:get(min_sync_interval, Options, ?MIN_SYNC_INTERVAL),
+        error_delay => maps:get(error_delay, Options, ?ERROR_DELAY),
+        error_retry_interval => maps:get(error_retry_interval, Options, ?ERROR_RETRY_INTERVAL),
+        batch_handler => maps:get(batch_handler, Options, default)
+    }.
 
+%%
+
+init({Pool, Id, State}) ->
+    true = gproc_pool:connect_worker(Pool, {Pool, Id}),
+    {ok, State};
+init(State) ->
+    {ok, State}.
+
+handle_call(suspend, _From, State) ->
+    NState = State#{state := suspended},
+    {reply, ok, NState};
+handle_call(activate, _From, State = #{state := suspended}) ->
+    NState = run_batch_loop([], State#{state := active}),
+    {reply, ok, NState};
+handle_call(activate, _From, State) ->
+    {reply, ok, State};
 handle_call(stats, _From, State = #{stash := Stash}) ->
     {reply, stash_stats(Stash), State};
 handle_call(_Call, _From, State) ->
@@ -162,11 +228,11 @@ handle_cast(_Msg, State) ->
     {noreply, State}.
 
 handle_info({timeout, _TRef, retry}, State) ->
-    NState = run_batch_loop([], maps:remove(retry_timer, State)),
+    NState = run_batch_loop([], State#{retry_timer := undefined}),
     {noreply, NState};
-handle_info(Push = ?PUSH(_, _), State) ->
+handle_info(Push = ?PUSH(_, _), State = #{min_sync_interval := MSI}) ->
     %% NOTE: Wait a bit to collect potentially overlapping operations.
-    ok = timer:sleep(?MIN_SYNC_INTERVAL),
+    ok = timer:sleep(MSI),
     NState = run_batch_loop([Push], State),
     {noreply, NState}.
 
@@ -175,12 +241,16 @@ terminate(_Reason, _State) ->
 
 %%
 
-run_batch_loop(Incoming, State = #{stash := Stash0}) ->
+run_batch_loop(Incoming, State = #{stash := Stash0, state := suspended}) ->
     Stash1 = stash_add(Incoming, Stash0),
     Stash2 = stash_drain(Stash1),
-    {Batch, Stash3} = mk_batch(Stash2),
+    State#{stash := Stash2};
+run_batch_loop(Incoming, State = #{stash := Stash0, max_batch_size := MBS}) ->
+    Stash1 = stash_add(Incoming, Stash0),
+    Stash2 = stash_drain(Stash1),
+    {Batch, Stash3} = mk_batch(Stash2, MBS),
     ?tp_ignore_side_effects_in_prod(router_syncer_new_batch, batch_stats(Batch, Stash3)),
-    case run_batch(Batch) of
+    case run_batch(Batch, State) of
         Status = #{} ->
             ok = send_replies(Status, Batch),
             NState = cancel_retry_timer(State#{stash := Stash3}),
@@ -203,37 +273,37 @@ run_batch_loop(Incoming, State = #{stash := Stash0}) ->
                 batch => batch_stats(Batch, Stash3)
             }),
             NState = State#{stash := Stash2},
-            ok = timer:sleep(?ERROR_DELAY),
+            ok = error_cooldown(NState),
             ensure_retry_timer(NState)
     end.
 
+error_cooldown(#{error_delay := ED}) ->
+    timer:sleep(ED).
+
+ensure_retry_timer(State = #{retry_timer := undefined, error_retry_interval := ERI}) ->
+    TRef = emqx_utils:start_timer(ERI, retry),
+    State#{retry_timer := TRef};
 ensure_retry_timer(State = #{retry_timer := _TRef}) ->
-    State;
+    State.
-ensure_retry_timer(State) ->
-    TRef = emqx_utils:start_timer(?ERROR_RETRY_INTERVAL, retry),
-    State#{retry_timer => TRef}.
 
 cancel_retry_timer(State = #{retry_timer := TRef}) ->
     ok = emqx_utils:cancel_timer(TRef),
-    maps:remove(retry_timer, State);
+    State#{retry_timer := undefined};
 cancel_retry_timer(State) ->
     State.
 
 %%
 
-mk_batch(Stash) when map_size(Stash) =< ?MAX_BATCH_SIZE ->
+mk_batch(Stash, BatchSize) when map_size(Stash) =< BatchSize ->
     %% This is perfect situation, we just use stash as batch w/o extra reallocations.
     {Stash, stash_new()};
-mk_batch(Stash) ->
+mk_batch(Stash, BatchSize) ->
     %% Take a subset of stashed operations to form a batch.
     %% Note that stash is an unordered map, it's not a queue. The order of operations is
     %% not preserved strictly, only loosely, because of how we start from high priority
     %% operations and go down to low priority ones. This might cause some operations to
     %% stay in stash for unfairly long time, when there are many high priority operations.
     %% However, it's unclear how likely this is to happen in practice.
-    mk_batch(Stash, ?MAX_BATCH_SIZE).
-
-mk_batch(Stash, BatchSize) ->
     mk_batch(?PRIO_HI, #{}, BatchSize, Stash).
 
 mk_batch(Prio, Batch, SizeLeft, Stash) ->
@@ -278,10 +348,12 @@ replyctx_send(Result, RefsPids) ->
 
 %%
 
-run_batch(Batch) when map_size(Batch) > 0 ->
+run_batch(Empty, _State) when Empty =:= #{} ->
+    #{};
+run_batch(Batch, #{batch_handler := default}) ->
     catch emqx_router:do_batch(Batch);
-run_batch(_Empty) ->
+run_batch(Batch, #{batch_handler := {Module, Function, Args}}) ->
-    #{}.
+    erlang:apply(Module, Function, [Batch | Args]).
 
 %%
 

apps/emqx/src/emqx_rpc.erl

@@ -137,7 +137,7 @@ maybe_badrpc(Delivery) ->
     Delivery.
 
 max_client_num() ->
-    emqx:get_config([rpc, tcp_client_num], ?DefaultClientNum).
+    emqx:get_config([rpc, client_num], ?DefaultClientNum).
 
 -spec unwrap_erpc(emqx_rpc:erpc(A) | [emqx_rpc:erpc(A)]) -> A | {error, _Err} | list().
 unwrap_erpc(Res) when is_list(Res) ->

apps/emqx/src/emqx_schema.erl

@@ -1970,10 +1970,6 @@ zones_field_schema() ->
         }
     ).
 
-desc("persistent_session_store") ->
-    "Settings for message persistence.";
-desc("persistent_session_builtin") ->
-    "Settings for the built-in storage engine of persistent messages.";
 desc("persistent_table_mria_opts") ->
     "Tuning options for the mria table.";
 desc("stats") ->
@@ -3643,9 +3639,9 @@ mqtt_general() ->
             )},
         {"retry_interval",
             sc(
-                timeout_duration(),
+                hoconsc:union([infinity, timeout_duration()]),
                 #{
-                    default => <<"30s">>,
+                    default => infinity,
                     desc => ?DESC(mqtt_retry_interval)
                 }
             )},

apps/emqx/src/emqx_secret.erl

@@ -30,7 +30,7 @@
 -define(LOADER, emqx_secret_loader).
 
 %%================================================================================
-%% API funcions
+%% API functions
 %%================================================================================
 
 %% @doc Wrap a term in a secret closure.

apps/emqx/src/emqx_session.erl

@@ -83,6 +83,7 @@
 
 -export([
     deliver/3,
+    handle_info/2,
     handle_timeout/3,
     disconnect/3,
     terminate/3
@@ -188,6 +189,10 @@
 -callback destroy(t() | clientinfo()) -> ok.
 -callback clear_will_message(t()) -> t().
 -callback publish_will_message_now(t(), message()) -> t().
+-callback handle_timeout(clientinfo(), common_timer_name() | custom_timer_name(), t()) ->
+    {ok, replies(), t()}
+    | {ok, replies(), timeout(), t()}.
+-callback handle_info(term(), t()) -> t().
 
 %%--------------------------------------------------------------------
 %% Create a Session
@@ -484,6 +489,14 @@ enrich_subopts(_Opt, _V, Msg, _) ->
 handle_timeout(ClientInfo, Timer, Session) ->
     ?IMPL(Session):handle_timeout(ClientInfo, Timer, Session).
 
+%%--------------------------------------------------------------------
+%% Generic Messages
+%%--------------------------------------------------------------------
+
+-spec handle_info(term(), t()) -> t().
+handle_info(Info, Session) ->
+    ?IMPL(Session):handle_info(Info, Session).
+
 %%--------------------------------------------------------------------
 
 -spec ensure_timer(custom_timer_name(), timeout(), map()) ->
@@ -601,7 +614,7 @@ should_keep(MsgDeliver) ->
     not is_banned_msg(MsgDeliver).
 
 is_banned_msg(#message{from = ClientId}) ->
-    [] =/= emqx_banned:look_up({clientid, ClientId}).
+    emqx_banned:check_clientid(ClientId).
 
 %%--------------------------------------------------------------------
 

apps/emqx/src/emqx_session_mem.erl

@@ -87,6 +87,7 @@
     deliver/3,
     replay/3,
     handle_timeout/3,
+    handle_info/2,
     disconnect/2,
     terminate/2
 ]).
@@ -597,14 +598,23 @@ handle_timeout(ClientInfo, retry_delivery, Session) ->
 handle_timeout(ClientInfo, expire_awaiting_rel, Session) ->
     expire(ClientInfo, Session).
 
+%%--------------------------------------------------------------------
+%% Geneic messages
+%%--------------------------------------------------------------------
+
+-spec handle_info(term(), session()) -> session().
+handle_info(Msg, Session) ->
+    ?SLOG(warning, #{msg => emqx_session_mem_unknown_message, message => Msg}),
+    Session.
+
 %%--------------------------------------------------------------------
 %% Retry Delivery
 %%--------------------------------------------------------------------
 
 -spec retry(clientinfo(), session()) ->
-    {ok, replies(), session()}.
+    {ok, replies(), session()} | {ok, replies(), timeout(), session()}.
-retry(ClientInfo, Session = #session{inflight = Inflight}) ->
+retry(ClientInfo, Session = #session{inflight = Inflight, retry_interval = Interval}) ->
-    case emqx_inflight:is_empty(Inflight) of
+    case emqx_inflight:is_empty(Inflight) orelse Interval =:= infinity of
         true ->
             {ok, [], Session};
         false ->

apps/emqx/src/emqx_shared_sub.erl

@@ -421,8 +421,12 @@ init_monitors() ->
 handle_call({subscribe, Group, Topic, SubPid}, _From, State = #state{pmon = PMon}) ->
     mria:dirty_write(?SHARED_SUBSCRIPTION, record(Group, Topic, SubPid)),
     case ets:member(?SHARED_SUBSCRIBER, {Group, Topic}) of
-        true -> ok;
+        true ->
-        false -> ok = emqx_router:do_add_route(Topic, {Group, node()})
+            ok;
+        false ->
+            ok = emqx_router:do_add_route(Topic, {Group, node()}),
+            _ = emqx_external_broker:add_shared_route(Topic, Group),
+            ok
     end,
     ok = maybe_insert_alive_tab(SubPid),
     ok = maybe_insert_round_robin_count({Group, Topic}),
@@ -545,7 +549,9 @@ is_alive_sub(Pid) ->
 
 delete_route_if_needed({Group, Topic} = GroupTopic) ->
     if_no_more_subscribers(GroupTopic, fun() ->
-        ok = emqx_router:do_delete_route(Topic, {Group, node()})
+        ok = emqx_router:do_delete_route(Topic, {Group, node()}),
+        _ = emqx_external_broker:delete_shared_route(Topic, Group),
+        ok
     end).
 
 get_default_shared_subscription_strategy() ->

apps/emqx/src/emqx_topic.erl

@@ -33,7 +33,8 @@
     feed_var/3,
     systop/1,
     parse/1,
-    parse/2
+    parse/2,
+    intersection/2
 ]).
 
 -export([
@@ -52,6 +53,8 @@
     ((C =:= '#' orelse C =:= <<"#">>) andalso REST =/= [])
 ).
 
+-define(IS_WILDCARD(W), W =:= '+' orelse W =:= '#').
+
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -98,6 +101,55 @@ match(_, ['#']) ->
 match(_, _) ->
     false.
 
+%% @doc Finds an intersection between two topics, two filters or a topic and a filter.
+%% The function is commutative: reversing parameters doesn't affect the returned value.
+%% Two topics intersect only when they are equal.
+%% The intersection of a topic and a filter is always either the topic itself or false (no intersection).
+%% The intersection of two filters is either false or a new topic filter that would match only those topics,
+%% that can be matched by both input filters.
+%% For example, the intersection of "t/global/#" and "t/+/1/+" is "t/global/1/+".
+-spec intersection(TopicOrFilter, TopicOrFilter) -> TopicOrFilter | false when
+    TopicOrFilter :: emqx_types:topic().
+intersection(Topic1, Topic2) when is_binary(Topic1), is_binary(Topic2) ->
+    case intersect_start(words(Topic1), words(Topic2)) of
+        false -> false;
+        Intersection -> join(Intersection)
+    end.
+
+intersect_start([<<"$", _/bytes>> | _], [W | _]) when ?IS_WILDCARD(W) ->
+    false;
+intersect_start([W | _], [<<"$", _/bytes>> | _]) when ?IS_WILDCARD(W) ->
+    false;
+intersect_start(Words1, Words2) ->
+    intersect(Words1, Words2).
+
+intersect(Words1, ['#']) ->
+    Words1;
+intersect(['#'], Words2) ->
+    Words2;
+intersect([W1], ['+']) ->
+    [W1];
+intersect(['+'], [W2]) ->
+    [W2];
+intersect([W1 | T1], [W2 | T2]) when ?IS_WILDCARD(W1), ?IS_WILDCARD(W2) ->
+    intersect_join(wildcard_intersection(W1, W2), intersect(T1, T2));
+intersect([W | T1], [W | T2]) ->
+    intersect_join(W, intersect(T1, T2));
+intersect([W1 | T1], [W2 | T2]) when ?IS_WILDCARD(W1) ->
+    intersect_join(W2, intersect(T1, T2));
+intersect([W1 | T1], [W2 | T2]) when ?IS_WILDCARD(W2) ->
+    intersect_join(W1, intersect(T1, T2));
+intersect([], []) ->
+    [];
+intersect(_, _) ->
+    false.
+
+intersect_join(_, false) -> false;
+intersect_join(W, Words) -> [W | Words].
+
+wildcard_intersection(W, W) -> W;
+wildcard_intersection(_, _) -> '+'.
+
 -spec match_share(Name, Filter) -> boolean() when
     Name :: share(),
     Filter :: topic() | share().

apps/emqx/src/emqx_topic_index.erl

@@ -23,6 +23,7 @@
 -export([delete/3]).
 -export([match/2]).
 -export([matches/3]).
+-export([matches_filter/3]).
 
 -export([make_key/2]).
 
@@ -72,6 +73,12 @@ match(Topic, Tab) ->
 matches(Topic, Tab, Opts) ->
     emqx_trie_search:matches(Topic, make_nextf(Tab), Opts).
 
+%% @doc Match given topic filter against the index and return _all_ matches.
+%% If `unique` option is given, return only unique matches by record ID.
+-spec matches_filter(emqx_types:topic(), ets:table(), emqx_trie_search:opts()) -> [match(_ID)].
+matches_filter(TopicFilter, Tab, Opts) ->
+    emqx_trie_search:matches_filter(TopicFilter, make_nextf(Tab), Opts).
+
 %% @doc Extract record ID from the match.
 -spec get_id(match(ID)) -> ID.
 get_id(Key) ->

apps/emqx/src/emqx_trie_search.erl

@@ -99,7 +99,7 @@
 -module(emqx_trie_search).
 
 -export([make_key/2, make_pat/2, filter/1]).
--export([match/2, matches/3, get_id/1, get_topic/1]).
+-export([match/2, matches/3, get_id/1, get_topic/1, matches_filter/3]).
 -export_type([key/1, word/0, words/0, nextf/0, opts/0]).
 
 -define(END, '$end_of_table').
@@ -183,9 +183,20 @@ match(Topic, NextF) ->
 matches(Topic, NextF, Opts) ->
     search(Topic, NextF, Opts).
 
+%% @doc Match given topic filter against the index and return _all_ matches.
+-spec matches_filter(emqx_types:topic(), nextf(), opts()) -> [key(_)].
+matches_filter(TopicFilter, NextF, Opts) ->
+    search(TopicFilter, NextF, [topic_filter | Opts]).
+
 %% @doc Entrypoint of the search for a given topic.
 search(Topic, NextF, Opts) ->
-    Words = topic_words(Topic),
+    %% A private opt
+    IsFilter = proplists:get_bool(topic_filter, Opts),
+    Words =
+        case IsFilter of
+            true -> filter_words(Topic);
+            false -> topic_words(Topic)
+        end,
     Base = base_init(Words),
     ORetFirst = proplists:get_bool(return_first, Opts),
     OUnique = proplists:get_bool(unique, Opts),
@@ -200,8 +211,10 @@ search(Topic, NextF, Opts) ->
         end,
     Matches =
         case search_new(Words, Base, NextF, Acc0) of
-            {Cursor, Acc} ->
+            {Cursor, Acc} when not IsFilter ->
                 match_topics(Topic, Cursor, NextF, Acc);
+            {_Cursor, Acc} ->
+                Acc;
             Acc ->
                 Acc
         end,
@@ -275,6 +288,17 @@ compare(['#'], _Words, _) ->
     % Closest possible next entries that we must not miss:
     % * a/+/+/d/# (same topic but a different ID)
     match_full;
+%% Filter search %%
+compare(_Filter, ['#'], _) ->
+    match_full;
+compare([_ | TF], ['+' | TW], Pos) ->
+    case compare(TF, TW, Pos + 1) of
+        lower ->
+            lower;
+        Other ->
+            Other
+    end;
+%% Filter search end %%
 compare(['+' | TF], [HW | TW], Pos) ->
     case compare(TF, TW, Pos + 1) of
         lower ->

apps/emqx/src/emqx_types.erl

@@ -267,6 +267,7 @@
     [
         {node(), topic(), deliver_result()}
         | {share, topic(), deliver_result()}
+        | {emqx_external_broker:dest(), topic(), deliver_result()}
         | persisted
     ]
     | disconnect.

apps/emqx/src/emqx_vm.erl

@@ -399,7 +399,6 @@ compat_windows(Fun) when is_function(Fun, 0) ->
             0.0
     end;
 compat_windows(Fun) ->
-    ?SLOG(warning, "Invalid function: ~p", [Fun]),
     error({badarg, Fun}).
 
 load(Avg) ->

apps/emqx/src/emqx_ws_connection.erl

@@ -303,7 +303,7 @@ websocket_init([Req, Opts]) ->
                 max_size => emqx_config:get_zone_conf(Zone, [mqtt, max_packet_size])
             },
             ParseState = emqx_frame:initial_parse_state(FrameOpts),
-            Serialize = emqx_frame:serialize_opts(),
+            Serialize = emqx_frame:initial_serialize_opts(FrameOpts),
             Channel = emqx_channel:init(ConnInfo, Opts),
             GcState = get_force_gc(Zone),
             StatsTimer = get_stats_enable(Zone),
@@ -455,8 +455,8 @@ websocket_info({incoming, Packet}, State) ->
     handle_incoming(Packet, State);
 websocket_info({outgoing, Packets}, State) ->
     return(enqueue(Packets, State));
-websocket_info({check_gc, Stats}, State) ->
+websocket_info({check_gc, Cnt, Oct}, State) ->
-    return(check_oom(run_gc(Stats, State)));
+    return(check_oom(run_gc(Cnt, Oct, State)));
 websocket_info(
     Deliver = {deliver, _Topic, _Msg},
     State = #state{listener = {Type, Listener}}
@@ -603,17 +603,23 @@ check_limiter(
     Data,
     WhenOk,
     Msgs,
-    #state{limiter_timer = undefined, limiter = Limiter} = State
+    #state{channel = Channel, limiter_timer = undefined, limiter = Limiter} = State
 ) ->
     case emqx_limiter_container:check_list(Needs, Limiter) of
         {ok, Limiter2} ->
             WhenOk(Data, Msgs, State#state{limiter = Limiter2});
         {pause, Time, Limiter2} ->
-            ?SLOG(debug, #{
+            ?SLOG_THROTTLE(
-                msg => "pause_time_due_to_rate_limit",
+                warning,
-                needs => Needs,
+                #{
-                time_in_ms => Time
+                    msg => socket_receive_paused_by_rate_limit,
-            }),
+                    paused_ms => Time
+                },
+                #{
+                    tag => "RATE",
+                    clientid => emqx_channel:info(clientid, Channel)
+                }
+            ),
 
             Retry = #retry{
                 types = [Type || {_, Type} <- Needs],
@@ -647,7 +653,7 @@ check_limiter(
     State#state{limiter_buffer = queue:in(New, Buffer)}.
 
 -spec retry_limiter(state()) -> state().
-retry_limiter(#state{limiter = Limiter} = State) ->
+retry_limiter(#state{channel = Channel, limiter = Limiter} = State) ->
     #retry{types = Types, data = Data, next = Next} = emqx_limiter_container:get_retry_context(
         Limiter
     ),
@@ -662,11 +668,17 @@ retry_limiter(#state{limiter = Limiter} = State) ->
                 }
             );
         {pause, Time, Limiter2} ->
-            ?SLOG(debug, #{
+            ?SLOG_THROTTLE(
-                msg => "pause_time_due_to_rate_limit",
+                warning,
-                types => Types,
+                #{
-                time_in_ms => Time
+                    msg => socket_receive_paused_by_rate_limit,
-            }),
+                    paused_ms => Time
+                },
+                #{
+                    tag => "RATE",
+                    clientid => emqx_channel:info(clientid, Channel)
+                }
+            ),
 
             TRef = start_timer(Time, limit_timeout),
 
@@ -682,8 +694,8 @@ when_msg_in(Packets, Msgs, State) ->
 %% Run GC, Check OOM
 %%--------------------------------------------------------------------
 
-run_gc(Stats, State = #state{gc_state = GcSt}) ->
+run_gc(Cnt, Oct, State = #state{gc_state = GcSt}) ->
-    case ?ENABLED(GcSt) andalso emqx_gc:run(Stats, GcSt) of
+    case ?ENABLED(GcSt) andalso emqx_gc:run(Cnt, Oct, GcSt) of
         false -> State;
         {_IsGC, GcSt1} -> State#state{gc_state = GcSt1}
     end.
@@ -796,11 +808,9 @@ handle_outgoing(
                 get_active_n(Type, Listener)
         of
             true ->
-                Stats = #{
+                Cnt = emqx_pd:reset_counter(outgoing_pubs),
-                    cnt => emqx_pd:reset_counter(outgoing_pubs),
+                Oct = emqx_pd:reset_counter(outgoing_bytes),
-                    oct => emqx_pd:reset_counter(outgoing_bytes)
+                postpone({check_gc, Cnt, Oct}, State);
-                },
-                postpone({check_gc, Stats}, State);
             false ->
                 State
         end,

apps/emqx/src/proto/emqx_shared_sub_proto_v1.erl

@@ -33,7 +33,7 @@ introduced_in() ->
     "5.0.8".
 
 %%================================================================================
-%% API funcions
+%% API functions
 %%================================================================================
 
 -spec send(node(), pid(), emqx_types:topic(), term()) -> true.

apps/emqx/test/emqx_banned_SUITE.erl

@@ -112,6 +112,10 @@ t_check(_) ->
     ?assertNot(emqx_banned:check(ClientInfoValidFull)),
     ?assertNot(emqx_banned:check(ClientInfoValidEmpty)),
     ?assertNot(emqx_banned:check(ClientInfoValidOnlyClientId)),
+
+    ?assert(emqx_banned:check_clientid(<<"BannedClient">>)),
+    ?assert(emqx_banned:check_clientid(<<"BannedClientRE">>)),
+
     ok = emqx_banned:delete(emqx_banned:who(clientid, <<"BannedClient">>)),
     ok = emqx_banned:delete(emqx_banned:who(username, <<"BannedUser">>)),
     ok = emqx_banned:delete(emqx_banned:who(peerhost, {192, 168, 0, 1})),
@@ -127,6 +131,10 @@ t_check(_) ->
     ?assertNot(emqx_banned:check(ClientInfoBannedUsernameRE)),
     ?assertNot(emqx_banned:check(ClientInfoBannedAddrNet)),
     ?assertNot(emqx_banned:check(ClientInfoValidFull)),
+
+    ?assertNot(emqx_banned:check_clientid(<<"BannedClient">>)),
+    ?assertNot(emqx_banned:check_clientid(<<"BannedClientRE">>)),
+
     ?assertEqual(0, emqx_banned:info(size)).
 
 t_unused(_) ->

apps/emqx/test/emqx_config_SUITE.erl

@@ -445,7 +445,7 @@ zone_global_defaults() ->
                 peer_cert_as_username => disabled,
                 response_information => [],
                 retain_available => true,
-                retry_interval => 30000,
+                retry_interval => infinity,
                 message_expiry_interval => infinity,
                 server_keepalive => disabled,
                 session_expiry_interval => 7200000,

apps/emqx/test/emqx_connection_SUITE.erl

@@ -333,6 +333,17 @@ t_handle_incoming(_) ->
     ),
     ?assertMatch({ok, _Out, _NState}, emqx_connection:handle_incoming(frame_error, st())).
 
+t_handle_outing_non_utf8_topic(_) ->
+    Topic = <<"测试"/utf16>>,
+    Publish = ?PUBLISH_PACKET(0, Topic, 1),
+    StrictOff = #{version => 5, max_size => 16#FFFF, strict_mode => false},
+    StOff = st(#{serialize => StrictOff}),
+    OffResult = emqx_connection:handle_outgoing(Publish, StOff),
+    ?assertMatch({ok, _}, OffResult),
+    StrictOn = #{version => 5, max_size => 16#FFFF, strict_mode => true},
+    StOn = st(#{serialize => StrictOn}),
+    ?assertError(frame_serialize_error, emqx_connection:handle_outgoing(Publish, StOn)).
+
 t_with_channel(_) ->
     State = st(),
     ok = meck:expect(emqx_channel, handle_in, fun(_, _) -> ok end),
@@ -515,7 +526,7 @@ t_oom_shutdown(_) ->
     with_conn(
         fun(Pid) ->
             Pid ! {tcp_passive, foo},
-            {ok, _} = ?block_until(#{?snk_kind := check_oom}, 1000),
+            {ok, _} = ?block_until(#{?snk_kind := check_oom_shutdown}, 1000),
             {ok, _} = ?block_until(#{?snk_kind := terminate}, 100),
             Trace = snabbkaffe:collect_trace(),
             ?assertEqual(1, length(?of_kind(terminate, Trace))),

apps/emqx/test/emqx_cth_cluster.erl

@@ -38,7 +38,7 @@
 %%    in `end_per_suite/1` or `end_per_group/2`) with the result from step 2.
 -module(emqx_cth_cluster).
 
--export([start/1, start/2, restart/1, restart/2]).
+-export([start/1, start/2, restart/1]).
 -export([stop/1, stop_node/1]).
 
 -export([start_bare_nodes/1, start_bare_nodes/2]).
@@ -158,18 +158,18 @@ wait_clustered([Node | Nodes] = All, Check, Deadline) ->
                     nodes_not_running => NodesNotRunnging
                 }}
             );
-        {false, Nodes} ->
+        {false, _Nodes} ->
             timer:sleep(100),
             wait_clustered(All, Check, Deadline)
     end.
 
-restart(NodeSpec) ->
+restart(NodeSpecs = [_ | _]) ->
-    restart(maps:get(name, NodeSpec), NodeSpec).
+    Nodes = [maps:get(name, Spec) || Spec <- NodeSpecs],
-
+    ct:pal("Stopping peer nodes: ~p", [Nodes]),
-restart(Node, Spec) ->
+    ok = stop(Nodes),
-    ct:pal("Stopping peer node ~p", [Node]),
+    start([Spec#{boot_type => restart} || Spec <- NodeSpecs]);
-    ok = emqx_cth_peer:stop(Node),
+restart(NodeSpec = #{}) ->
-    start([Spec#{boot_type => restart}]).
+    restart([NodeSpec]).
 
 mk_nodespecs(Nodes, ClusterOpts) ->
     NodeSpecs = lists:zipwith(

apps/emqx/test/emqx_cth_peer.erl

@@ -22,6 +22,7 @@
 -export([start/2, start/3, start/4]).
 -export([start_link/2, start_link/3, start_link/4]).
 -export([stop/1]).
+-export([kill/1]).
 
 start(Name, Args) ->
     start(Name, Args, []).
@@ -62,10 +63,36 @@ stop(Node) when is_atom(Node) ->
             unlink(Pid),
             ok = peer:stop(Pid);
         false ->
-            ct:pal("The control process for node ~p is unexpetedly down", [Node]),
+            ct:pal("The control process for node ~p is unexpectedly down", [Node]),
             ok
     end.
 
+%% @doc Kill a node abruptly, through mechanisms provided by OS.
+%% Relies on POSIX `kill`.
+kill(Node) ->
+    try erpc:call(Node, os, getpid, []) of
+        OSPid ->
+            Pid = whereis(Node),
+            _ = is_pid(Pid) andalso unlink(Pid),
+            Result = kill_os_process(OSPid),
+            %% Either ensure control process stops, or try to stop if not killed.
+            _ = is_pid(Pid) andalso catch peer:stop(Pid),
+            Result
+    catch
+        error:{erpc, _} = Reason ->
+            {error, Reason}
+    end.
+
+kill_os_process(OSPid) ->
+    Cmd = "kill -SIGKILL " ++ OSPid,
+    Port = erlang:open_port({spawn, Cmd}, [binary, exit_status, hide]),
+    receive
+        {Port, {exit_status, 0}} ->
+            ok;
+        {Port, {exit_status, EC}} ->
+            {error, EC}
+    end.
+
 parse_node_name(NodeName) ->
     case string:tokens(atom_to_list(NodeName), "@") of
         [Name, Host] ->

apps/emqx/test/emqx_cth_suite.erl

@@ -64,6 +64,7 @@
 
 -export([work_dir/1]).
 -export([work_dir/2]).
+-export([clean_work_dir/1]).
 
 -export([load_apps/1]).
 -export([start_apps/2]).
@@ -162,6 +163,7 @@ start(Apps, SuiteOpts = #{work_dir := WorkDir}) ->
     % 4. Setup isolated mnesia directory
     ok = emqx_common_test_helpers:load(mnesia),
     ok = application:set_env(mnesia, dir, filename:join([WorkDir, mnesia])),
+    ok = application:set_env(emqx_durable_storage, db_data_dir, filename:join([WorkDir, ds])),
     % 5. Start ekka separately.
     % For some reason it's designed to be started in non-regular way, so we have to track
     % applications started in the process manually.
@@ -432,6 +434,16 @@ work_dir(TCName, CTConfig) ->
     WorkDir = work_dir(CTConfig),
     filename:join(WorkDir, TCName).
 
+%% @doc Delete contents of the workdir.
+clean_work_dir(WorkDir) ->
+    ct:pal("Cleaning workdir ~p", [WorkDir]),
+    case re:run(WorkDir, "./_build/test/logs/") of
+        {match, _} ->
+            file:del_dir_r(WorkDir);
+        nomatch ->
+            error({unsafe_workdir, WorkDir})
+    end.
+
 %%
 
 start_ekka() ->

apps/emqx/test/emqx_persistent_messages_SUITE.erl

@@ -32,9 +32,23 @@
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
+%% Needed for standalone mode:
+-ifndef(EMQX_RELEASE_EDITION).
+-define(EMQX_RELEASE_EDITION, ce).
+-endif.
+
+-if(?EMQX_RELEASE_EDITION == ee).
+
 init_per_suite(Config) ->
     Config.
 
+-else.
+
+init_per_suite(Config) ->
+    {skip, no_replication}.
+
+-endif.
+
 end_per_suite(_Config) ->
     ok.
 
@@ -465,7 +479,7 @@ t_metrics_not_dropped(_Config) ->
 t_replication_options(_Config) ->
     ?assertMatch(
         #{
-            backend := builtin,
+            backend := builtin_raft,
             replication_options := #{
                 wal_max_size_bytes := 16000000,
                 wal_max_batch_size := 1024,
@@ -570,7 +584,7 @@ wait_shards_online(Nodes = [Node | _]) ->
     ?retry(500, 10, [?assertEqual(NShards, shards_online(N)) || N <- Nodes]).
 
 shards_online(Node) ->
-    length(erpc:call(Node, emqx_ds_builtin_db_sup, which_shards, [?PERSISTENT_MESSAGE_DB])).
+    length(erpc:call(Node, emqx_ds_builtin_raft_db_sup, which_shards, [?PERSISTENT_MESSAGE_DB])).
 
 get_mqtt_port(Node, Type) ->
     {_IP, Port} = erpc:call(Node, emqx_config, get, [[listeners, Type, default, bind]]),

apps/emqx/test/emqx_persistent_session_SUITE.erl

@@ -81,7 +81,8 @@ init_per_group(persistence_enabled, Config) ->
                 "  heartbeat_interval = 100ms\n"
                 "  renew_streams_interval = 100ms\n"
                 "  session_gc_interval = 2s\n"
-                "}"},
+                "}\n"
+                "durable_storage.messages.backend = builtin_local"},
         {persistence, ds}
         | Config
     ];

apps/emqx/test/emqx_quic_multistreams_SUITE.erl

@@ -841,7 +841,7 @@ t_conn_change_client_addr(Config) ->
     NewPort = select_port(),
     {ok, OldAddr} = quicer:sockname(Conn),
     ?assertEqual(
-        ok, quicer:setopt(Conn, param_conn_local_address, "127.0.0.1:" ++ integer_to_list(NewPort))
+        ok, quicer:setopt(Conn, local_address, "127.0.0.1:" ++ integer_to_list(NewPort))
     ),
     {ok, NewAddr} = quicer:sockname(Conn),
     ct:pal("NewAddr: ~p, Old Addr: ~p", [NewAddr, OldAddr]),
@@ -1681,7 +1681,7 @@ t_client_conn_bump_streams(Config) ->
     ]),
     {ok, _} = emqtt:quic_connect(C),
     {quic, Conn, _Stream} = proplists:get_value(socket, emqtt:info(C)),
-    ok = quicer:setopt(Conn, param_conn_settings, #{peer_unidi_stream_count => 20}).
+    ok = quicer:setopt(Conn, settings, #{peer_unidi_stream_count => 20}).
 
 t_olp_true(Config) ->
     meck:new(emqx_olp, [passthrough, no_history]),

apps/emqx/test/emqx_session_mem_SUITE.erl

@@ -78,7 +78,7 @@ t_session_init(_) ->
     ?assertEqual(0, emqx_session_mem:info(inflight_cnt, Session)),
     ?assertEqual(64, emqx_session_mem:info(inflight_max, Session)),
     ?assertEqual(1, emqx_session_mem:info(next_pkt_id, Session)),
-    ?assertEqual(30000, emqx_session_mem:info(retry_interval, Session)),
+    ?assertEqual(infinity, emqx_session_mem:info(retry_interval, Session)),
     ?assertEqual(0, emqx_mqueue:len(emqx_session_mem:info(mqueue, Session))),
     ?assertEqual(0, emqx_session_mem:info(awaiting_rel_cnt, Session)),
     ?assertEqual(100, emqx_session_mem:info(awaiting_rel_max, Session)),
@@ -95,7 +95,7 @@ t_session_info(_) ->
         #{
             subscriptions := #{},
             upgrade_qos := false,
-            retry_interval := 30000,
+            retry_interval := infinity,
             await_rel_timeout := 300000
         },
         maps:from_list(emqx_session_mem:info(Keys, session()))

apps/emqx/test/emqx_takeover_SUITE.erl

@@ -884,26 +884,23 @@ t_kick_session(Config) ->
         {will_qos, 1}
     ],
     Commands =
-        %% GIVEN: client connect with willmsg payload <<"willpayload_kick">>
+        lists:flatten([
-        [{fun start_client/5, [ClientId, ClientId, ?QOS_1, WillOpts]}] ++
+            %% GIVEN: client connect with willmsg payload <<"willpayload_kick">>
-            [
+            {fun start_client/5, [ClientId, ClientId, ?QOS_1, WillOpts]},
-                {fun start_client/5, [
+            {fun start_client/5, [
-                    <<ClientId/binary, <<"_willsub">>/binary>>, WillTopic, ?QOS_1, []
+                <<ClientId/binary, <<"_willsub">>/binary>>, WillTopic, ?QOS_1, []
-                ]}
+            ]},
-            ] ++
+            %% kick may fail (not found) without this delay
-            [
+            {
-                %% kick may fail (not found) without this delay
+                fun(CTX) ->
-                {
+                    timer:sleep(300),
-                    fun(CTX) ->
+                    CTX
-                        timer:sleep(100),
+                end,
-                        CTX
+                []
-                    end,
+            },
-                    []
-                }
-            ] ++
             %% WHEN: client is kicked with kick_session
-            [{fun kick_client/2, [ClientId]}],
+            {fun kick_client/2, [ClientId]}
-
+        ]),
     FCtx = lists:foldl(
         fun({Fun, Args}, Ctx) ->
             ct:pal("COMMAND: ~p ~p", [element(2, erlang:fun_info(Fun, name)), Args]),
@@ -1045,9 +1042,15 @@ assert_client_exit(Pid, v5, takenover) ->
     %% @ref: MQTT 5.0 spec [MQTT-3.1.4-3]
     ?assertReceive({'EXIT', Pid, {disconnected, ?RC_SESSION_TAKEN_OVER, _}});
 assert_client_exit(Pid, v3, takenover) ->
-    ?assertReceive({'EXIT', Pid, {shutdown, tcp_closed}});
+    ?assertReceive(
+        {'EXIT', Pid, {shutdown, Reason}} when
+            Reason =:= tcp_closed orelse
+                Reason =:= closed,
+        1_000,
+        #{pid => Pid}
+    );
 assert_client_exit(Pid, v3, kicked) ->
-    ?assertReceive({'EXIT', Pid, _});
+    ?assertReceive({'EXIT', Pid, _}, 1_000, #{pid => Pid});
 assert_client_exit(Pid, v5, kicked) ->
     ?assertReceive({'EXIT', Pid, {disconnected, ?RC_ADMINISTRATIVE_ACTION, _}});
 assert_client_exit(Pid, _, killed) ->

apps/emqx/test/emqx_topic_SUITE.erl

@@ -28,6 +28,7 @@
     [
         wildcard/1,
         match/2,
+        intersection/2,
         validate/1,
         prepend/2,
         join/1,
@@ -128,6 +129,63 @@ t_match_perf(_) ->
     true = match(Name, Filter),
     ok = bench('match/2', fun emqx_topic:match/2, [Name, Filter]).
 
+t_intersect(_) ->
+    <<"t/global/1/+">> = intersection(<<"t/global/#">>, <<"t/+/1/+">>),
+    <<"t/global/#">> = intersection(<<"t/global/#">>, <<"#">>),
+    <<"t/global/#">> = intersection(<<"t/global/#">>, <<"t/global/#">>),
+    <<"1/2/3/4/5">> = intersection(<<"1/+/3/+/5/#">>, <<"+/2/+/4/+">>),
+    <<"t/local/1">> = intersection(<<"t/local/1/#">>, <<"t/local/+">>),
+    false = intersection(<<"t/global/#">>, <<"t/local/+">>),
+    false = intersection(<<"t/local/1/+">>, <<"t/local/+">>).
+
+t_intersect_topic_wildcard(_) ->
+    <<"t/test/1">> = intersection(<<"t/test/#">>, <<"t/test/1">>),
+    <<"t/test/1/1">> = intersection(<<"t/test/1/1">>, <<"t/test/#">>),
+    false = intersection(<<"t/test/1/1">>, <<"t/test/+">>),
+    <<"t/test/1/1">> = intersection(<<"t/test/1/1">>, <<"t/test/1/1">>),
+    false = intersection(<<"t/test/1">>, <<"t/test/2">>),
+    false = intersection(<<"t/test/1">>, <<"t/test/1/2">>).
+
+t_intersect_commutes(_) ->
+    ?assertEqual(
+        intersection(<<"t/+/1/+">>, <<"t/global/#">>),
+        intersection(<<"t/global/#">>, <<"t/+/1/+">>)
+    ),
+    ?assertEqual(
+        intersection(<<"#">>, <<"t/global/#">>),
+        intersection(<<"t/global/#">>, <<"#">>)
+    ),
+    ?assertEqual(
+        intersection(<<"+/2/+/4/+">>, <<"1/+/3/+/5/#">>),
+        intersection(<<"1/+/3/+/5/#">>, <<"+/2/+/4/+">>)
+    ),
+    ?assertEqual(
+        intersection(<<"t/local/+">>, <<"t/local/1/#">>),
+        intersection(<<"t/local/1/#">>, <<"t/local/+">>)
+    ),
+    ?assertEqual(
+        intersection(<<"t/local/+">>, <<"t/global/#">>),
+        intersection(<<"t/global/#">>, <<"t/local/+">>)
+    ),
+    ?assertEqual(
+        intersection(<<"t/local/+">>, <<"t/local/1/+">>),
+        intersection(<<"t/local/1/+">>, <<"t/local/+">>)
+    ),
+    ?assertEqual(
+        intersection(<<"t/test/#">>, <<"t/test/1/1">>),
+        intersection(<<"t/test/1/1">>, <<"t/test/#">>)
+    ),
+    ?assertEqual(
+        intersection(<<"t/test/+">>, <<"t/test/1/1">>),
+        intersection(<<"t/test/1/1">>, <<"t/test/+">>)
+    ).
+
+t_sys_intersect(_) ->
+    <<"$SYS/broker/+">> = intersection(<<"$SYS/broker/#">>, <<"$SYS/+/+">>),
+    <<"$SYS/broker">> = intersection(<<"$SYS/broker">>, <<"$SYS/+">>),
+    false = intersection(<<"$SYS/broker">>, <<"+/+">>),
+    false = intersection(<<"$SYS/broker">>, <<"#">>).
+
 t_validate(_) ->
     true = validate(<<"a/+/#">>),
     true = validate(<<"a/b/c/d">>),

apps/emqx/test/emqx_ws_connection_SUITE.erl

@@ -540,7 +540,7 @@ t_parse_incoming_frame_error(_) ->
 
 t_handle_incomming_frame_error(_) ->
     FrameError = {frame_error, bad_qos},
-    Serialize = emqx_frame:serialize_fun(#{version => 5, max_size => 16#FFFF}),
+    Serialize = emqx_frame:serialize_fun(#{version => 5, max_size => 16#FFFF, strict_mode => false}),
     {[{close, bad_qos}], _St} = ?ws_conn:handle_incoming(FrameError, st(#{serialize => Serialize})).
 % ?assertEqual(<<224,2,129,0>>, iolist_to_binary(IoData)).
 
@@ -556,7 +556,7 @@ t_handle_outgoing(_) ->
 t_run_gc(_) ->
     GcSt = emqx_gc:init(#{count => 10, bytes => 100}),
     WsSt = st(#{gc_state => GcSt}),
-    ?ws_conn:run_gc(#{cnt => 100, oct => 10000}, WsSt).
+    ?ws_conn:run_gc(100, 10000, WsSt).
 
 t_enqueue(_) ->
     Packet = ?PUBLISH_PACKET(?QOS_0),

apps/emqx_auth/etc/acl.conf

@@ -4,7 +4,7 @@
 
 {allow, {ipaddr, "127.0.0.1"}, all, ["$SYS/#", "#"]}.
 
-{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}.
+{deny, all, subscribe, ["$SYS/#", {eq, "#"}, {eq, "+/#"}]}.
 
 {allow, all}.
 %% NOTE! when deploy in production:

apps/emqx_auth/include/emqx_authn.hrl

@@ -21,8 +21,6 @@
 
 -define(AUTHN, emqx_authn_chains).
 
--define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}").
-
 %% has to be the same as the root field name defined in emqx_schema
 -define(CONF_NS, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME).
 -define(CONF_NS_ATOM, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
@@ -32,4 +30,17 @@
 
 -define(AUTHN_RESOURCE_GROUP, <<"authn">>).
 
+%% VAR_NS_CLIENT_ATTRS is added here because it can be initialized before authn.
+%% NOTE: authn return may add more to (or even overwrite) client_attrs.
+-define(AUTHN_DEFAULT_ALLOWED_VARS, [
+    ?VAR_USERNAME,
+    ?VAR_CLIENTID,
+    ?VAR_PASSWORD,
+    ?VAR_PEERHOST,
+    ?VAR_CERT_SUBJECT,
+    ?VAR_CERT_CN_NAME,
+    ?VAR_CERT_PEM,
+    ?VAR_NS_CLIENT_ATTRS
+]).
+
 -endif.

apps/emqx_auth/include/emqx_authz.hrl

@@ -38,8 +38,6 @@
 -define(ROOT_KEY, [authorization]).
 -define(CONF_KEY_PATH, [authorization, sources]).
 
--define(RE_PLACEHOLDER, "\\$\\{[a-z0-9_]+\\}").
-
 %% has to be the same as the root field name defined in emqx_schema
 -define(CONF_NS, ?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME).
 -define(CONF_NS_ATOM, ?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_ATOM).

apps/emqx_auth/mix.exs

@@ -28,6 +28,7 @@ defmodule EMQXAuth.MixProject do
 
   def deps() do
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx, in_umbrella: true},
       {:emqx_utils, in_umbrella: true}
     ]