Merge remote-tracking branch 'origin/master' into 0221-sync-release-55

.ci/docker-compose-file/.env

@@ -10,7 +10,7 @@ CASSANDRA_TAG=3.11
 MINIO_TAG=RELEASE.2023-03-20T20-16-18Z
 OPENTS_TAG=9aa7f88
 KINESIS_TAG=2.1
-HSTREAMDB_TAG=v0.16.1
+HSTREAMDB_TAG=v0.19.3
 HSTREAMDB_ZK_TAG=3.8.1
 
 MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server

.ci/docker-compose-file/docker-compose-rabbitmq.yaml

@@ -9,10 +9,12 @@ services:
     expose:
       - "15672"
       - "5672"
+      - "5671"
     # We don't want to take ports from the host
     #ports:
     #   - "15672:15672"
     #   - "5672:5672"
+    #   - "5671:5671"
     volumes:
       - ./certs/ca.crt:/opt/certs/ca.crt
       - ./certs/server.crt:/opt/certs/server.crt

.ci/docker-compose-file/docker-compose-toxiproxy.yaml

@@ -39,6 +39,10 @@ services:
       - 19042:9042
       # Cassandra TLS
       - 19142:9142
+      # Cassandra No Auth
+      - 19043:9043
+      # Cassandra TLS No Auth
+      - 19143:9143
       # S3
       - 19000:19000
       # S3 TLS

.ci/docker-compose-file/toxiproxy.json

@@ -96,6 +96,18 @@
     "upstream": "cassandra:9142",
     "enabled": true
   },
+  {
+    "name": "cassa_no_auth_tcp",
+    "listen": "0.0.0.0:9043",
+    "upstream": "cassandra_noauth:9042",
+    "enabled": true
+  },
+  {
+    "name": "cassa_no_auth_tls",
+    "listen": "0.0.0.0:9143",
+    "upstream": "cassandra_noauth:9142",
+    "enabled": true
+  },
   {
     "name": "sqlserver",
     "listen": "0.0.0.0:1433",

.github/actions/package-macos/action.yaml

@@ -51,7 +51,7 @@ runs:
             echo "SELF_HOSTED=false" >> $GITHUB_OUTPUT
             ;;
         esac
-    - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+    - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
       id: cache
       if: steps.prepare.outputs.SELF_HOSTED != 'true'
       with:

.github/actions/prepare-jmeter/action.yaml

@@ -8,7 +8,7 @@ inputs:
 runs:
   using: composite
   steps:
-    - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+    - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
       with:
         name: emqx-docker
         path: /tmp
@@ -31,7 +31,7 @@ runs:
         architecture: x64 # (x64 or x86) - defaults to x64
         # https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md
         distribution: 'zulu'
-    - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+    - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
       with:
         name: apache-jmeter.tgz
     - name: install jmeter

.github/workflows/_pr_entrypoint.yaml

@@ -144,11 +144,11 @@ jobs:
           echo "PROFILE=${PROFILE}" | tee -a .env
           echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env
           zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip .
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: ${{ matrix.profile }}
           path: ${{ matrix.profile }}.zip
-          retention-days: 1
+          retention-days: 7
 
   run_emqx_app_tests:
     needs:

.github/workflows/_push-entrypoint.yaml

@@ -28,7 +28,6 @@ jobs:
       profile: ${{ steps.parse-git-ref.outputs.profile }}
       release: ${{ steps.parse-git-ref.outputs.release }}
       latest: ${{ steps.parse-git-ref.outputs.latest }}
-      version: ${{ steps.parse-git-ref.outputs.version }}
       ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
       ct-host: ${{ steps.matrix.outputs.ct-host }}
       ct-docker: ${{ steps.matrix.outputs.ct-docker }}
@@ -46,18 +45,16 @@ jobs:
         shell: bash
         run: |
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
-      - name: Detect emqx profile and version
+      - name: Detect emqx profile
         id: parse-git-ref
         run: |
           JSON="$(./scripts/parse-git-ref.sh $GITHUB_REF)"
           PROFILE=$(echo "$JSON" | jq -cr '.profile')
           RELEASE=$(echo "$JSON" | jq -cr '.release')
           LATEST=$(echo "$JSON"  | jq -cr '.latest')
-          VERSION="$(./pkg-vsn.sh "$PROFILE")"
           echo "profile=$PROFILE" | tee -a $GITHUB_OUTPUT
           echo "release=$RELEASE" | tee -a $GITHUB_OUTPUT
           echo "latest=$LATEST"   | tee -a $GITHUB_OUTPUT
-          echo "version=$VERSION" | tee -a $GITHUB_OUTPUT
       - name: Build matrix
         id: matrix
         run: |
@@ -91,7 +88,7 @@ jobs:
     uses: ./.github/workflows/build_packages.yaml
     with:
       profile: ${{ needs.prepare.outputs.profile }}
-      publish: ${{ needs.prepare.outputs.release }}
+      publish: true
       otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
       elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
       builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
@@ -104,8 +101,7 @@ jobs:
     uses: ./.github/workflows/build_and_push_docker_images.yaml
     with:
       profile: ${{ needs.prepare.outputs.profile }}
-      version: ${{ needs.prepare.outputs.version }}
+      publish: true
-      publish: ${{ needs.prepare.outputs.release }}
       latest: ${{ needs.prepare.outputs.latest }}
       # TODO: revert this back to needs.prepare.outputs.otp_vsn when OTP 26 bug is fixed
       otp_vsn: 25.3.2-2
@@ -153,7 +149,7 @@ jobs:
           echo "PROFILE=${PROFILE}" | tee -a .env
           echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env
           zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip .
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: ${{ matrix.profile }}
           path: ${{ matrix.profile }}.zip

.github/workflows/build_and_push_docker_images.yaml

@@ -10,15 +10,12 @@ on:
       profile:
         required: true
         type: string
-      version:
-        required: true
-        type: string
       latest:
         required: true
         type: string
       publish:
         required: true
-        type: string
+        type: boolean
       otp_vsn:
         required: true
         type: string
@@ -45,8 +42,6 @@ on:
         required: false
         type: string
         default: 'emqx'
-      version:
-        required: true
       latest:
         required: false
         type: boolean
@@ -72,8 +67,11 @@ permissions:
   contents: read
 
 jobs:
-  docker:
+  build:
-    runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
+    runs-on: ${{ github.repository_owner == 'emqx' && fromJSON(format('["self-hosted","ephemeral","linux","{0}"]', matrix.arch)) || 'ubuntu-22.04' }}
+    container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ inputs.otp_vsn }}-debian11"
+    outputs:
+      PKG_VSN: ${{ steps.build.outputs.PKG_VSN }}
 
     strategy:
       fail-fast: false
@@ -81,34 +79,77 @@ jobs:
         profile:
           - ${{ inputs.profile }}
           - ${{ inputs.profile }}-elixir
-        registry:
+        arch:
-          - 'docker.io'
+          - x64
-          - 'public.ecr.aws'
+          - arm64
-        exclude:
-          - profile: emqx-enterprise
-            registry: 'public.ecr.aws'
-          - profile: emqx-enterprise-elixir
-            registry: 'public.ecr.aws'
 
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           ref: ${{ github.event.inputs.ref }}
-        fetch-depth: 0
+      - run: git config --global --add safe.directory "$PWD"
+      - name: build release tarball
+        id: build
+        run: |
+          make ${{ matrix.profile }}-tgz
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: "${{ matrix.profile }}-${{ matrix.arch }}.tar.gz"
+          path: "_packages/emqx*/emqx-*.tar.gz"
+          retention-days: 7
+          overwrite: true
+          if-no-files-found: error
+
+  docker:
+    runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
+    needs:
+      - build
+    defaults:
+      run:
+        shell: bash
+
+    strategy:
+      fail-fast: false
+      matrix:
+        profile:
+          - ${{ inputs.profile }}
+          - ${{ inputs.profile }}-elixir
+
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ github.event.inputs.ref }}
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
+        with:
+          pattern: "${{ matrix.profile }}-*.tar.gz"
+          path: _packages
+          merge-multiple: true
+
+      - name: Move artifacts to root directory
+        env:
+          PROFILE: ${{ inputs.profile }}
+        run: |
+          ls -lR _packages/$PROFILE
+          mv _packages/$PROFILE/*.tar.gz ./
+      - name: Enable containerd image store on Docker Engine
+        run: |
+          echo "$(jq '. += {"features": {"containerd-snapshotter": true}}' /etc/docker/daemon.json)" > daemon.json
+          sudo mv daemon.json /etc/docker/daemon.json
+          sudo systemctl restart docker
 
       - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
       - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
 
       - name: Login to hub.docker.com
         uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
-      if: matrix.registry == 'docker.io'
+        if: inputs.publish || github.repository_owner != 'emqx'
         with:
           username: ${{ secrets.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_HUB_TOKEN }}
 
       - name: Login to AWS ECR
         uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
-      if: matrix.registry == 'public.ecr.aws'
+        if: inputs.publish || github.repository_owner != 'emqx'
         with:
           registry: public.ecr.aws
           username: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -118,17 +159,50 @@ jobs:
       - name: Build docker image
         env:
           PROFILE: ${{ matrix.profile }}
-        DOCKER_REGISTRY: ${{ matrix.registry }}
+          DOCKER_REGISTRY: 'docker.io,public.ecr.aws'
           DOCKER_ORG: ${{ github.repository_owner }}
           DOCKER_LATEST: ${{ inputs.latest }}
-        DOCKER_PUSH: ${{ inputs.publish == 'true' || inputs.publish || github.repository_owner != 'emqx' }}
+          DOCKER_PUSH: false
           DOCKER_BUILD_NOCACHE: true
           DOCKER_PLATFORMS: linux/amd64,linux/arm64
-        EMQX_RUNNER: 'debian:11-slim'
+          DOCKER_LOAD: true
+          EMQX_RUNNER: 'public.ecr.aws/debian/debian:11-slim@sha256:22cfb3c06a7dd5e18d86123a73405664475b9d9fa209cbedcf4c50a25649cc74'
           EMQX_DOCKERFILE: 'deploy/docker/Dockerfile'
-        PKG_VSN: ${{ inputs.version }}
+          PKG_VSN: ${{ needs.build.outputs.PKG_VSN }}
           EMQX_BUILDER_VERSION: ${{ inputs.builder_vsn }}
           EMQX_BUILDER_OTP: ${{ inputs.otp_vsn }}
           EMQX_BUILDER_ELIXIR: ${{ inputs.elixir_vsn }}
+          EMQX_SOURCE_TYPE: tgz
         run: |
           ./build ${PROFILE} docker
+          cat .emqx_docker_image_tags
+          echo "_EMQX_DOCKER_IMAGE_TAG=$(head -n 1 .emqx_docker_image_tags)" >> $GITHUB_ENV
+
+      - name: smoke test
+        timeout-minutes: 1
+        run: |
+          for tag in $(cat .emqx_docker_image_tags); do
+            CID=$(docker run -d -P $tag)
+            HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
+            ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
+            docker rm -f $CID
+          done
+      - name: dashboard tests
+        working-directory: ./scripts/ui-tests
+        timeout-minutes: 5
+        run: |
+          set -eu
+          docker compose up --abort-on-container-exit --exit-code-from selenium
+          docker compose rm -fsv
+      - name: test node_dump
+        run: |
+          CID=$(docker run -d -P $_EMQX_DOCKER_IMAGE_TAG)
+          docker exec -t -u root -w /root $CID bash -c 'apt-get -y update && apt-get -y install net-tools'
+          docker exec -t -u root $CID node_dump
+          docker rm -f $CID
+      - name: push images
+        if: inputs.publish || github.repository_owner != 'emqx'
+        run: |
+          for tag in $(cat .emqx_docker_image_tags); do
+            docker push $tag
+          done

.github/workflows/build_docker_for_test.yaml

@@ -47,17 +47,17 @@ jobs:
         id: build
         run: |
           make ${EMQX_NAME}-docker
-          echo "EMQX_IMAGE_TAG=$(cat .docker_image_tag)" >> $GITHUB_ENV
+          echo "_EMQX_DOCKER_IMAGE_TAG=$(head -n 1 .emqx_docker_image_tags)" >> $GITHUB_ENV
       - name: smoke test
         run: |
-          CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
+          CID=$(docker run -d --rm -P $_EMQX_DOCKER_IMAGE_TAG)
           HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
           ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
           docker stop $CID
       - name: export docker image
         run: |
-          docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz
+          docker save $_EMQX_DOCKER_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: "${{ env.EMQX_NAME }}-docker"
           path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz"

.github/workflows/build_packages.yaml

@@ -12,7 +12,7 @@ on:
         type: string
       publish:
         required: true
-        type: string
+        type: boolean
       otp_vsn:
         required: true
         type: string
@@ -74,12 +74,12 @@ jobs:
       matrix:
         profile:
           - ${{ inputs.profile }}
-        otp:
-          - ${{ inputs.otp_vsn }}
         os:
           - macos-12
           - macos-12-arm64
           - macos-13
+        otp:
+          - ${{ inputs.otp_vsn }}
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -95,30 +95,20 @@ jobs:
         apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
         apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
         apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-    - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: success()
       with:
-        name: ${{ matrix.profile }}
+        name: ${{ matrix.profile }}-${{ matrix.os }}-${{ matrix.otp }}
         path: _packages/${{ matrix.profile }}/
         retention-days: 7
 
   linux:
-    runs-on: [self-hosted, ephemeral, linux, "${{ matrix.arch }}"]
+    runs-on: [self-hosted, ephemeral, linux, "${{ matrix.arch == 'arm64' && 'arm64' || 'x64' }}"]
-    # always run in builder container because the host might have the wrong OTP version etc.
-    # otherwise buildx.sh does not run docker if arch and os matches the target arch and os.
-    container:
-      image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
-
     strategy:
       fail-fast: false
       matrix:
         profile:
           - ${{ inputs.profile }}
-        otp:
-          - ${{ inputs.otp_vsn }}
-        arch:
-          - x64
-          - arm64
         os:
           - ubuntu22.04
           - ubuntu20.04
@@ -131,70 +121,53 @@ jobs:
           - el7
           - amzn2
           - amzn2023
+        arch:
+          - amd64
+          - arm64
+        with_elixir:
+          - 'no'
+        otp:
+          - ${{ inputs.otp_vsn }}
         builder:
           - ${{ inputs.builder_vsn }}
         elixir:
           - ${{ inputs.elixir_vsn }}
-        with_elixir:
-          - 'no'
         include:
           - profile: ${{ inputs.profile }}
-            otp: ${{ inputs.otp_vsn }}
-            arch: x64
             os: ubuntu22.04
+            arch: amd64
+            with_elixir: 'yes'
+            otp: ${{ inputs.otp_vsn }}
             builder: ${{ inputs.builder_vsn }}
             elixir: ${{ inputs.elixir_vsn }}
-            with_elixir: 'yes'
 
     defaults:
       run:
         shell: bash
 
     steps:
-    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         ref: ${{ github.event.inputs.ref }}
         fetch-depth: 0
-
-    - name: fix workdir
-      run: |
-        set -eu
-        git config --global --add safe.directory "$GITHUB_WORKSPACE"
-        # Align path for CMake caches
-        if [ ! "$PWD" = "/emqx" ]; then
-          ln -s $PWD /emqx
-          cd /emqx
-        fi
-        echo "pwd is $PWD"
-
     - name: build emqx packages
       env:
         PROFILE: ${{ matrix.profile }}
+        ARCH: ${{ matrix.arch }}
+        OS: ${{ matrix.os }}
         IS_ELIXIR: ${{ matrix.with_elixir }}
-        ACLOCAL_PATH: "/usr/share/aclocal:/usr/local/share/aclocal"
+        BUILDER: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+        BUILDER_SYSTEM: force_docker
       run: |
-        set -eu
+        ./scripts/buildx.sh \
-        if [ "${IS_ELIXIR:-}" == 'yes' ]; then
+          --profile $PROFILE \
-          make "${PROFILE}-elixir-tgz"
+          --arch $ARCH \
-        else
+          --builder $BUILDER \
-          make "${PROFILE}-tgz"
+          --elixir $IS_ELIXIR \
-          make "${PROFILE}-pkg"
+          --pkgtype pkg
-        fi
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
-    - name: test emqx packages
-      env:
-        PROFILE: ${{ matrix.profile }}
-        IS_ELIXIR: ${{ matrix.with_elixir }}
-      run: |
-        set -eu
-        if [ "${IS_ELIXIR:-}" == 'yes' ]; then
-          ./scripts/pkg-tests.sh "${PROFILE}-elixir-tgz"
-        else
-          ./scripts/pkg-tests.sh "${PROFILE}-tgz"
-          ./scripts/pkg-tests.sh "${PROFILE}-pkg"
-        fi
-    - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
       with:
-        name: ${{ matrix.profile }}
+        name: ${{ matrix.profile }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.with_elixir == 'yes' && '-elixir' || '' }}-${{ matrix.builder }}-${{ matrix.otp }}-${{ matrix.elixir }}
         path: _packages/${{ matrix.profile }}/
         retention-days: 7
 
@@ -203,17 +176,18 @@ jobs:
     needs:
       - mac
       - linux
-    if: inputs.publish == 'true' || inputs.publish
+    if: inputs.publish
     strategy:
       fail-fast: false
       matrix:
         profile:
           - ${{ inputs.profile }}
     steps:
-    - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+    - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
       with:
-        name: ${{ matrix.profile }}
+        pattern: "${{ matrix.profile }}-*"
         path: packages/${{ matrix.profile }}
+        merge-multiple: true
     - name: install dos2unix
       run: sudo apt-get update -y && sudo apt install -y dos2unix
     - name: get packages
@@ -226,7 +200,7 @@ jobs:
           echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
         done
         cd -
-    - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+    - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/build_packages_cron.yaml

@@ -66,14 +66,14 @@ jobs:
           set -eu
           ./scripts/pkg-tests.sh "${PROFILE}-tgz"
           ./scripts/pkg-tests.sh "${PROFILE}-pkg"
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: success()
         with:
           name: ${{ matrix.profile[0] }}-${{ matrix.os }}
           path: _packages/${{ matrix.profile[0] }}/
           retention-days: 7
       - name: Send notification to Slack
-        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
@@ -111,14 +111,14 @@ jobs:
           apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
           apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
           apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: success()
         with:
           name: ${{ matrix.profile }}-${{ matrix.os }}
           path: _packages/${{ matrix.profile }}/
           retention-days: 7
       - name: Send notification to Slack
-        uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
         if: failure()
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/build_slim_packages.yaml

@@ -88,13 +88,13 @@ jobs:
       run: |
         make ${EMQX_NAME}-elixir-pkg
         ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       with:
         name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}-${{ matrix.profile[3] }}-${{ matrix.profile[4] }}"
         path: _packages/${{ matrix.profile[0] }}/*
         retention-days: 7
         compression-level: 0
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       with:
         name: "${{ matrix.profile[0] }}-schema-dump-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}-${{ matrix.profile[3] }}-${{ matrix.profile[4] }}"
         path: |
@@ -128,7 +128,7 @@ jobs:
         apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
         apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
         apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       with:
         name: ${{ matrix.os }}
         path: _packages/**/*

.github/workflows/check_deps_integrity.yaml

@@ -36,7 +36,7 @@ jobs:
           MIX_ENV: emqx-enterprise
           PROFILE: emqx-enterprise
       - name: Upload produced lock files
-        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: failure()
         with:
           name: produced_lock_files

.github/workflows/performance_test.yaml

@@ -52,7 +52,7 @@ jobs:
       id: package_file
       run: |
         echo "PACKAGE_FILE=$(find _packages/emqx -name 'emqx-*.deb' | head -n 1 | xargs basename)" >> $GITHUB_OUTPUT
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       with:
         name: emqx-ubuntu20.04
         path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }}
@@ -66,7 +66,7 @@ jobs:
 
     steps:
     - name: Configure AWS Credentials
-      uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
@@ -77,7 +77,7 @@ jobs:
         repository: emqx/tf-emqx-performance-test
         path: tf-emqx-performance-test
         ref: v0.2.3
-    - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+    - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
       with:
         name: emqx-ubuntu20.04
         path: tf-emqx-performance-test/
@@ -105,7 +105,7 @@ jobs:
         terraform destroy -auto-approve
         aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
     - name: Send notification to Slack
-      uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+      uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
       with:
         payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
     - name: terraform destroy
@@ -113,13 +113,13 @@ jobs:
       working-directory: ./tf-emqx-performance-test
       run: |
         terraform destroy -auto-approve
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: success()
       with:
         name: metrics
         path: |
           "./tf-emqx-performance-test/*.tar.gz"
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: failure()
       with:
         name: terraform
@@ -137,7 +137,7 @@ jobs:
 
     steps:
     - name: Configure AWS Credentials
-      uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
@@ -148,7 +148,7 @@ jobs:
         repository: emqx/tf-emqx-performance-test
         path: tf-emqx-performance-test
         ref: v0.2.3
-    - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+    - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
       with:
         name: emqx-ubuntu20.04
         path: tf-emqx-performance-test/
@@ -176,7 +176,7 @@ jobs:
         terraform destroy -auto-approve
         aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
     - name: Send notification to Slack
-      uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+      uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
       with:
         payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
     - name: terraform destroy
@@ -184,13 +184,13 @@ jobs:
       working-directory: ./tf-emqx-performance-test
       run: |
         terraform destroy -auto-approve
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: success()
       with:
         name: metrics
         path: |
           "./tf-emqx-performance-test/*.tar.gz"
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: failure()
       with:
         name: terraform
@@ -209,7 +209,7 @@ jobs:
 
     steps:
     - name: Configure AWS Credentials
-      uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
@@ -220,7 +220,7 @@ jobs:
         repository: emqx/tf-emqx-performance-test
         path: tf-emqx-performance-test
         ref: v0.2.3
-    - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+    - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
       with:
         name: emqx-ubuntu20.04
         path: tf-emqx-performance-test/
@@ -249,7 +249,7 @@ jobs:
         terraform destroy -auto-approve
         aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
     - name: Send notification to Slack
-      uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+      uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
       with:
         payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
     - name: terraform destroy
@@ -257,13 +257,13 @@ jobs:
       working-directory: ./tf-emqx-performance-test
       run: |
         terraform destroy -auto-approve
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: success()
       with:
         name: metrics
         path: |
           "./tf-emqx-performance-test/*.tar.gz"
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: failure()
       with:
         name: terraform
@@ -283,7 +283,7 @@ jobs:
 
     steps:
     - name: Configure AWS Credentials
-      uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
       with:
         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
         aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
@@ -294,7 +294,7 @@ jobs:
         repository: emqx/tf-emqx-performance-test
         path: tf-emqx-performance-test
         ref: v0.2.3
-    - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+    - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
       with:
         name: emqx-ubuntu20.04
         path: tf-emqx-performance-test/
@@ -322,7 +322,7 @@ jobs:
         terraform destroy -auto-approve
         aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
     - name: Send notification to Slack
-      uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
+      uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
       with:
         payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
     - name: terraform destroy
@@ -330,13 +330,13 @@ jobs:
       working-directory: ./tf-emqx-performance-test
       run: |
         terraform destroy -auto-approve
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: success()
       with:
         name: metrics
         path: |
           "./tf-emqx-performance-test/*.tar.gz"
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: failure()
       with:
         name: terraform

.github/workflows/release.yaml

@@ -31,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+      - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/run_conf_tests.yaml

@@ -25,7 +25,7 @@ jobs:
           - emqx
           - emqx-enterprise
     steps:
-      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ matrix.profile }}
       - name: extract artifact
@@ -40,7 +40,7 @@ jobs:
         if: failure()
         run: |
           cat _build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.*
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: failure()
         with:
           name: conftest-logs-${{ matrix.profile }}

.github/workflows/run_docker_tests.yaml

@@ -37,14 +37,14 @@ jobs:
 
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ env.EMQX_NAME }}-docker
           path: /tmp
       - name: load docker image
         run: |
-          EMQX_IMAGE_TAG=$(docker load < /tmp/${EMQX_NAME}-docker-${PKG_VSN}.tar.gz 2>/dev/null | sed 's/Loaded image: //g')
+          _EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/${EMQX_NAME}-docker-${PKG_VSN}.tar.gz 2>/dev/null | sed 's/Loaded image: //g')
-          echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV
+          echo "_EMQX_DOCKER_IMAGE_TAG=$_EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
       - name: dashboard tests
         working-directory: ./scripts/ui-tests
         run: |
@@ -52,7 +52,7 @@ jobs:
           docker compose up --abort-on-container-exit --exit-code-from selenium
       - name: test two nodes cluster with proto_dist=inet_tls in docker
         run: |
-          ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
+          ./scripts/test/start-two-nodes-in-docker.sh -P $_EMQX_DOCKER_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
           HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy)
           ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
           ./scripts/test/start-two-nodes-in-docker.sh -c
@@ -84,7 +84,7 @@ jobs:
           - rlog
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ env.EMQX_NAME }}-docker
           path: /tmp
@@ -113,4 +113,4 @@ jobs:
       - name: test node_dump
         run: |
           docker exec -t -u root node1.emqx.io bash -c 'apt-get -y update && apt-get -y install net-tools'
-          docker exec node1.emqx.io node_dump
+          docker exec -t -u root node1.emqx.io node_dump

.github/workflows/run_emqx_app_tests.yaml

@@ -58,7 +58,7 @@ jobs:
         ./rebar3 eunit -v --name 'eunit@127.0.0.1'
         ./rebar3 as standalone_test ct --name 'test@127.0.0.1' -v --readable=true
         ./rebar3 proper -d test/props
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: failure()
       with:
         name: logs-emqx-app-tests

.github/workflows/run_helm_tests.yaml

@@ -45,7 +45,7 @@ jobs:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
         path: source
-    - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+    - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
       with:
         name: "${{ env.EMQX_NAME }}-docker"
         path: /tmp

.github/workflows/run_jmeter_tests.yaml

@@ -16,7 +16,7 @@ jobs:
     steps:
     - name: Cache Jmeter
       id: cache-jmeter
-      uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+      uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
       with:
         path: /tmp/apache-jmeter.tgz
         key: apache-jmeter-5.4.3.tgz
@@ -35,7 +35,7 @@ jobs:
         else
           wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz $ARCHIVE_URL
         fi
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       with:
         name: apache-jmeter.tgz
         path: /tmp/apache-jmeter.tgz
@@ -86,7 +86,7 @@ jobs:
           echo "check logs failed"
           exit 1
         fi
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: always()
       with:
         name: jmeter_logs-advanced_feat-${{ matrix.scripts_type }}
@@ -153,7 +153,7 @@ jobs:
       if: failure()
       run: |
         docker compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: always()
       with:
         name: jmeter_logs-pgsql_authn_authz-${{ matrix.scripts_type }}_${{ matrix.pgsql_tag }}
@@ -213,7 +213,7 @@ jobs:
           echo "check logs failed"
           exit 1
         fi
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: always()
       with:
         name: jmeter_logs-mysql_authn_authz-${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}
@@ -265,7 +265,7 @@ jobs:
           echo "check logs failed"
           exit 1
         fi
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: always()
       with:
         name: jmeter_logs-JWT_authn-${{ matrix.scripts_type }}
@@ -309,7 +309,7 @@ jobs:
           echo "check logs failed"
           exit 1
         fi
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       if: always()
       with:
         name: jmeter_logs-built_in_database_authn_authz-${{ matrix.scripts_type }}

.github/workflows/run_relup_tests.yaml

@@ -25,7 +25,7 @@ jobs:
       run:
         shell: bash
     steps:
-    - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+    - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
       with:
         name: emqx-enterprise
     - name: extract artifact
@@ -45,7 +45,7 @@ jobs:
       run: |
         export PROFILE='emqx-enterprise'
         make emqx-enterprise-tgz
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       name: Upload built emqx and test scenario
       with:
         name: relup_tests_emqx_built
@@ -72,7 +72,7 @@ jobs:
       run:
         shell: bash
     steps:
-    - uses: erlef/setup-beam@a34c98fd51e370b4d4981854aba1eb817ce4e483 # v1.17.0
+    - uses: erlef/setup-beam@8b9cac4c04dbcd7bf8fd673e16f988225d89b09b # v1.17.2
       with:
         otp-version: 26.2.1
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -88,7 +88,7 @@ jobs:
         ./configure
         make
         echo "$(pwd)/bin" >> $GITHUB_PATH
-    - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+    - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
       name: Download built emqx and test scenario
       with:
         name: relup_tests_emqx_built
@@ -111,7 +111,7 @@ jobs:
           docker logs node2.emqx.io | tee lux_logs/emqx2.log
           exit 1
         fi
-    - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       name: Save debug data
       if: failure()
       with:

.github/workflows/run_test_cases.yaml

@@ -41,7 +41,7 @@ jobs:
     container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
 
     steps:
-      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ matrix.profile }}
       - name: extract artifact
@@ -64,7 +64,7 @@ jobs:
           CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
         run: make proper
 
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: coverdata-${{ matrix.profile }}-${{ matrix.otp }}
           path: _build/test/cover
@@ -83,7 +83,7 @@ jobs:
         shell: bash
 
     steps:
-      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ matrix.profile }}
       - name: extract artifact
@@ -108,7 +108,7 @@ jobs:
           ENABLE_COVER_COMPILE: 1
           CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
         run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }}
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: coverdata-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
           path: _build/test/cover
@@ -116,7 +116,7 @@ jobs:
       - name: compress logs
         if: failure()
         run: tar -czf logs.tar.gz _build/test/logs
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: failure()
         with:
           name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
@@ -138,7 +138,7 @@ jobs:
         shell: bash
 
     steps:
-      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ matrix.profile }}
       - name: extract artifact
@@ -155,7 +155,7 @@ jobs:
           CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
         run: |
           make "${{ matrix.app }}-ct"
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: coverdata-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
           path: _build/test/cover
@@ -164,7 +164,7 @@ jobs:
       - name: compress logs
         if: failure()
         run: tar -czf logs.tar.gz _build/test/logs
-      - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: failure()
         with:
           name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
@@ -196,7 +196,7 @@ jobs:
         profile:
           - emqx-enterprise
     steps:
-      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ matrix.profile }}
       - name: extract artifact
@@ -204,7 +204,7 @@ jobs:
           unzip -o -q ${{ matrix.profile }}.zip
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
 
-      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         name: download coverdata
         with:
           pattern: coverdata-${{ matrix.profile }}-*

.github/workflows/scorecard.yaml

@@ -39,7 +39,7 @@ jobs:
           publish_results: true
 
       - name: "Upload artifact"
-        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: SARIF file
           path: results.sarif
@@ -47,6 +47,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@8e0b1c74b1d5a0077b04d064c76ee714d3da7637 # v2.22.1
+        uses: github/codeql-action/upload-sarif@7e187e1c529d80bac7b87a16e7a792427f65cf02 # v2.22.1
         with:
           sarif_file: results.sarif

.github/workflows/spellcheck.yaml

@@ -19,7 +19,7 @@ jobs:
         - emqx-enterprise
     runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
     steps:
-      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           pattern: "${{ matrix.profile }}-schema-dump-*-x64"
           merge-multiple: true

.github/workflows/static_checks.yaml

@@ -30,14 +30,14 @@ jobs:
         include: ${{ fromJson(inputs.ct-matrix) }}
     container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
     steps:
-      - uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
           name: ${{ matrix.profile }}
       - name: extract artifact
         run: |
           unzip -o -q ${{ matrix.profile }}.zip
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
-      - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+      - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
         with:
           path: "emqx_dialyzer_${{ matrix.otp }}_plt"
           key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*') }}

.github/workflows/upload-helm-charts.yaml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+      - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.gitignore

@@ -73,3 +73,4 @@ apps/emqx_conf/etc/emqx.conf.all.rendered*
 rebar-git-cache.tar
 # build docker image locally
 .docker_image_tag
+.git/

Makefile

@@ -21,7 +21,7 @@ endif
 # Dashboard version
 # from https://github.com/emqx/emqx-dashboard5
 export EMQX_DASHBOARD_VERSION ?= v1.7.0
-export EMQX_EE_DASHBOARD_VERSION ?= e1.5.0
+export EMQX_EE_DASHBOARD_VERSION ?= e1.5.1-s3-beta.1
 
 PROFILE ?= emqx
 REL_PROFILES := emqx emqx-enterprise
@@ -316,10 +316,9 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt))))
 .PHONY: fmt
 fmt: $(REBAR)
 	@$(SCRIPTS)/erlfmt -w 'apps/*/{src,include,priv,test,integration_test}/**/*.{erl,hrl,app.src,eterm}'
-	@$(SCRIPTS)/erlfmt -w '**/*.escript' --exclude-files '_build/**'
+	@$(SCRIPTS)/erlfmt -w 'apps/*/rebar.config' 'apps/emqx/rebar.config.script' '.ci/fvt_tests/http_server/rebar.config'
-	@$(SCRIPTS)/erlfmt -w '**/rebar.config' --exclude-files '_build/**'
+	@$(SCRIPTS)/erlfmt -w 'rebar.config' 'rebar.config.erl'
-	@$(SCRIPTS)/erlfmt -w 'rebar.config.erl'
+	@$(SCRIPTS)/erlfmt -w 'scripts/*.escript' 'bin/*.escript' 'bin/nodetool'
-	@$(SCRIPTS)/erlfmt -w 'bin/nodetool'
 	@mix format
 
 .PHONY: clean-test-cluster-config

README-CN.md

@@ -4,6 +4,7 @@
 [![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml)
 [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/emqx/emqx/badge)](https://securityscorecards.dev/viewer/?uri=github.com/emqx/emqx)
 [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
 [![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES)
 [![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)

README-RU.md

@@ -4,6 +4,7 @@
 [![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml)
 [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/emqx/emqx/badge)](https://securityscorecards.dev/viewer/?uri=github.com/emqx/emqx)
 [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
 [![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES)
 [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)

README.md

@@ -4,6 +4,7 @@
 [![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml)
 [![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/emqx/emqx/badge)](https://securityscorecards.dev/viewer/?uri=github.com/emqx/emqx)
 [![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
 [![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES)
 [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)

Windows.md

@@ -1,131 +0,0 @@
-# Build and run EMQX on Windows
-
-NOTE: The instructions and examples are based on Windows 10.
-
-## Build Environment
-
-### Visual studio for C/C++ compile and link
-
-EMQX includes Erlang NIF (Native Implemented Function) components, implemented
-in C/C++. To compile and link C/C++ libraries, the easiest way is perhaps to
-install Visual Studio.
-
-Visual Studio 2019 is used in our tests.
-If you are like me (@zmstone), do not know where to start,
-please follow this OTP guide:
-https://github.com/erlang/otp/blob/master/HOWTO/INSTALL-WIN32.md
-
-NOTE: To avoid surprises, you may need to add below two paths to `Path` environment variable
-and order them before other paths.
-
-```
-C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\bin\Hostx64\x64
-C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build
-```
-
-Depending on your visual studio version and OS, the paths may differ.
-The first path is for rebar3 port compiler to find `cl.exe` and `link.exe`
-The second path is for CMD to setup environment variables.
-
-### Erlang/OTP
-
-Install Erlang/OTP 24 from https://www.erlang.org/downloads
-You may need to edit the `Path` environment variable to allow running
-Erlang commands such as `erl` from powershell.
-
-To validate Erlang installation in CMD or powershell:
-
-* Start (or restart) CMD or powershell
-
-* Execute `erl` command to enter Erlang shell
-
-* Evaluate Erlang expression `halt().` to exit Erlang shell.
-
-e.g.
-
-```
-PS C:\Users\zmsto> erl
-Eshell V12.2.1  (abort with ^G)
-1> halt().
-```
-
-### bash
-
-All EMQX build/run scripts are either in `bash` or `escript`.
-`escript` is installed as a part of Erlang. To install a `bash`
-environment in Windows, there are quite a few options.
-
-Cygwin is what we tested with.
-
-* Add `cygwin\bin` dir to `Path` environment variable
-  To do so, search for Edit environment variable in control panel and
-  add `C:\tools\cygwin\bin` (depending on the location where it was installed)
-  to `Path` list.
-
-* Validate installation.
-  Start (restart) CMD or powershell console and execute `which bash`, it should
-  print out `/usr/bin/bash`
-
-NOTE: Make sure cygwin's bin dir is added before `C:\Windows\system32` in `Path`,
-otherwise the build scripts may end up using binaries from wsl instead of cygwin.
-
-### Other tools
-
-Some of the unix world tools are required to build EMQX.  Including:
-
-* git
-* curl
-* make
-* cmake
-* jq
-* zip / unzip
-
-We recommend using [scoop](https://scoop.sh/), or [Chocolatey](https://chocolatey.org/install) to install the tools.
-
-When using scoop:
-
-```
-scoop install git curl make cmake jq zip unzip
-```
-
-## Build EMQX source code
-
-* Clone the repo: `git clone https://github.com/emqx/emqx.git`
-
-* Start CMD console
-
-* Execute `vcvarsall.bat x86_amd64` to load environment variables
-
-* Change to emqx directory and execute `make`
-
-### Possible errors
-
-* `'cl.exe' is not recognized as an internal or external command`
-  This error is likely due to Visual Studio executables are not set in `Path` environment variable.
-  To fix it, either add path like `C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\bin\Hostx64\x64`
-  to `Paht`. Or make sure `vcvarsall.bat x86_amd64` is executed prior to the `make` command
-
-* `fatal error C1083: Cannot open include file: 'assert.h': No such file or directory`
-  If Visual Studio is installed correctly, this is likely `LIB` and `LIB_PATH` environment
-  variables are not set. Make sure `vcvarsall.bat x86_amd64` is executed prior to the `make` command
-
-* `link: extra operand 'some.obj'`
-  This is likely due to the usage of GNU `lnik.exe` but not the one from Visual Studio.
-  Execute `link.exe --version` to inspect which one is in use. The one installed from
-  Visual Studio should print out `Microsoft (R) Incremental Linker`.
-  To fix it, Visual Studio's bin paths should be ordered prior to Cygwin's (or similar installation's)
-  bin paths in `Path` environment variable.
-
-## Run EMQX
-
-To start EMQX broker.
-
-Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQX.
-
-Then execute `_build\emqx\rel\emqx>.\bin\emqx_ctl status` to check status.
-If everything works fine, it should print out
-
-```
-Node 'emqx@127.0.0.1' 4.3-beta.1 is started
-Application emqx 4.3.0 is running
-```

apps/emqx/include/emqx.hrl

@@ -88,10 +88,7 @@
 %%--------------------------------------------------------------------
 
 -record(banned, {
-    who ::
+    who :: emqx_types:banned_who(),
-        {clientid, binary()}
-        | {peerhost, inet:ip_address()}
-        | {username, binary()},
     by :: binary(),
     reason :: binary(),
     at :: integer(),

apps/emqx/include/emqx_cm.hrl

@@ -23,11 +23,20 @@
 -define(CHAN_INFO_TAB, emqx_channel_info).
 -define(CHAN_LIVE_TAB, emqx_channel_live).
 
-%% Mria/Mnesia Tables for channel management.
+%% Mria table for session registration.
 -define(CHAN_REG_TAB, emqx_channel_registry).
 
 -define(T_KICK, 5_000).
 -define(T_GET_INFO, 5_000).
 -define(T_TAKEOVER, 15_000).
 
+-define(CM_POOL, emqx_cm_pool).
+
+%% Registered sessions.
+-record(channel, {
+    chid :: emqx_types:clientid() | '_',
+    %% pid field is extended in 5.6.0 to support recording unregistration timestamp.
+    pid :: pid() | non_neg_integer() | '$1'
+}).
+
 -endif.

apps/emqx/include/http_api.hrl

@@ -17,6 +17,7 @@
 %% HTTP API Auth
 -define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD').
 -define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET').
+-define(API_KEY_NOT_ALLOW, 'API_KEY_NOT_ALLOW').
 -define(API_KEY_NOT_ALLOW_MSG, <<"This API Key don't have permission to access this resource">>).
 
 %% Bad Request

apps/emqx/include/logger.hrl

@@ -40,6 +40,21 @@
     end
 ).
 
+%% NOTE: do not forget to use atom for msg and add every used msg to
+%% the default value of `log.thorttling.msgs` list.
+-define(SLOG_THROTTLE(Level, Data),
+    ?SLOG_THROTTLE(Level, Data, #{})
+).
+
+-define(SLOG_THROTTLE(Level, Data, Meta),
+    case emqx_log_throttler:allow(Level, maps:get(msg, Data)) of
+        true ->
+            ?SLOG(Level, Data, Meta);
+        false ->
+            ok
+    end
+).
+
 -define(AUDIT_HANDLER, emqx_audit).
 -define(TRACE_FILTER, emqx_trace_filter).
 -define(OWN_KEYS, [level, filters, filter_default, handlers]).

apps/emqx/include/types.hrl

@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--type maybe(T) :: undefined | T.
+-type option(T) :: undefined | T.
 
 -type startlink_ret() :: {ok, pid()} | ignore | {error, term()}.
 

apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 -module(emqx_persistent_session_ds_SUITE).
 
@@ -18,6 +18,9 @@
 %% CT boilerplate
 %%------------------------------------------------------------------------------
 
+suite() ->
+    [{timetrap, {seconds, 60}}].
+
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
@@ -51,12 +54,12 @@ init_per_testcase(TestCase, Config) when
 init_per_testcase(t_session_gc = TestCase, Config) ->
     Opts = #{
         n => 3,
-        roles => [core, core, replicant],
+        roles => [core, core, core],
         extra_emqx_conf =>
             "\n session_persistence {"
             "\n   last_alive_update_interval = 500ms "
-            "\n   session_gc_interval = 2s "
+            "\n   session_gc_interval = 1s "
-            "\n   session_gc_batch_size = 1 "
+            "\n   session_gc_batch_size = 2 "
             "\n }"
     },
     Cluster = cluster(Opts),
@@ -88,7 +91,7 @@ end_per_testcase(_TestCase, _Config) ->
     ok.
 
 %%------------------------------------------------------------------------------
-%% Helper fns
+%% Helper functions
 %%------------------------------------------------------------------------------
 
 cluster(#{n := N} = Opts) ->
@@ -144,9 +147,10 @@ start_client(Opts0 = #{}) ->
         proto_ver => v5,
         properties => #{'Session-Expiry-Interval' => 300}
     },
-    Opts = maps:to_list(emqx_utils_maps:deep_merge(Defaults, Opts0)),
+    Opts = emqx_utils_maps:deep_merge(Defaults, Opts0),
-    ct:pal("starting client with opts:\n  ~p", [Opts]),
+    ?tp(notice, "starting client", Opts),
-    {ok, Client} = emqtt:start_link(Opts),
+    {ok, Client} = emqtt:start_link(maps:to_list(Opts)),
+    unlink(Client),
     on_exit(fun() -> catch emqtt:stop(Client) end),
     Client.
 
@@ -161,58 +165,27 @@ is_persistent_connect_opts(#{properties := #{'Session-Expiry-Interval' := EI}})
     EI > 0.
 
 list_all_sessions(Node) ->
-    erpc:call(Node, emqx_persistent_session_ds, list_all_sessions, []).
+    erpc:call(Node, emqx_persistent_session_ds_state, list_sessions, []).
 
 list_all_subscriptions(Node) ->
-    erpc:call(Node, emqx_persistent_session_ds, list_all_subscriptions, []).
+    Sessions = list_all_sessions(Node),
+    lists:flatmap(
+        fun(ClientId) ->
+            #{s := #{subscriptions := Subs}} = erpc:call(
+                Node, emqx_persistent_session_ds, print_session, [ClientId]
+            ),
+            maps:to_list(Subs)
+        end,
+        Sessions
+    ).
 
 list_all_pubranges(Node) ->
     erpc:call(Node, emqx_persistent_session_ds, list_all_pubranges, []).
 
-prop_only_cores_run_gc(CoreNodes) ->
-    {"only core nodes run gc", fun(Trace) -> ?MODULE:prop_only_cores_run_gc(Trace, CoreNodes) end}.
-prop_only_cores_run_gc(Trace, CoreNodes) ->
-    GCNodes = lists:usort([
-        N
-     || #{
-            ?snk_kind := K,
-            ?snk_meta := #{node := N}
-        } <- Trace,
-        lists:member(K, [ds_session_gc, ds_session_gc_lock_taken]),
-        N =/= node()
-    ]),
-    ?assertEqual(lists:usort(CoreNodes), GCNodes).
-
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
 
-t_non_persistent_session_subscription(_Config) ->
-    ClientId = atom_to_binary(?FUNCTION_NAME),
-    SubTopicFilter = <<"t/#">>,
-    ?check_trace(
-        begin
-            ?tp(notice, "starting", #{}),
-            Client = start_client(#{
-                clientid => ClientId,
-                properties => #{'Session-Expiry-Interval' => 0}
-            }),
-            {ok, _} = emqtt:connect(Client),
-            ?tp(notice, "subscribing", #{}),
-            {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client, SubTopicFilter, qos2),
-
-            ok = emqtt:stop(Client),
-
-            ok
-        end,
-        fun(Trace) ->
-            ct:pal("trace:\n  ~p", [Trace]),
-            ?assertEqual([], ?of_kind(ds_session_subscription_added, Trace)),
-            ok
-        end
-    ),
-    ok.
-
 t_session_subscription_idempotency(Config) ->
     [Node1Spec | _] = ?config(node_specs, Config),
     [Node1] = ?config(nodes, Config),
@@ -220,6 +193,7 @@ t_session_subscription_idempotency(Config) ->
     SubTopicFilter = <<"t/+">>,
     ClientId = <<"myclientid">>,
     ?check_trace(
+        #{timetrap => 30_000},
         begin
             ?force_ordering(
                 #{?snk_kind := persistent_session_ds_subscription_added},
@@ -281,11 +255,11 @@ t_session_unsubscription_idempotency(Config) ->
     SubTopicFilter = <<"t/+">>,
     ClientId = <<"myclientid">>,
     ?check_trace(
+        #{timetrap => 30_000},
         begin
             ?force_ordering(
                 #{
-                    ?snk_kind := persistent_session_ds_subscription_delete,
+                    ?snk_kind := persistent_session_ds_subscription_delete
-                    ?snk_span := {complete, _}
                 },
                 _NEvents0 = 1,
                 #{?snk_kind := will_restart_node},
@@ -385,6 +359,7 @@ do_t_session_discard(Params) ->
     ReconnectOpts = ReconnectOpts0#{clientid => ClientId},
     SubTopicFilter = <<"t/+">>,
     ?check_trace(
+        #{timetrap => 30_000},
         begin
             ?tp(notice, "starting", #{}),
             Client0 = start_client(#{
@@ -402,27 +377,26 @@ do_t_session_discard(Params) ->
             ?retry(
                 _Sleep0 = 100,
                 _Attempts0 = 50,
-                true = map_size(emqx_persistent_session_ds:list_all_streams()) > 0
+                #{} = emqx_persistent_session_ds_state:print_session(ClientId)
             ),
             ok = emqtt:stop(Client0),
             ?tp(notice, "disconnected", #{}),
 
             ?tp(notice, "reconnecting", #{}),
-            %% we still have streams
+            %% we still have the session:
-            ?assert(map_size(emqx_persistent_session_ds:list_all_streams()) > 0),
+            ?assertMatch(#{}, emqx_persistent_session_ds_state:print_session(ClientId)),
             Client1 = start_client(ReconnectOpts),
             {ok, _} = emqtt:connect(Client1),
             ?assertEqual([], emqtt:subscriptions(Client1)),
             case is_persistent_connect_opts(ReconnectOpts) of
                 true ->
-                    ?assertMatch(#{ClientId := _}, emqx_persistent_session_ds:list_all_sessions());
+                    ?assertMatch(#{}, emqx_persistent_session_ds_state:print_session(ClientId));
                 false ->
-                    ?assertEqual(#{}, emqx_persistent_session_ds:list_all_sessions())
+                    ?assertEqual(
+                        undefined, emqx_persistent_session_ds_state:print_session(ClientId)
+                    )
             end,
-            ?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()),
             ?assertEqual([], emqx_persistent_session_ds_router:topics()),
-            ?assertEqual(#{}, emqx_persistent_session_ds:list_all_streams()),
-            ?assertEqual(#{}, emqx_persistent_session_ds:list_all_pubranges()),
             ok = emqtt:stop(Client1),
             ?tp(notice, "disconnected", #{}),
 
@@ -436,6 +410,8 @@ do_t_session_discard(Params) ->
     ok.
 
 t_session_expiration1(Config) ->
+    %% This testcase verifies that the properties passed in the
+    %% CONNECT packet are respected by the GC process:
     ClientId = atom_to_binary(?FUNCTION_NAME),
     Opts = #{
         clientid => ClientId,
@@ -448,6 +424,9 @@ t_session_expiration1(Config) ->
     do_t_session_expiration(Config, Opts).
 
 t_session_expiration2(Config) ->
+    %% This testcase updates the expiry interval for the session in
+    %% the _DISCONNECT_ packet. This setting should be respected by GC
+    %% process:
     ClientId = atom_to_binary(?FUNCTION_NAME),
     Opts = #{
         clientid => ClientId,
@@ -462,6 +441,8 @@ t_session_expiration2(Config) ->
     do_t_session_expiration(Config, Opts).
 
 do_t_session_expiration(_Config, Opts) ->
+    %% Sequence is a list of pairs of properties passed through the
+    %% CONNECT and for the DISCONNECT for each session:
     #{
         clientid := ClientId,
         sequence := [
@@ -472,13 +453,14 @@ do_t_session_expiration(_Config, Opts) ->
     } = Opts,
     CommonParams = #{proto_ver => v5, clientid => ClientId},
     ?check_trace(
+        #{timetrap => 30_000},
         begin
             Topic = <<"some/topic">>,
             Params0 = maps:merge(CommonParams, FirstConn),
             Client0 = start_client(Params0),
             {ok, _} = emqtt:connect(Client0),
             {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client0, Topic, ?QOS_2),
-            Subs0 = emqx_persistent_session_ds:list_all_subscriptions(),
+            #{s := #{subscriptions := Subs0}} = emqx_persistent_session_ds:print_session(ClientId),
             ?assertEqual(1, map_size(Subs0), #{subs => Subs0}),
             Info0 = maps:from_list(emqtt:info(Client0)),
             ?assertEqual(0, maps:get(session_present, Info0), #{info => Info0}),
@@ -493,7 +475,7 @@ do_t_session_expiration(_Config, Opts) ->
             ?assertEqual([], Subs1),
             emqtt:disconnect(Client1, ?RC_NORMAL_DISCONNECTION, SecondDisconn),
 
-            ct:sleep(1_500),
+            ct:sleep(2_500),
 
             Params2 = maps:merge(CommonParams, ThirdConn),
             Client2 = start_client(Params2),
@@ -505,9 +487,9 @@ do_t_session_expiration(_Config, Opts) ->
             emqtt:publish(Client2, Topic, <<"payload">>),
             ?assertNotReceive({publish, #{topic := Topic}}),
             %% ensure subscriptions are absent from table.
-            ?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()),
+            #{s := #{subscriptions := Subs3}} = emqx_persistent_session_ds:print_session(ClientId),
+            ?assertEqual([], maps:to_list(Subs3)),
             emqtt:disconnect(Client2, ?RC_NORMAL_DISCONNECTION, ThirdDisconn),
-
             ok
         end,
         []
@@ -515,14 +497,13 @@ do_t_session_expiration(_Config, Opts) ->
     ok.
 
 t_session_gc(Config) ->
-    GCInterval = ?config(gc_interval, Config),
     [Node1, Node2, _Node3] = Nodes = ?config(nodes, Config),
-    CoreNodes = [Node1, Node2],
     [
         Port1,
         Port2,
         Port3
     ] = lists:map(fun(N) -> get_mqtt_port(N, tcp) end, Nodes),
+    ct:pal("Ports: ~p", [[Port1, Port2, Port3]]),
     CommonParams = #{
         clean_start => false,
         proto_ver => v5
@@ -539,15 +520,16 @@ t_session_gc(Config) ->
     end,
 
     ?check_trace(
+        #{timetrap => 30_000},
         begin
-            ClientId0 = <<"session_gc0">>,
-            Client0 = StartClient(ClientId0, Port1, 30),
-
             ClientId1 = <<"session_gc1">>,
-            Client1 = StartClient(ClientId1, Port2, 1),
+            Client1 = StartClient(ClientId1, Port1, 30),
 
             ClientId2 = <<"session_gc2">>,
-            Client2 = StartClient(ClientId2, Port3, 1),
+            Client2 = StartClient(ClientId2, Port2, 1),
+
+            ClientId3 = <<"session_gc3">>,
+            Client3 = StartClient(ClientId3, Port3, 1),
 
             lists:foreach(
                 fun(Client) ->
@@ -557,55 +539,48 @@ t_session_gc(Config) ->
                     {ok, _} = emqtt:publish(Client, Topic, Payload, ?QOS_1),
                     ok
                 end,
-                [Client0, Client1, Client2]
+                [Client1, Client2, Client3]
             ),
 
             %% Clients are still alive; no session is garbage collected.
-            Res0 = ?block_until(
+            ?assertMatch(
+                {ok, _},
+                ?block_until(
                     #{
                         ?snk_kind := ds_session_gc,
                         ?snk_span := {complete, _},
                         ?snk_meta := #{node := N}
-                } when
+                    } when N =/= node()
-                    N =/= node(),
-                3 * GCInterval + 1_000
-            ),
-            ?assertMatch({ok, _}, Res0),
-            {ok, #{?snk_meta := #{time := T0}}} = Res0,
-            Sessions0 = list_all_sessions(Node1),
-            Subs0 = list_all_subscriptions(Node1),
-            ?assertEqual(3, map_size(Sessions0), #{sessions => Sessions0}),
-            ?assertEqual(3, map_size(Subs0), #{subs => Subs0}),
-
-            %% Now we disconnect 2 of them; only those should be GC'ed.
-            ?assertMatch(
-                {ok, {ok, _}},
-                ?wait_async_action(
-                    emqtt:stop(Client1),
-                    #{?snk_kind := terminate},
-                    1_000
                 )
             ),
-            ct:pal("disconnected client1"),
+            ?assertMatch([_, _, _], list_all_sessions(Node1), sessions),
+            ?assertMatch([_, _, _], list_all_subscriptions(Node1), subscriptions),
+
+            %% Now we disconnect 2 of them; only those should be GC'ed.
+
             ?assertMatch(
                 {ok, {ok, _}},
                 ?wait_async_action(
                     emqtt:stop(Client2),
-                    #{?snk_kind := terminate},
+                    #{?snk_kind := terminate}
-                    1_000
                 )
             ),
-            ct:pal("disconnected client2"),
+            ?tp(notice, "disconnected client1", #{}),
+            ?assertMatch(
+                {ok, {ok, _}},
+                ?wait_async_action(
+                    emqtt:stop(Client3),
+                    #{?snk_kind := terminate}
+                )
+            ),
+            ?tp(notice, "disconnected client2", #{}),
             ?assertMatch(
                 {ok, _},
                 ?block_until(
                     #{
                         ?snk_kind := ds_session_gc_cleaned,
-                        ?snk_meta := #{node := N, time := T},
+                        session_id := ClientId2
-                        session_ids := [ClientId1]
+                    }
-                    } when
-                        N =/= node() andalso T > T0,
-                    4 * GCInterval + 1_000
                 )
             ),
             ?assertMatch(
@@ -613,22 +588,14 @@ t_session_gc(Config) ->
                 ?block_until(
                     #{
                         ?snk_kind := ds_session_gc_cleaned,
-                        ?snk_meta := #{node := N, time := T},
+                        session_id := ClientId3
-                        session_ids := [ClientId2]
+                    }
-                    } when
-                        N =/= node() andalso T > T0,
-                    4 * GCInterval + 1_000
                 )
             ),
-            Sessions1 = list_all_sessions(Node1),
+            ?retry(50, 3, [ClientId1] = list_all_sessions(Node1)),
-            Subs1 = list_all_subscriptions(Node1),
+            ?assertMatch([_], list_all_subscriptions(Node1), subscriptions),
-            ?assertEqual(1, map_size(Sessions1), #{sessions => Sessions1}),
-            ?assertEqual(1, map_size(Subs1), #{subs => Subs1}),
-
             ok
         end,
-        [
+        []
-            prop_only_cores_run_gc(CoreNodes)
-        ]
     ),
     ok.

apps/emqx/priv/bpapi.versions

@@ -22,6 +22,8 @@
 {emqx_delayed,3}.
 {emqx_ds,1}.
 {emqx_ds,2}.
+{emqx_ds,3}.
+{emqx_ds,4}.
 {emqx_eviction_agent,1}.
 {emqx_eviction_agent,2}.
 {emqx_exhook,1}.

apps/emqx/rebar.config

@@ -28,9 +28,9 @@
     {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.1"}}},
-    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.18.3"}}},
+    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.18.4"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.4"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.41.0"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
@@ -71,7 +71,7 @@
     {statistics, true}
 ]}.
 
-{project_plugins, [erlfmt]}.
+{project_plugins, [{erlfmt, "1.3.0"}]}.
 
 {erlfmt, [
     {files, [

apps/emqx/src/emqx_access_control.erl

@@ -183,8 +183,13 @@ log_result(#{username := Username}, Topic, Action, From, Result) ->
         }
     end,
     case Result of
-        allow -> ?SLOG(info, (LogMeta())#{msg => "authorization_permission_allowed"});
+        allow ->
-        deny -> ?SLOG(warning, (LogMeta())#{msg => "authorization_permission_denied"})
+            ?SLOG(info, (LogMeta())#{msg => "authorization_permission_allowed"});
+        deny ->
+            ?SLOG_THROTTLE(
+                warning,
+                (LogMeta())#{msg => authorization_permission_denied}
+            )
     end.
 
 %% @private Format authorization rules source.

apps/emqx/src/emqx_alarm.erl

@@ -21,12 +21,9 @@
 -include("emqx.hrl").
 -include("logger.hrl").
 
-%% Mnesia bootstrap
+-export([create_tables/0]).
--export([mnesia/1]).
-
--boot_mnesia({mnesia, [boot]}).
-
 -export([start_link/0]).
+
 %% API
 -export([
     activate/1,
@@ -86,7 +83,7 @@
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------
 
-mnesia(boot) ->
+create_tables() ->
     ok = mria:create_table(
         ?ACTIVATED_ALARM,
         [
@@ -106,7 +103,8 @@ mnesia(boot) ->
             {record_name, deactivated_alarm},
             {attributes, record_info(fields, deactivated_alarm)}
         ]
-    ).
+    ),
+    [?ACTIVATED_ALARM, ?DEACTIVATED_ALARM].
 
 %%--------------------------------------------------------------------
 %% API

apps/emqx/src/emqx_banned.erl

@@ -25,9 +25,7 @@
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 %% Mnesia bootstrap
--export([mnesia/1]).
+-export([create_tables/0]).
-
--boot_mnesia({mnesia, [boot]}).
 
 -export([start_link/0, stop/0]).
 
@@ -39,7 +37,9 @@
     info/1,
     format/1,
     parse/1,
-    clear/0
+    clear/0,
+    who/2,
+    tables/0
 ]).
 
 %% gen_server callbacks
@@ -61,7 +61,8 @@
 
 -elvis([{elvis_style, state_record_and_type, disable}]).
 
--define(BANNED_TAB, ?MODULE).
+-define(BANNED_INDIVIDUAL_TAB, ?MODULE).
+-define(BANNED_RULE_TAB, emqx_banned_rules).
 
 %% The default expiration time should be infinite
 %% but for compatibility, a large number (1 years) is used here to represent the 'infinite'
@@ -76,20 +77,26 @@
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------
 
-mnesia(boot) ->
+create_tables() ->
-    ok = mria:create_table(?BANNED_TAB, [
+    Options = [
         {type, set},
         {rlog_shard, ?COMMON_SHARD},
         {storage, disc_copies},
         {record_name, banned},
         {attributes, record_info(fields, banned)},
         {storage_properties, [{ets, [{read_concurrency, true}]}]}
-    ]).
+    ],
+    ok = mria:create_table(?BANNED_INDIVIDUAL_TAB, Options),
+    ok = mria:create_table(?BANNED_RULE_TAB, Options),
+    [?BANNED_INDIVIDUAL_TAB, ?BANNED_RULE_TAB].
 
 %%--------------------------------------------------------------------
 %% Data backup
 %%--------------------------------------------------------------------
-backup_tables() -> [?BANNED_TAB].
+backup_tables() -> tables().
+
+-spec tables() -> [atom()].
+tables() -> [?BANNED_RULE_TAB, ?BANNED_INDIVIDUAL_TAB].
 
 %% @doc Start the banned server.
 -spec start_link() -> startlink_ret().
@@ -104,16 +111,10 @@ stop() -> gen_server:stop(?MODULE).
 check(ClientInfo) ->
     do_check({clientid, maps:get(clientid, ClientInfo, undefined)}) orelse
         do_check({username, maps:get(username, ClientInfo, undefined)}) orelse
-        do_check({peerhost, maps:get(peerhost, ClientInfo, undefined)}).
+        do_check({peerhost, maps:get(peerhost, ClientInfo, undefined)}) orelse
-
+        do_check_rules(ClientInfo).
-do_check({_, undefined}) ->
-    false;
-do_check(Who) when is_tuple(Who) ->
-    case mnesia:dirty_read(?BANNED_TAB, Who) of
-        [] -> false;
-        [#banned{until = Until}] -> Until > erlang:system_time(second)
-    end.
 
+-spec format(emqx_types:banned()) -> map().
 format(#banned{
     who = Who0,
     by = By,
@@ -121,7 +122,7 @@ format(#banned{
     at = At,
     until = Until
 }) ->
-    {As, Who} = maybe_format_host(Who0),
+    {As, Who} = format_who(Who0),
     #{
         as => As,
         who => Who,
@@ -131,6 +132,7 @@ format(#banned{
         until => to_rfc3339(Until)
     }.
 
+-spec parse(map()) -> emqx_types:banned() | {error, term()}.
 parse(Params) ->
     case parse_who(Params) of
         {error, Reason} ->
@@ -155,24 +157,6 @@ parse(Params) ->
                     {error, ErrorReason}
             end
     end.
-parse_who(#{as := As, who := Who}) ->
-    parse_who(#{<<"as">> => As, <<"who">> => Who});
-parse_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) ->
-    case inet:parse_address(binary_to_list(Peerhost0)) of
-        {ok, Peerhost} -> {peerhost, Peerhost};
-        {error, einval} -> {error, "bad peerhost"}
-    end;
-parse_who(#{<<"as">> := As, <<"who">> := Who}) ->
-    {As, Who}.
-
-maybe_format_host({peerhost, Host}) ->
-    AddrBinary = list_to_binary(inet:ntoa(Host)),
-    {peerhost, AddrBinary};
-maybe_format_host({As, Who}) ->
-    {As, Who}.
-
-to_rfc3339(Timestamp) ->
-    emqx_utils_calendar:epoch_to_rfc3339(Timestamp, second).
 
 -spec create(emqx_types:banned() | map()) ->
     {ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}.
@@ -194,7 +178,7 @@ create(#{
 create(Banned = #banned{who = Who}) ->
     case look_up(Who) of
         [] ->
-            insert_banned(Banned),
+            insert_banned(table(Who), Banned),
             {ok, Banned};
         [OldBanned = #banned{until = Until}] ->
             %% Don't support shorten or extend the until time by overwrite.
@@ -204,33 +188,52 @@ create(Banned = #banned{who = Who}) ->
                     {error, {already_exist, OldBanned}};
                 %% overwrite expired one is ok.
                 false ->
-                    insert_banned(Banned),
+                    insert_banned(table(Who), Banned),
                     {ok, Banned}
             end
     end.
 
+-spec look_up(emqx_types:banned_who() | map()) -> [emqx_types:banned()].
 look_up(Who) when is_map(Who) ->
     look_up(parse_who(Who));
 look_up(Who) ->
-    mnesia:dirty_read(?BANNED_TAB, Who).
+    mnesia:dirty_read(table(Who), Who).
 
--spec delete(
+-spec delete(map() | emqx_types:banned_who()) -> ok.
-    {clientid, emqx_types:clientid()}
-    | {username, emqx_types:username()}
-    | {peerhost, emqx_types:peerhost()}
-) -> ok.
 delete(Who) when is_map(Who) ->
     delete(parse_who(Who));
 delete(Who) ->
-    mria:dirty_delete(?BANNED_TAB, Who).
+    mria:dirty_delete(table(Who), Who).
 
-info(InfoKey) ->
+-spec info(size) -> non_neg_integer().
-    mnesia:table_info(?BANNED_TAB, InfoKey).
+info(size) ->
+    mnesia:table_info(?BANNED_INDIVIDUAL_TAB, size) + mnesia:table_info(?BANNED_RULE_TAB, size).
 
+-spec clear() -> ok.
 clear() ->
-    _ = mria:clear_table(?BANNED_TAB),
+    _ = mria:clear_table(?BANNED_INDIVIDUAL_TAB),
+    _ = mria:clear_table(?BANNED_RULE_TAB),
     ok.
 
+%% Creating banned with `#banned{}` records is exposed as a public API
+%% so we need helpers to create the `who` field of `#banned{}` records
+-spec who(atom(), binary() | inet:ip_address() | esockd_cidr:cidr()) -> emqx_types:banned_who().
+who(clientid, ClientId) when is_binary(ClientId) -> {clientid, ClientId};
+who(username, Username) when is_binary(Username) -> {username, Username};
+who(peerhost, Peerhost) when is_tuple(Peerhost) -> {peerhost, Peerhost};
+who(peerhost, Peerhost) when is_binary(Peerhost) ->
+    {ok, Addr} = inet:parse_address(binary_to_list(Peerhost)),
+    {peerhost, Addr};
+who(clientid_re, RE) when is_binary(RE) ->
+    {ok, RECompiled} = re:compile(RE),
+    {clientid_re, {RECompiled, RE}};
+who(username_re, RE) when is_binary(RE) ->
+    {ok, RECompiled} = re:compile(RE),
+    {username_re, {RECompiled, RE}};
+who(peerhost_net, CIDR) when is_tuple(CIDR) -> {peerhost_net, CIDR};
+who(peerhost_net, CIDR) when is_binary(CIDR) ->
+    {peerhost_net, esockd_cidr:parse(binary_to_list(CIDR), true)}.
+
 %%--------------------------------------------------------------------
 %% gen_server callbacks
 %%--------------------------------------------------------------------
@@ -265,6 +268,81 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
+do_check({_, undefined}) ->
+    false;
+do_check(Who) when is_tuple(Who) ->
+    case mnesia:dirty_read(table(Who), Who) of
+        [] -> false;
+        [#banned{until = Until}] -> Until > erlang:system_time(second)
+    end.
+
+do_check_rules(ClientInfo) ->
+    Rules = all_rules(),
+    Now = erlang:system_time(second),
+    lists:any(
+        fun(Rule) -> is_rule_actual(Rule, Now) andalso do_check_rule(Rule, ClientInfo) end, Rules
+    ).
+
+is_rule_actual(#banned{until = Until}, Now) ->
+    Until > Now.
+
+do_check_rule(#banned{who = {clientid_re, {RE, _}}}, #{clientid := ClientId}) ->
+    is_binary(ClientId) andalso re:run(ClientId, RE) =/= nomatch;
+do_check_rule(#banned{who = {clientid_re, _}}, #{}) ->
+    false;
+do_check_rule(#banned{who = {username_re, {RE, _}}}, #{username := Username}) ->
+    is_binary(Username) andalso re:run(Username, RE) =/= nomatch;
+do_check_rule(#banned{who = {username_re, _}}, #{}) ->
+    false;
+do_check_rule(#banned{who = {peerhost_net, CIDR}}, #{peerhost := Peerhost}) ->
+    esockd_cidr:match(Peerhost, CIDR);
+do_check_rule(#banned{who = {peerhost_net, _}}, #{}) ->
+    false.
+
+parse_who(#{as := As, who := Who}) ->
+    parse_who(#{<<"as">> => As, <<"who">> => Who});
+parse_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) ->
+    case inet:parse_address(binary_to_list(Peerhost0)) of
+        {ok, Peerhost} -> {peerhost, Peerhost};
+        {error, einval} -> {error, "bad peerhost"}
+    end;
+parse_who(#{<<"as">> := peerhost_net, <<"who">> := CIDRString}) ->
+    try esockd_cidr:parse(binary_to_list(CIDRString), true) of
+        CIDR -> {peerhost_net, CIDR}
+    catch
+        error:Error -> {error, Error}
+    end;
+parse_who(#{<<"as">> := AsRE, <<"who">> := Who}) when
+    AsRE =:= clientid_re orelse AsRE =:= username_re
+->
+    case re:compile(Who) of
+        {ok, RE} -> {AsRE, {RE, Who}};
+        {error, _} = Error -> Error
+    end;
+parse_who(#{<<"as">> := As, <<"who">> := Who}) when As =:= clientid orelse As =:= username ->
+    {As, Who}.
+
+format_who({peerhost, Host}) ->
+    AddrBinary = list_to_binary(inet:ntoa(Host)),
+    {peerhost, AddrBinary};
+format_who({peerhost_net, CIDR}) ->
+    CIDRBinary = list_to_binary(esockd_cidr:to_string(CIDR)),
+    {peerhost_net, CIDRBinary};
+format_who({AsRE, {_RE, REOriginal}}) when AsRE =:= clientid_re orelse AsRE =:= username_re ->
+    {AsRE, REOriginal};
+format_who({As, Who}) when As =:= clientid orelse As =:= username ->
+    {As, Who}.
+
+to_rfc3339(Timestamp) ->
+    emqx_utils_calendar:epoch_to_rfc3339(Timestamp, second).
+
+table({username, _Username}) -> ?BANNED_INDIVIDUAL_TAB;
+table({clientid, _ClientId}) -> ?BANNED_INDIVIDUAL_TAB;
+table({peerhost, _Peerhost}) -> ?BANNED_INDIVIDUAL_TAB;
+table({username_re, _UsernameRE}) -> ?BANNED_RULE_TAB;
+table({clientid_re, _ClientIdRE}) -> ?BANNED_RULE_TAB;
+table({peerhost_net, _PeerhostNet}) -> ?BANNED_RULE_TAB.
+
 -ifdef(TEST).
 ensure_expiry_timer(State) ->
     State#{expiry_timer := emqx_utils:start_timer(10, expire)}.
@@ -274,19 +352,27 @@ ensure_expiry_timer(State) ->
 -endif.
 
 expire_banned_items(Now) ->
+    lists:foreach(
+        fun(Tab) ->
+            expire_banned_items(Now, Tab)
+        end,
+        [?BANNED_INDIVIDUAL_TAB, ?BANNED_RULE_TAB]
+    ).
+
+expire_banned_items(Now, Tab) ->
     mnesia:foldl(
         fun
             (B = #banned{until = Until}, _Acc) when Until < Now ->
-                mnesia:delete_object(?BANNED_TAB, B, sticky_write);
+                mnesia:delete_object(Tab, B, sticky_write);
             (_, _Acc) ->
                 ok
         end,
         ok,
-        ?BANNED_TAB
+        Tab
     ).
 
-insert_banned(Banned) ->
+insert_banned(Tab, Banned) ->
-    mria:dirty_write(?BANNED_TAB, Banned),
+    mria:dirty_write(Tab, Banned),
     on_banned(Banned).
 
 on_banned(#banned{who = {clientid, ClientId}}) ->
@@ -302,3 +388,6 @@ on_banned(#banned{who = {clientid, ClientId}}) ->
     ok;
 on_banned(_) ->
     ok.
+
+all_rules() ->
+    ets:tab2list(?BANNED_RULE_TAB).

apps/emqx/src/emqx_broker.erl

@@ -85,13 +85,13 @@
 %% Guards
 -define(IS_SUBID(Id), (is_binary(Id) orelse is_atom(Id))).
 
--define(cast_or_eval(Pid, Msg, Expr),
+-define(cast_or_eval(PICK, Msg, Expr),
-    case Pid =:= self() of
+    case PICK of
-        true ->
+        __X_Pid when __X_Pid =:= self() ->
             _ = Expr,
             ok;
-        false ->
+        __X_Pid ->
-            cast(Pid, Msg)
+            cast(__X_Pid, Msg)
     end
 ).
 
@@ -243,7 +243,7 @@ publish(Msg) when is_record(Msg, message) ->
             [];
         Msg1 = #message{topic = Topic} ->
             PersistRes = persist_publish(Msg1),
-            PersistRes ++ route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1))
+            route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1), PersistRes)
     end.
 
 persist_publish(Msg) ->
@@ -283,18 +283,20 @@ delivery(Msg) -> #delivery{sender = self(), message = Msg}.
 %% Route
 %%--------------------------------------------------------------------
 
--spec route([emqx_types:route_entry()], emqx_types:delivery()) ->
+-spec route([emqx_types:route_entry()], emqx_types:delivery(), nil() | [persisted]) ->
     emqx_types:publish_result().
-route([], #delivery{message = Msg}) ->
+route([], #delivery{message = Msg}, _PersistRes = []) ->
     ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]),
     ok = inc_dropped_cnt(Msg),
     [];
-route(Routes, Delivery) ->
+route([], _Delivery, PersistRes = [_ | _]) ->
+    PersistRes;
+route(Routes, Delivery, PersistRes) ->
     lists:foldl(
         fun(Route, Acc) ->
             [do_route(Route, Delivery) | Acc]
         end,
-        [],
+        PersistRes,
         Routes
     ).
 
@@ -438,7 +440,7 @@ subscribed(SubId, Topic) when ?IS_SUBID(SubId) ->
     SubPid = emqx_broker_helper:lookup_subpid(SubId),
     ets:member(?SUBOPTION, {Topic, SubPid}).
 
--spec get_subopts(pid(), emqx_types:topic() | emqx_types:share()) -> maybe(emqx_types:subopts()).
+-spec get_subopts(pid(), emqx_types:topic() | emqx_types:share()) -> option(emqx_types:subopts()).
 get_subopts(SubPid, Topic) when is_pid(SubPid), ?IS_TOPIC(Topic) ->
     lookup_value(?SUBOPTION, {Topic, SubPid});
 get_subopts(SubId, Topic) when ?IS_SUBID(SubId) ->

apps/emqx/src/emqx_broker_helper.erl

@@ -71,11 +71,11 @@ register_sub(SubPid, SubId) when is_pid(SubPid) ->
             error(subid_conflict)
     end.
 
--spec lookup_subid(pid()) -> maybe(emqx_types:subid()).
+-spec lookup_subid(pid()) -> option(emqx_types:subid()).
 lookup_subid(SubPid) when is_pid(SubPid) ->
     emqx_utils_ets:lookup_value(?SUBMON, SubPid).
 
--spec lookup_subpid(emqx_types:subid()) -> maybe(pid()).
+-spec lookup_subpid(emqx_types:subid()) -> option(pid()).
 lookup_subpid(SubId) ->
     emqx_utils_ets:lookup_value(?SUBID, SubId).
 

apps/emqx/src/emqx_broker_sup.erl

@@ -23,6 +23,10 @@
 -export([init/1]).
 
 start_link() ->
+    ok = mria:wait_for_tables(
+        emqx_shared_sub:create_tables() ++
+            emqx_exclusive_subscription:create_tables()
+    ),
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 %%--------------------------------------------------------------------

apps/emqx/src/emqx_channel.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -84,21 +84,21 @@
     %% MQTT ClientInfo
     clientinfo :: emqx_types:clientinfo(),
     %% MQTT Session
-    session :: maybe(emqx_session:t()),
+    session :: option(emqx_session:t()),
     %% Keepalive
-    keepalive :: maybe(emqx_keepalive:keepalive()),
+    keepalive :: option(emqx_keepalive:keepalive()),
     %% MQTT Will Msg
-    will_msg :: maybe(emqx_types:message()),
+    will_msg :: option(emqx_types:message()),
     %% MQTT Topic Aliases
     topic_aliases :: emqx_types:topic_aliases(),
     %% MQTT Topic Alias Maximum
-    alias_maximum :: maybe(map()),
+    alias_maximum :: option(map()),
     %% Authentication Data Cache
-    auth_cache :: maybe(map()),
+    auth_cache :: option(map()),
     %% Quota checkers
     quota :: emqx_limiter_container:container(),
     %% Timers
-    timers :: #{atom() => disabled | maybe(reference())},
+    timers :: #{atom() => disabled | option(reference())},
     %% Conn State
     conn_state :: conn_state(),
     %% Takeover
@@ -191,7 +191,11 @@ info(topic_aliases, #channel{topic_aliases = Aliases}) ->
 info(alias_maximum, #channel{alias_maximum = Limits}) ->
     Limits;
 info(timers, #channel{timers = Timers}) ->
-    Timers.
+    Timers;
+info(session_state, #channel{session = Session}) ->
+    Session;
+info(impl, #channel{session = Session}) ->
+    emqx_session:info(impl, Session).
 
 set_conn_state(ConnState, Channel) ->
     Channel#channel{conn_state = ConnState}.
@@ -536,13 +540,17 @@ handle_in(?AUTH_PACKET(), Channel) ->
     handle_out(disconnect, ?RC_IMPLEMENTATION_SPECIFIC_ERROR, Channel);
 handle_in({frame_error, Reason}, Channel = #channel{conn_state = idle}) ->
     shutdown(shutdown_count(frame_error, Reason), Channel);
-handle_in({frame_error, frame_too_large}, Channel = #channel{conn_state = connecting}) ->
+handle_in(
+    {frame_error, #{cause := frame_too_large} = R}, Channel = #channel{conn_state = connecting}
+) ->
     shutdown(
-        shutdown_count(frame_error, frame_too_large), ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE), Channel
+        shutdown_count(frame_error, R), ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE), Channel
     );
 handle_in({frame_error, Reason}, Channel = #channel{conn_state = connecting}) ->
     shutdown(shutdown_count(frame_error, Reason), ?CONNACK_PACKET(?RC_MALFORMED_PACKET), Channel);
-handle_in({frame_error, frame_too_large}, Channel = #channel{conn_state = ConnState}) when
+handle_in(
+    {frame_error, #{cause := frame_too_large}}, Channel = #channel{conn_state = ConnState}
+) when
     ConnState =:= connected orelse ConnState =:= reauthenticating
 ->
     handle_out(disconnect, {?RC_PACKET_TOO_LARGE, frame_too_large}, Channel);
@@ -608,10 +616,10 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
             Msg = packet_to_message(NPacket, NChannel),
             do_publish(PacketId, Msg, NChannel);
         {error, Rc = ?RC_NOT_AUTHORIZED, NChannel} ->
-            ?SLOG(
+            ?SLOG_THROTTLE(
                 warning,
                 #{
-                    msg => "cannot_publish_to_topic",
+                    msg => cannot_publish_to_topic_due_to_not_authorized,
                     reason => emqx_reason_codes:name(Rc)
                 },
                 #{topic => Topic}
@@ -627,10 +635,10 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
                     handle_out(disconnect, Rc, NChannel)
             end;
         {error, Rc = ?RC_QUOTA_EXCEEDED, NChannel} ->
-            ?SLOG(
+            ?SLOG_THROTTLE(
                 warning,
                 #{
-                    msg => "cannot_publish_to_topic",
+                    msg => cannot_publish_to_topic_due_to_quota_exceeded,
                     reason => emqx_reason_codes:name(Rc)
                 },
                 #{topic => Topic}
@@ -928,7 +936,8 @@ handle_deliver(
     Delivers1 = maybe_nack(Delivers),
     Messages = emqx_session:enrich_delivers(ClientInfo, Delivers1, Session),
     NSession = emqx_session_mem:enqueue(ClientInfo, Messages, Session),
-    {ok, Channel#channel{session = NSession}};
+    %% we need to update stats here, as the stats_timer is canceled after disconnected
+    {ok, {event, updated}, Channel#channel{session = NSession}};
 handle_deliver(Delivers, Channel) ->
     Delivers1 = emqx_external_trace:start_trace_send(Delivers, trace_info(Channel)),
     do_handle_deliver(Delivers1, Channel).
@@ -1546,6 +1555,8 @@ set_username(
 set_username(_ConnPkt, ClientInfo) ->
     {ok, ClientInfo}.
 
+%% The `is_bridge` bit flag in CONNECT packet (parsed as `bridge_mode`)
+%% is invented by mosquitto, named 'try_private': https://mosquitto.org/man/mosquitto-conf-5.html
 set_bridge_mode(#mqtt_packet_connect{is_bridge = true}, ClientInfo) ->
     {ok, ClientInfo#{is_bridge => true}};
 set_bridge_mode(_ConnPkt, _ClientInfo) ->
@@ -2002,14 +2013,15 @@ merge_default_subopts(SubOpts) ->
 %%--------------------------------------------------------------------
 %% Enrich ConnAck Caps
 
-enrich_connack_caps(
+enrich_connack_caps(AckProps, ?IS_MQTT_V5 = Channel) ->
-    AckProps,
+    #channel{
-    ?IS_MQTT_V5 = #channel{
         clientinfo = #{
             zone := Zone
+        },
+        conninfo = #{
+            receive_maximum := ReceiveMaximum
         }
-    }
+    } = Channel,
-) ->
     #{
         max_packet_size := MaxPktSize,
         max_qos_allowed := MaxQoS,
@@ -2024,7 +2036,8 @@ enrich_connack_caps(
         'Topic-Alias-Maximum' => MaxAlias,
         'Wildcard-Subscription-Available' => flag(Wildcard),
         'Subscription-Identifier-Available' => 1,
-        'Shared-Subscription-Available' => flag(Shared)
+        'Shared-Subscription-Available' => flag(Shared),
+        'Receive-Maximum' => ReceiveMaximum
     },
     %% MQTT 5.0 - 3.2.2.3.4:
     %% It is a Protocol Error to include Maximum QoS more than once,
@@ -2318,6 +2331,8 @@ shutdown(Reason, Reply, Packet, Channel) ->
 
 %% process exits with {shutdown, #{shutdown_count := Kind}} will trigger
 %% the connection supervisor (esockd) to keep a shutdown-counter grouped by Kind
+shutdown_count(_Kind, #{cause := Cause} = Reason) when is_atom(Cause) ->
+    Reason#{shutdown_count => Cause};
 shutdown_count(Kind, Reason) when is_map(Reason) ->
     Reason#{shutdown_count => Kind};
 shutdown_count(Kind, Reason) ->

apps/emqx/src/emqx_cm.erl

@@ -1,5 +1,5 @@
 %%-------------------------------------------------------------------
-%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -124,7 +124,8 @@
     {?CHAN_TAB, 'channels.count', 'channels.max'},
     {?CHAN_TAB, 'sessions.count', 'sessions.max'},
     {?CHAN_CONN_TAB, 'connections.count', 'connections.max'},
-    {?CHAN_LIVE_TAB, 'live_connections.count', 'live_connections.max'}
+    {?CHAN_LIVE_TAB, 'live_connections.count', 'live_connections.max'},
+    {?CHAN_REG_TAB, 'cluster_sessions.count', 'cluster_sessions.max'}
 ]).
 
 %% Batch drain
@@ -200,12 +201,12 @@ do_unregister_channel({_ClientId, ChanPid} = Chan) ->
     true.
 
 %% @doc Get info of a channel.
--spec get_chan_info(emqx_types:clientid()) -> maybe(emqx_types:infos()).
+-spec get_chan_info(emqx_types:clientid()) -> option(emqx_types:infos()).
 get_chan_info(ClientId) ->
     with_channel(ClientId, fun(ChanPid) -> get_chan_info(ClientId, ChanPid) end).
 
 -spec do_get_chan_info(emqx_types:clientid(), chan_pid()) ->
-    maybe(emqx_types:infos()).
+    option(emqx_types:infos()).
 do_get_chan_info(ClientId, ChanPid) ->
     Chan = {ClientId, ChanPid},
     try
@@ -215,7 +216,7 @@ do_get_chan_info(ClientId, ChanPid) ->
     end.
 
 -spec get_chan_info(emqx_types:clientid(), chan_pid()) ->
-    maybe(emqx_types:infos()).
+    option(emqx_types:infos()).
 get_chan_info(ClientId, ChanPid) ->
     wrap_rpc(emqx_cm_proto_v2:get_chan_info(ClientId, ChanPid)).
 
@@ -230,12 +231,12 @@ set_chan_info(ClientId, Info) when ?IS_CLIENTID(ClientId) ->
     end.
 
 %% @doc Get channel's stats.
--spec get_chan_stats(emqx_types:clientid()) -> maybe(emqx_types:stats()).
+-spec get_chan_stats(emqx_types:clientid()) -> option(emqx_types:stats()).
 get_chan_stats(ClientId) ->
     with_channel(ClientId, fun(ChanPid) -> get_chan_stats(ClientId, ChanPid) end).
 
 -spec do_get_chan_stats(emqx_types:clientid(), chan_pid()) ->
-    maybe(emqx_types:stats()).
+    option(emqx_types:stats()).
 do_get_chan_stats(ClientId, ChanPid) ->
     Chan = {ClientId, ChanPid},
     try
@@ -245,7 +246,7 @@ do_get_chan_stats(ClientId, ChanPid) ->
     end.
 
 -spec get_chan_stats(emqx_types:clientid(), chan_pid()) ->
-    maybe(emqx_types:stats()).
+    option(emqx_types:stats()).
 get_chan_stats(ClientId, ChanPid) ->
     wrap_rpc(emqx_cm_proto_v2:get_chan_stats(ClientId, ChanPid)).
 
@@ -325,7 +326,7 @@ takeover_session_end({ConnMod, ChanPid}) ->
     end.
 
 -spec pick_channel(emqx_types:clientid()) ->
-    maybe(pid()).
+    option(pid()).
 pick_channel(ClientId) ->
     case lookup_channels(ClientId) of
         [] ->
@@ -670,7 +671,11 @@ handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}
     ChanPids = [Pid | emqx_utils:drain_down(BatchSize)],
     {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon),
     lists:foreach(fun mark_channel_disconnected/1, ChanPids),
-    ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]),
+    ok = emqx_pool:async_submit_to_pool(
+        ?CM_POOL,
+        fun lists:foreach/2,
+        [fun ?MODULE:clean_down/1, Items]
+    ),
     {noreply, State#{chan_pmon := PMon1}};
 handle_info(Info, State) ->
     ?SLOG(error, #{msg => "unexpected_info", info => Info}),

apps/emqx/src/emqx_cm_locker.erl

@@ -32,7 +32,7 @@ start_link() ->
     ekka_locker:start_link(?MODULE).
 
 -spec trans(
-    maybe(emqx_types:clientid()),
+    option(emqx_types:clientid()),
     fun(([node()]) -> any())
 ) -> any().
 trans(undefined, Fun) ->

apps/emqx/src/emqx_cm_registry.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2019-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -19,18 +19,15 @@
 
 -behaviour(gen_server).
 
--include("emqx.hrl").
--include("emqx_cm.hrl").
--include("logger.hrl").
--include("types.hrl").
-
 -export([start_link/0]).
 
--export([is_enabled/0]).
+-export([is_enabled/0, is_hist_enabled/0]).
 
 -export([
     register_channel/1,
-    unregister_channel/1
+    register_channel2/1,
+    unregister_channel/1,
+    unregister_channel2/1
 ]).
 
 -export([lookup_channels/1]).
@@ -50,10 +47,13 @@
     do_cleanup_channels/1
 ]).
 
--define(REGISTRY, ?MODULE).
+-include("emqx.hrl").
--define(LOCK, {?MODULE, cleanup_down}).
+-include("emqx_cm.hrl").
+-include("logger.hrl").
+-include("types.hrl").
 
--record(channel, {chid, pid}).
+-define(REGISTRY, ?MODULE).
+-define(NODE_DOWN_CLEANUP_LOCK, {?MODULE, cleanup_down}).
 
 %% @doc Start the global channel registry.
 -spec start_link() -> startlink_ret().
@@ -69,6 +69,11 @@ start_link() ->
 is_enabled() ->
     emqx:get_config([broker, enable_session_registry]).
 
+%% @doc Is the global session registration history enabled?
+-spec is_hist_enabled() -> boolean().
+is_hist_enabled() ->
+    retain_duration() > 0.
+
 %% @doc Register a global channel.
 -spec register_channel(
     emqx_types:clientid()
@@ -77,11 +82,21 @@ is_enabled() ->
 register_channel(ClientId) when is_binary(ClientId) ->
     register_channel({ClientId, self()});
 register_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) ->
+    IsHistEnabled = is_hist_enabled(),
     case is_enabled() of
-        true -> mria:dirty_write(?CHAN_REG_TAB, record(ClientId, ChanPid));
+        true when IsHistEnabled ->
-        false -> ok
+            mria:async_dirty(?CM_SHARD, fun ?MODULE:register_channel2/1, [record(ClientId, ChanPid)]);
+        true ->
+            mria:dirty_write(?CHAN_REG_TAB, record(ClientId, ChanPid));
+        false ->
+            ok
     end.
 
+%% @private
+register_channel2(#channel{chid = ClientId} = Record) ->
+    _ = delete_hist_d(ClientId),
+    mria:dirty_write(?CHAN_REG_TAB, Record).
+
 %% @doc Unregister a global channel.
 -spec unregister_channel(
     emqx_types:clientid()
@@ -90,19 +105,54 @@ register_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid)
 unregister_channel(ClientId) when is_binary(ClientId) ->
     unregister_channel({ClientId, self()});
 unregister_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) ->
+    IsHistEnabled = is_hist_enabled(),
     case is_enabled() of
-        true -> mria:dirty_delete_object(?CHAN_REG_TAB, record(ClientId, ChanPid));
+        true when IsHistEnabled ->
-        false -> ok
+            mria:async_dirty(?CM_SHARD, fun ?MODULE:unregister_channel2/1, [
+                record(ClientId, ChanPid)
+            ]);
+        true ->
+            mria:dirty_delete_object(?CHAN_REG_TAB, record(ClientId, ChanPid));
+        false ->
+            ok
     end.
 
+%% @private
+unregister_channel2(#channel{chid = ClientId} = Record) ->
+    mria:dirty_delete_object(?CHAN_REG_TAB, Record),
+    ok = insert_hist_d(ClientId).
+
 %% @doc Lookup the global channels.
 -spec lookup_channels(emqx_types:clientid()) -> list(pid()).
 lookup_channels(ClientId) ->
-    [ChanPid || #channel{pid = ChanPid} <- mnesia:dirty_read(?CHAN_REG_TAB, ClientId)].
+    lists:filtermap(
+        fun
+            (#channel{pid = ChanPid}) when is_pid(ChanPid) ->
+                case is_pid_down(ChanPid) of
+                    true ->
+                        false;
+                    _ ->
+                        {true, ChanPid}
+                end;
+            (_) ->
+                false
+        end,
+        mnesia:dirty_read(?CHAN_REG_TAB, ClientId)
+    ).
+
+%% Return 'true' or 'false' if it's a local pid.
+%% Otherwise return 'unknown'.
+is_pid_down(Pid) when node(Pid) =:= node() ->
+    not erlang:is_process_alive(Pid);
+is_pid_down(_) ->
+    unknown.
 
 record(ClientId, ChanPid) ->
     #channel{chid = ClientId, pid = ChanPid}.
 
+hist(ClientId) ->
+    #channel{chid = ClientId, pid = now_ts()}.
+
 %%--------------------------------------------------------------------
 %% gen_server callbacks
 %%--------------------------------------------------------------------
@@ -158,15 +208,95 @@ code_change(_OldVsn, State, _Extra) ->
 
 cleanup_channels(Node) ->
     global:trans(
-        {?LOCK, self()},
+        {?NODE_DOWN_CLEANUP_LOCK, self()},
         fun() ->
             mria:transaction(?CM_SHARD, fun ?MODULE:do_cleanup_channels/1, [Node])
         end
     ).
 
 do_cleanup_channels(Node) ->
-    Pat = [{#channel{pid = '$1', _ = '_'}, [{'==', {node, '$1'}, Node}], ['$_']}],
+    Pat = [
-    lists:foreach(fun delete_channel/1, mnesia:select(?CHAN_REG_TAB, Pat, write)).
+        {
+            #channel{pid = '$1', _ = '_'},
+            _Match = [{'andalso', {is_pid, '$1'}, {'==', {node, '$1'}, Node}}],
+            _Return = ['$_']
+        }
+    ],
+    IsHistEnabled = is_hist_enabled(),
+    lists:foreach(
+        fun(Chan) -> delete_channel(IsHistEnabled, Chan) end,
+        mnesia:select(?CHAN_REG_TAB, Pat, write)
+    ).
 
-delete_channel(Chan) ->
+delete_channel(IsHistEnabled, Chan) ->
-    mnesia:delete_object(?CHAN_REG_TAB, Chan, write).
+    mnesia:delete_object(?CHAN_REG_TAB, Chan, write),
+    case IsHistEnabled of
+        true ->
+            insert_hist_t(Chan#channel.chid);
+        false ->
+            ok
+    end.
+
+%%--------------------------------------------------------------------
+%% History entry operations
+
+%% Insert unregistration history in a transaction when unregistering the last channel for a clientid.
+insert_hist_t(ClientId) ->
+    case delete_hist_t(ClientId) of
+        true ->
+            ok;
+        false ->
+            mnesia:write(?CHAN_REG_TAB, hist(ClientId), write)
+    end.
+
+%% Dirty insert unregistration history.
+%% Since dirty opts are used, async pool workers may race deletes and inserts,
+%% so there could be more than one history records for a clientid,
+%% but it should be eventually consistent after the client re-registers or the periodic cleanup.
+insert_hist_d(ClientId) ->
+    %% delete old hist records first
+    case delete_hist_d(ClientId) of
+        true ->
+            ok;
+        false ->
+            mria:dirty_write(?CHAN_REG_TAB, hist(ClientId))
+    end.
+
+%% Current timestamp in seconds.
+now_ts() ->
+    erlang:system_time(seconds).
+
+%% Delete all history records for a clientid, return true if there is a Pid found.
+delete_hist_t(ClientId) ->
+    fold_hist(
+        fun(Hist) -> mnesia:delete_object(?CHAN_REG_TAB, Hist, write) end,
+        mnesia:read(?CHAN_REG_TAB, ClientId, write)
+    ).
+
+%% Delete all history records for a clientid, return true if there is a Pid found.
+delete_hist_d(ClientId) ->
+    fold_hist(
+        fun(Hist) -> mria:dirty_delete_object(?CHAN_REG_TAB, Hist) end,
+        mnesia:dirty_read(?CHAN_REG_TAB, ClientId)
+    ).
+
+%% Fold over the history records, return true if there is a Pid found.
+fold_hist(F, List) ->
+    lists:foldl(
+        fun(#channel{pid = Ts} = Record, HasPid) ->
+            case is_integer(Ts) of
+                true ->
+                    ok = F(Record),
+                    HasPid;
+                false ->
+                    true
+            end
+        end,
+        false,
+        List
+    ).
+
+%% Return the session registration history retain duration.
+-spec retain_duration() -> non_neg_integer().
+retain_duration() ->
+    emqx:get_config([broker, session_history_retain]).

apps/emqx/src/emqx_cm_registry_keeper.erl

@@ -0,0 +1,194 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc This module implements the global session registry history cleaner.
+-module(emqx_cm_registry_keeper).
+-behaviour(gen_server).
+
+-export([
+    start_link/0,
+    count/1
+]).
+
+%% gen_server callbacks
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    terminate/2,
+    code_change/3
+]).
+
+-include_lib("stdlib/include/ms_transform.hrl").
+-include("emqx_cm.hrl").
+
+-define(CACHE_COUNT_THRESHOLD, 1000).
+-define(MIN_COUNT_INTERVAL_SECONDS, 5).
+-define(CLEANUP_CHUNK_SIZE, 10000).
+
+-define(IS_HIST_ENABLED(RETAIN), (RETAIN > 0)).
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+init(_) ->
+    case mria_config:whoami() =:= replicant of
+        true ->
+            ignore;
+        false ->
+            ok = send_delay_start(),
+            {ok, #{next_clientid => undefined}}
+    end.
+
+%% @doc Count the number of sessions.
+%% Include sessions which are expired since the given timestamp if `since' is greater than 0.
+-spec count(non_neg_integer()) -> non_neg_integer().
+count(Since) ->
+    Retain = retain_duration(),
+    Now = now_ts(),
+    %% Get table size if hist is not enabled or
+    %% Since is before the earliest possible retention time.
+    IsCountAll = (not ?IS_HIST_ENABLED(Retain) orelse (Now - Retain >= Since)),
+    case IsCountAll of
+        true ->
+            mnesia:table_info(?CHAN_REG_TAB, size);
+        false ->
+            %% make a gen call to avoid many callers doing the same concurrently
+            gen_server:call(?MODULE, {count, Since}, infinity)
+    end.
+
+handle_call({count, Since}, _From, State) ->
+    {LastCountTime, LastCount} =
+        case State of
+            #{last_count_time := T, last_count := C} ->
+                {T, C};
+            _ ->
+                {0, 0}
+        end,
+    Now = now_ts(),
+    Total = mnesia:table_info(?CHAN_REG_TAB, size),
+    %% Always count if the table is small enough
+    %% or when the last count is too old
+    IsTableSmall = (Total < ?CACHE_COUNT_THRESHOLD),
+    IsLastCountOld = (Now - LastCountTime > ?MIN_COUNT_INTERVAL_SECONDS),
+    case IsTableSmall orelse IsLastCountOld of
+        true ->
+            Count = do_count(Since),
+            CountFinishedAt = now_ts(),
+            {reply, Count, State#{last_count_time => CountFinishedAt, last_count => Count}};
+        false ->
+            {reply, LastCount, State}
+    end;
+handle_call(_Request, _From, State) ->
+    {reply, ok, State}.
+
+handle_cast(_Msg, State) ->
+    {noreply, State}.
+
+handle_info(start, #{next_clientid := NextClientId} = State) ->
+    case is_hist_enabled() of
+        true ->
+            NewNext =
+                case cleanup_one_chunk(NextClientId) of
+                    '$end_of_table' ->
+                        ok = send_delay_start(),
+                        undefined;
+                    Id ->
+                        _ = erlang:garbage_collect(),
+                        Id
+                end,
+            {noreply, State#{next_clientid := NewNext}};
+        false ->
+            %% if not enabled, delay and check again
+            %% because it might be enabled from online config change while waiting
+            ok = send_delay_start(),
+            {noreply, State}
+    end;
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+cleanup_one_chunk(NextClientId) ->
+    Retain = retain_duration(),
+    Now = now_ts(),
+    IsExpired = fun(#channel{pid = Ts}) ->
+        is_integer(Ts) andalso (Ts < Now - Retain)
+    end,
+    cleanup_loop(NextClientId, ?CLEANUP_CHUNK_SIZE, IsExpired).
+
+cleanup_loop(ClientId, 0, _IsExpired) ->
+    ClientId;
+cleanup_loop('$end_of_table', _Count, _IsExpired) ->
+    '$end_of_table';
+cleanup_loop(undefined, Count, IsExpired) ->
+    cleanup_loop(mnesia:dirty_first(?CHAN_REG_TAB), Count, IsExpired);
+cleanup_loop(ClientId, Count, IsExpired) ->
+    Records = mnesia:dirty_read(?CHAN_REG_TAB, ClientId),
+    Next = mnesia:dirty_next(?CHAN_REG_TAB, ClientId),
+    lists:foreach(
+        fun(R) ->
+            case IsExpired(R) of
+                true ->
+                    mria:dirty_delete_object(?CHAN_REG_TAB, R);
+                false ->
+                    ok
+            end
+        end,
+        Records
+    ),
+    cleanup_loop(Next, Count - 1, IsExpired).
+
+is_hist_enabled() ->
+    retain_duration() > 0.
+
+%% Return the session registration history retain duration in seconds.
+-spec retain_duration() -> non_neg_integer().
+retain_duration() ->
+    emqx:get_config([broker, session_history_retain]).
+
+cleanup_delay() ->
+    Default = timer:minutes(2),
+    case retain_duration() of
+        0 ->
+            %% prepare for online config change
+            Default;
+        RetainSeconds ->
+            Min = max(timer:seconds(1), timer:seconds(RetainSeconds) div 4),
+            min(Min, Default)
+    end.
+
+send_delay_start() ->
+    Delay = cleanup_delay(),
+    ok = send_delay_start(Delay).
+
+send_delay_start(Delay) ->
+    _ = erlang:send_after(Delay, self(), start),
+    ok.
+
+now_ts() ->
+    erlang:system_time(seconds).
+
+do_count(Since) ->
+    Ms = ets:fun2ms(fun(#channel{pid = V}) ->
+        is_pid(V) orelse (is_integer(V) andalso (V >= Since))
+    end),
+    ets:select_count(?CHAN_REG_TAB, Ms).

apps/emqx/src/emqx_cm_sup.erl

@@ -25,11 +25,14 @@
 %% for test
 -export([restart_flapping/0]).
 
+-include("emqx_cm.hrl").
+
 %%--------------------------------------------------------------------
 %% API
 %%--------------------------------------------------------------------
 
 start_link() ->
+    ok = mria:wait_for_tables(emqx_banned:create_tables()),
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 %%--------------------------------------------------------------------
@@ -45,7 +48,9 @@ init([]) ->
     Banned = child_spec(emqx_banned, 1000, worker),
     Flapping = child_spec(emqx_flapping, 1000, worker),
     Locker = child_spec(emqx_cm_locker, 5000, worker),
+    CmPool = emqx_pool_sup:spec(emqx_cm_pool_sup, [?CM_POOL, random, {emqx_pool, start_link, []}]),
     Registry = child_spec(emqx_cm_registry, 5000, worker),
+    RegistryKeeper = child_spec(emqx_cm_registry_keeper, 5000, worker),
     Manager = child_spec(emqx_cm, 5000, worker),
     DSSessionGCSup = child_spec(emqx_persistent_session_ds_sup, infinity, supervisor),
     Children =
@@ -53,7 +58,9 @@ init([]) ->
             Banned,
             Flapping,
             Locker,
+            CmPool,
             Registry,
+            RegistryKeeper,
             Manager,
             DSSessionGCSup
         ],

apps/emqx/src/emqx_config_handler.erl

@@ -675,9 +675,19 @@ merge_to_override_config(RawConf, Opts) ->
     maps:merge(UpgradedOldConf, RawConf).
 
 upgrade_conf(Conf) ->
+    ConfigLoader = emqx_app:get_config_loader(),
+    %% ensure module loaded
+    _ = ConfigLoader:module_info(),
+    case erlang:function_exported(ConfigLoader, schema_module, 0) of
+        true ->
+            try_upgrade_conf(apply(ConfigLoader, schema_module, []), Conf);
+        false ->
+            %% this happens during emqx app standalone test
+            Conf
+    end.
+
+try_upgrade_conf(SchemaModule, Conf) ->
     try
-        ConfLoader = emqx_app:get_config_loader(),
-        SchemaModule = apply(ConfLoader, schema_module, []),
         apply(SchemaModule, upgrade_raw_conf, [Conf])
     catch
         ErrorType:Reason:Stack ->

apps/emqx/src/emqx_connection.erl

@@ -99,13 +99,13 @@
     %% Channel State
     channel :: emqx_channel:channel(),
     %% GC State
-    gc_state :: maybe(emqx_gc:gc_state()),
+    gc_state :: option(emqx_gc:gc_state()),
     %% Stats Timer
-    stats_timer :: disabled | maybe(reference()),
+    stats_timer :: disabled | option(reference()),
     %% Idle Timeout
     idle_timeout :: integer() | infinity,
     %% Idle Timer
-    idle_timer :: maybe(reference()),
+    idle_timer :: option(reference()),
     %% Zone name
     zone :: atom(),
     %% Listener Type and Name
@@ -121,7 +121,7 @@
     limiter_timer :: undefined | reference(),
 
     %% QUIC conn owner pid if in use.
-    quic_conn_pid :: maybe(pid())
+    quic_conn_pid :: option(pid())
 }).
 
 -record(retry, {

apps/emqx/src/emqx_exclusive_subscription.erl

@@ -22,14 +22,11 @@
 -logger_header("[exclusive]").
 
 %% Mnesia bootstrap
--export([mnesia/1]).
+-export([create_tables/0]).
 
 %% For upgrade
 -export([on_add_module/0, on_delete_module/0]).
 
--boot_mnesia({mnesia, [boot]}).
--copy_mnesia({mnesia, [copy]}).
-
 -export([
     check_subscribe/2,
     unsubscribe/2,
@@ -53,7 +50,7 @@
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------
 
-mnesia(boot) ->
+create_tables() ->
     StoreProps = [
         {ets, [
             {read_concurrency, true},
@@ -68,14 +65,14 @@ mnesia(boot) ->
         {attributes, record_info(fields, exclusive_subscription)},
         {storage_properties, StoreProps}
     ]),
-    ok = mria_rlog:wait_for_shards([?EXCLUSIVE_SHARD], infinity).
+    [?TAB].
 
 %%--------------------------------------------------------------------
 %% Upgrade
 %%--------------------------------------------------------------------
 
 on_add_module() ->
-    mnesia(boot).
+    mria:wait_for_tables(create_tables()).
 
 on_delete_module() ->
     clear().

apps/emqx/src/emqx_flapping.erl

@@ -150,7 +150,7 @@ handle_cast(
             ),
             Now = erlang:system_time(second),
             Banned = #banned{
-                who = {clientid, ClientId},
+                who = emqx_banned:who(clientid, ClientId),
                 by = <<"flapping detector">>,
                 reason = <<"flapping is detected">>,
                 at = Now,

apps/emqx/src/emqx_frame.erl

@@ -168,7 +168,7 @@ parse_remaining_len(Rest, Header, Options) ->
 parse_remaining_len(_Bin, _Header, _Multiplier, Length, #{max_size := MaxSize}) when
     Length > MaxSize
 ->
-    ?PARSE_ERR(frame_too_large);
+    ?PARSE_ERR(#{cause => frame_too_large, limit => MaxSize, received => Length});
 parse_remaining_len(<<>>, Header, Multiplier, Length, Options) ->
     {more, {{len, #{hdr => Header, len => {Multiplier, Length}}}, Options}};
 %% Match DISCONNECT without payload
@@ -189,12 +189,12 @@ parse_remaining_len(
 parse_remaining_len(
     <<0:8, _Rest/binary>>, _Header = #mqtt_packet_header{type = ?PINGRESP}, 1, 0, _Options
 ) ->
-    ?PARSE_ERR(#{hint => unexpected_packet, header_type => 'PINGRESP'});
+    ?PARSE_ERR(#{cause => unexpected_packet, header_type => 'PINGRESP'});
 %% All other types of messages should not have a zero remaining length.
 parse_remaining_len(
     <<0:8, _Rest/binary>>, Header, 1, 0, _Options
 ) ->
-    ?PARSE_ERR(#{hint => zero_remaining_len, header_type => Header#mqtt_packet_header.type});
+    ?PARSE_ERR(#{cause => zero_remaining_len, header_type => Header#mqtt_packet_header.type});
 %% Match PUBACK, PUBREC, PUBREL, PUBCOMP, UNSUBACK...
 parse_remaining_len(<<0:1, 2:7, Rest/binary>>, Header, 1, 0, Options) ->
     parse_frame(Rest, Header, 2, Options);
@@ -213,7 +213,7 @@ parse_remaining_len(
 ) ->
     FrameLen = Value + Len * Multiplier,
     case FrameLen > MaxSize of
-        true -> ?PARSE_ERR(frame_too_large);
+        true -> ?PARSE_ERR(#{cause => frame_too_large, limit => MaxSize, received => FrameLen});
         false -> parse_frame(Rest, Header, FrameLen, Options)
     end.
 
@@ -267,7 +267,7 @@ packet(Header, Variable, Payload) ->
     #mqtt_packet{header = Header, variable = Variable, payload = Payload}.
 
 parse_connect(FrameBin, StrictMode) ->
-    {ProtoName, Rest} = parse_utf8_string_with_hint(FrameBin, StrictMode, invalid_proto_name),
+    {ProtoName, Rest} = parse_utf8_string_with_cause(FrameBin, StrictMode, invalid_proto_name),
     case ProtoName of
         <<"MQTT">> ->
             ok;
@@ -277,7 +277,7 @@ parse_connect(FrameBin, StrictMode) ->
             %% from spec: the server MAY send disconnect with reason code 0x84
             %% we chose to close socket because the client is likely not talking MQTT anyway
             ?PARSE_ERR(#{
-                hint => invalid_proto_name,
+                cause => invalid_proto_name,
                 expected => <<"'MQTT' or 'MQIsdp'">>,
                 received => ProtoName
             })
@@ -296,11 +296,12 @@ parse_connect2(
         1 -> ?PARSE_ERR(reserved_connect_flag)
     end,
     {Properties, Rest3} = parse_properties(Rest2, ProtoVer, StrictMode),
-    {ClientId, Rest4} = parse_utf8_string_with_hint(Rest3, StrictMode, invalid_clientid),
+    {ClientId, Rest4} = parse_utf8_string_with_cause(Rest3, StrictMode, invalid_clientid),
     ConnPacket = #mqtt_packet_connect{
         proto_name = ProtoName,
         proto_ver = ProtoVer,
         %% For bridge mode, non-standard implementation
+        %% Invented by mosquitto, named 'try_private': https://mosquitto.org/man/mosquitto-conf-5.html
         is_bridge = (BridgeTag =:= 8),
         clean_start = bool(CleanStart),
         will_flag = bool(WillFlag),
@@ -314,14 +315,14 @@ parse_connect2(
     {Username, Rest6} = parse_optional(
         Rest5,
         fun(Bin) ->
-            parse_utf8_string_with_hint(Bin, StrictMode, invalid_username)
+            parse_utf8_string_with_cause(Bin, StrictMode, invalid_username)
         end,
         bool(UsernameFlag)
     ),
     {Password, Rest7} = parse_optional(
         Rest6,
         fun(Bin) ->
-            parse_utf8_string_with_hint(Bin, StrictMode, invalid_password)
+            parse_utf8_string_with_cause(Bin, StrictMode, invalid_password)
         end,
         bool(PasswordFlag)
     ),
@@ -329,10 +330,14 @@ parse_connect2(
         <<>> ->
             ConnPacket1#mqtt_packet_connect{username = Username, password = Password};
         _ ->
-            ?PARSE_ERR(malformed_connect_data)
+            ?PARSE_ERR(#{
+                cause => malformed_connect,
+                unexpected_trailing_bytes => size(Rest7)
+            })
     end;
-parse_connect2(_ProtoName, _, _) ->
+parse_connect2(_ProtoName, Bin, _StrictMode) ->
-    ?PARSE_ERR(malformed_connect_header).
+    %% sent less than 32 bytes
+    ?PARSE_ERR(#{cause => malformed_connect, header_bytes => Bin}).
 
 parse_packet(
     #mqtt_packet_header{type = ?CONNECT},
@@ -361,7 +366,7 @@ parse_packet(
     Bin,
     #{strict_mode := StrictMode, version := Ver}
 ) ->
-    {TopicName, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_topic),
+    {TopicName, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_topic),
     {PacketId, Rest1} =
         case QoS of
             ?QOS_0 -> {undefined, Rest};
@@ -473,7 +478,7 @@ parse_packet(
     {Properties, <<>>} = parse_properties(Rest, ?MQTT_PROTO_V5, StrictMode),
     #mqtt_packet_auth{reason_code = ReasonCode, properties = Properties};
 parse_packet(Header, _FrameBin, _Options) ->
-    ?PARSE_ERR(#{hint => malformed_packet, header_type => Header#mqtt_packet_header.type}).
+    ?PARSE_ERR(#{cause => malformed_packet, header_type => Header#mqtt_packet_header.type}).
 
 parse_will_message(
     Packet = #mqtt_packet_connect{
@@ -484,8 +489,8 @@ parse_will_message(
     StrictMode
 ) ->
     {Props, Rest} = parse_properties(Bin, Ver, StrictMode),
-    {Topic, Rest1} = parse_utf8_string_with_hint(Rest, StrictMode, invalid_topic),
+    {Topic, Rest1} = parse_utf8_string_with_cause(Rest, StrictMode, invalid_topic),
-    {Payload, Rest2} = parse_binary_data(Rest1),
+    {Payload, Rest2} = parse_will_payload(Rest1),
     {
         Packet#mqtt_packet_connect{
             will_props = Props,
@@ -517,7 +522,7 @@ parse_properties(Bin, ?MQTT_PROTO_V5, StrictMode) ->
             {parse_property(PropsBin, #{}, StrictMode), Rest1};
         _ ->
             ?PARSE_ERR(#{
-                hint => user_property_not_enough_bytes,
+                cause => user_property_not_enough_bytes,
                 parsed_key_length => Len,
                 remaining_bytes_length => byte_size(Rest)
             })
@@ -530,10 +535,10 @@ parse_property(<<16#01, Val, Bin/binary>>, Props, StrictMode) ->
 parse_property(<<16#02, Val:32/big, Bin/binary>>, Props, StrictMode) ->
     parse_property(Bin, Props#{'Message-Expiry-Interval' => Val}, StrictMode);
 parse_property(<<16#03, Bin/binary>>, Props, StrictMode) ->
-    {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_content_type),
+    {Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_content_type),
     parse_property(Rest, Props#{'Content-Type' => Val}, StrictMode);
 parse_property(<<16#08, Bin/binary>>, Props, StrictMode) ->
-    {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_response_topic),
+    {Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_response_topic),
     parse_property(Rest, Props#{'Response-Topic' => Val}, StrictMode);
 parse_property(<<16#09, Len:16/big, Val:Len/binary, Bin/binary>>, Props, StrictMode) ->
     parse_property(Bin, Props#{'Correlation-Data' => Val}, StrictMode);
@@ -543,12 +548,12 @@ parse_property(<<16#0B, Bin/binary>>, Props, StrictMode) ->
 parse_property(<<16#11, Val:32/big, Bin/binary>>, Props, StrictMode) ->
     parse_property(Bin, Props#{'Session-Expiry-Interval' => Val}, StrictMode);
 parse_property(<<16#12, Bin/binary>>, Props, StrictMode) ->
-    {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_assigned_client_id),
+    {Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_assigned_client_id),
     parse_property(Rest, Props#{'Assigned-Client-Identifier' => Val}, StrictMode);
 parse_property(<<16#13, Val:16, Bin/binary>>, Props, StrictMode) ->
     parse_property(Bin, Props#{'Server-Keep-Alive' => Val}, StrictMode);
 parse_property(<<16#15, Bin/binary>>, Props, StrictMode) ->
-    {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_authn_method),
+    {Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_authn_method),
     parse_property(Rest, Props#{'Authentication-Method' => Val}, StrictMode);
 parse_property(<<16#16, Len:16/big, Val:Len/binary, Bin/binary>>, Props, StrictMode) ->
     parse_property(Bin, Props#{'Authentication-Data' => Val}, StrictMode);
@@ -559,13 +564,13 @@ parse_property(<<16#18, Val:32, Bin/binary>>, Props, StrictMode) ->
 parse_property(<<16#19, Val, Bin/binary>>, Props, StrictMode) ->
     parse_property(Bin, Props#{'Request-Response-Information' => Val}, StrictMode);
 parse_property(<<16#1A, Bin/binary>>, Props, StrictMode) ->
-    {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_response_info),
+    {Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_response_info),
     parse_property(Rest, Props#{'Response-Information' => Val}, StrictMode);
 parse_property(<<16#1C, Bin/binary>>, Props, StrictMode) ->
-    {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_server_reference),
+    {Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_server_reference),
     parse_property(Rest, Props#{'Server-Reference' => Val}, StrictMode);
 parse_property(<<16#1F, Bin/binary>>, Props, StrictMode) ->
-    {Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_reason_string),
+    {Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_reason_string),
     parse_property(Rest, Props#{'Reason-String' => Val}, StrictMode);
 parse_property(<<16#21, Val:16/big, Bin/binary>>, Props, StrictMode) ->
     parse_property(Bin, Props#{'Receive-Maximum' => Val}, StrictMode);
@@ -634,7 +639,7 @@ parse_utf8_pair(<<LenK:16/big, Rest/binary>>, _StrictMode) when
     LenK > byte_size(Rest)
 ->
     ?PARSE_ERR(#{
-        hint => user_property_not_enough_bytes,
+        cause => user_property_not_enough_bytes,
         parsed_key_length => LenK,
         remaining_bytes_length => byte_size(Rest)
     });
@@ -643,7 +648,7 @@ parse_utf8_pair(<<LenK:16/big, _Key:LenK/binary, LenV:16/big, Rest/binary>>, _St
     LenV > byte_size(Rest)
 ->
     ?PARSE_ERR(#{
-        hint => malformed_user_property_value,
+        cause => malformed_user_property_value,
         parsed_key_length => LenK,
         parsed_value_length => LenV,
         remaining_bytes_length => byte_size(Rest)
@@ -652,16 +657,16 @@ parse_utf8_pair(Bin, _StrictMode) when
     4 > byte_size(Bin)
 ->
     ?PARSE_ERR(#{
-        hint => user_property_not_enough_bytes,
+        cause => user_property_not_enough_bytes,
         total_bytes => byte_size(Bin)
     }).
 
-parse_utf8_string_with_hint(Bin, StrictMode, Hint) ->
+parse_utf8_string_with_cause(Bin, StrictMode, Cause) ->
     try
         parse_utf8_string(Bin, StrictMode)
     catch
         throw:{?FRAME_PARSE_ERROR, Reason} when is_map(Reason) ->
-            ?PARSE_ERR(Reason#{hint => Hint})
+            ?PARSE_ERR(Reason#{cause => Cause})
     end.
 
 parse_optional(Bin, F, true) ->
@@ -677,7 +682,7 @@ parse_utf8_string(<<Len:16/big, Rest/binary>>, _) when
     Len > byte_size(Rest)
 ->
     ?PARSE_ERR(#{
-        hint => malformed_utf8_string,
+        cause => malformed_utf8_string,
         parsed_length => Len,
         remaining_bytes_length => byte_size(Rest)
     });
@@ -686,20 +691,24 @@ parse_utf8_string(Bin, _) when
 ->
     ?PARSE_ERR(#{reason => malformed_utf8_string_length}).
 
-parse_binary_data(<<Len:16/big, Data:Len/binary, Rest/binary>>) ->
+parse_will_payload(<<Len:16/big, Data:Len/binary, Rest/binary>>) ->
     {Data, Rest};
-parse_binary_data(<<Len:16/big, Rest/binary>>) when
+parse_will_payload(<<Len:16/big, Rest/binary>>) when
     Len > byte_size(Rest)
 ->
     ?PARSE_ERR(#{
-        hint => malformed_binary_data,
+        cause => malformed_will_payload,
         parsed_length => Len,
-        remaining_bytes_length => byte_size(Rest)
+        remaining_bytes => byte_size(Rest)
     });
-parse_binary_data(Bin) when
+parse_will_payload(Bin) when
     2 > byte_size(Bin)
 ->
-    ?PARSE_ERR(malformed_binary_data_length).
+    ?PARSE_ERR(#{
+        cause => malformed_will_payload,
+        length_bytes => size(Bin),
+        expected_bytes => 2
+    }).
 
 %%--------------------------------------------------------------------
 %% Serialize MQTT Packet
@@ -772,6 +781,7 @@ serialize_variable(
         proto_name = ProtoName,
         proto_ver = ProtoVer,
         %% For bridge mode, non-standard implementation
+        %% Invented by mosquitto, named 'try_private': https://mosquitto.org/man/mosquitto-conf-5.html
         is_bridge = IsBridge,
         clean_start = CleanStart,
         will_flag = WillFlag,

apps/emqx/src/emqx_gc.erl

@@ -86,11 +86,11 @@ do_run([{K, N} | T], St) ->
     end.
 
 %% @doc Info of GC state.
--spec info(maybe(gc_state())) -> maybe(map()).
+-spec info(option(gc_state())) -> option(map()).
 info(?GCS(St)) -> St.
 
 %% @doc Reset counters to zero.
--spec reset(maybe(gc_state())) -> gc_state().
+-spec reset(option(gc_state())) -> gc_state().
 reset(?GCS(St)) ->
     ?GCS(do_reset(St)).
 

apps/emqx/src/emqx_hooks.erl

@@ -76,7 +76,7 @@
 
 -record(callback, {
     action :: action(),
-    filter :: maybe(filter()),
+    filter :: option(filter()),
     priority :: integer()
 }).
 

apps/emqx/src/emqx_kernel_sup.erl

@@ -40,7 +40,8 @@ init([]) ->
             child_spec(emqx_authn_authz_metrics_sup, supervisor),
             child_spec(emqx_ocsp_cache, worker),
             child_spec(emqx_crl_cache, worker),
-            child_spec(emqx_tls_lib_sup, supervisor)
+            child_spec(emqx_tls_lib_sup, supervisor),
+            child_spec(emqx_log_throttler, worker)
         ]
     }}.
 

apps/emqx/src/emqx_log_throttler.erl

@@ -0,0 +1,151 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_log_throttler).
+
+-behaviour(gen_server).
+
+-include("logger.hrl").
+-include("types.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-export([start_link/0]).
+
+%% throttler API
+-export([allow/2]).
+
+%% gen_server callbacks
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    terminate/2,
+    code_change/3
+]).
+
+-define(SEQ_ID(Msg), {?MODULE, Msg}).
+-define(NEW_SEQ, atomics:new(1, [{signed, false}])).
+-define(GET_SEQ(Msg), persistent_term:get(?SEQ_ID(Msg), undefined)).
+-define(RESET_SEQ(SeqRef), atomics:put(SeqRef, 1, 0)).
+-define(INC_SEQ(SeqRef), atomics:add(SeqRef, 1, 1)).
+-define(GET_DROPPED(SeqRef), atomics:get(SeqRef, 1) - 1).
+-define(IS_ALLOWED(SeqRef), atomics:add_get(SeqRef, 1, 1) =:= 1).
+
+-define(NEW_THROTTLE(Msg, SeqRef), persistent_term:put(?SEQ_ID(Msg), SeqRef)).
+
+-define(MSGS_LIST, emqx:get_config([log, throttling, msgs], [])).
+-define(TIME_WINDOW_MS, timer:seconds(emqx:get_config([log, throttling, time_window], 60))).
+
+-spec allow(logger:level(), atom()) -> boolean().
+allow(debug, _Msg) ->
+    true;
+allow(_Level, Msg) when is_atom(Msg) ->
+    Seq = persistent_term:get(?SEQ_ID(Msg), undefined),
+    case Seq of
+        undefined ->
+            %% This is either a race condition (emqx_log_throttler is not started yet)
+            %% or a developer mistake (msg used in ?SLOG_THROTTLE/2,3 macro is
+            %% not added to the default value of `log.throttling.msgs`.
+            ?SLOG(info, #{
+                msg => "missing_log_throttle_sequence",
+                throttled_msg => Msg
+            }),
+            true;
+        SeqRef ->
+            ?IS_ALLOWED(SeqRef)
+    end.
+
+-spec start_link() -> startlink_ret().
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+%%--------------------------------------------------------------------
+%% gen_server callbacks
+%%--------------------------------------------------------------------
+
+init([]) ->
+    ok = lists:foreach(fun(Msg) -> ?NEW_THROTTLE(Msg, ?NEW_SEQ) end, ?MSGS_LIST),
+    CurrentPeriodMs = ?TIME_WINDOW_MS,
+    TimerRef = schedule_refresh(CurrentPeriodMs),
+    {ok, #{timer_ref => TimerRef, current_period_ms => CurrentPeriodMs}}.
+
+handle_call(Req, _From, State) ->
+    ?SLOG(error, #{msg => "unexpected_call", call => Req}),
+    {reply, ignored, State}.
+
+handle_cast(Msg, State) ->
+    ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
+    {noreply, State}.
+
+handle_info(refresh, #{current_period_ms := PeriodMs} = State) ->
+    Msgs = ?MSGS_LIST,
+    DroppedStats = lists:foldl(
+        fun(Msg, Acc) ->
+            case ?GET_SEQ(Msg) of
+                %% Should not happen, unless the static ids list is updated at run-time.
+                undefined ->
+                    ?NEW_THROTTLE(Msg, ?NEW_SEQ),
+                    ?tp(log_throttler_new_msg, #{throttled_msg => Msg}),
+                    Acc;
+                SeqRef ->
+                    Dropped = ?GET_DROPPED(SeqRef),
+                    ok = ?RESET_SEQ(SeqRef),
+                    ?tp(log_throttler_dropped, #{dropped_count => Dropped, throttled_msg => Msg}),
+                    maybe_add_dropped(Msg, Dropped, Acc)
+            end
+        end,
+        #{},
+        Msgs
+    ),
+    maybe_log_dropped(DroppedStats, PeriodMs),
+    NewPeriodMs = ?TIME_WINDOW_MS,
+    State1 = State#{
+        timer_ref => schedule_refresh(NewPeriodMs),
+        current_period_ms => NewPeriodMs
+    },
+    {noreply, State1};
+handle_info(Info, State) ->
+    ?SLOG(error, #{msg => "unxpected_info", info => Info}),
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%--------------------------------------------------------------------
+%% internal functions
+%%--------------------------------------------------------------------
+
+maybe_add_dropped(Msg, Dropped, DroppedAcc) when Dropped > 0 ->
+    DroppedAcc#{Msg => Dropped};
+maybe_add_dropped(_Msg, _Dropped, DroppedAcc) ->
+    DroppedAcc.
+
+maybe_log_dropped(DroppedStats, PeriodMs) when map_size(DroppedStats) > 0 ->
+    ?SLOG(warning, #{
+        msg => "log_events_throttled_during_last_period",
+        dropped => DroppedStats,
+        period => emqx_utils_calendar:human_readable_duration_string(PeriodMs)
+    });
+maybe_log_dropped(_DroppedStats, _PeriodMs) ->
+    ok.
+
+schedule_refresh(PeriodMs) ->
+    ?tp(log_throttler_sched_refresh, #{new_period_ms => PeriodMs}),
+    erlang:send_after(PeriodMs, ?MODULE, refresh).

apps/emqx/src/emqx_maybe.erl

@@ -23,30 +23,30 @@
 -export([define/2]).
 -export([apply/2]).
 
--type t(T) :: maybe(T).
+-type t(T) :: option(T).
 -export_type([t/1]).
 
--spec to_list(maybe(A)) -> [A].
+-spec to_list(option(A)) -> [A].
 to_list(undefined) ->
     [];
 to_list(Term) ->
     [Term].
 
--spec from_list([A]) -> maybe(A).
+-spec from_list([A]) -> option(A).
 from_list([]) ->
     undefined;
 from_list([Term]) ->
     Term.
 
--spec define(maybe(A), B) -> A | B.
+-spec define(option(A), B) -> A | B.
 define(undefined, Term) ->
     Term;
 define(Term, _) ->
     Term.
 
 %% @doc Apply a function to a maybe argument.
--spec apply(fun((A) -> B), maybe(A)) ->
+-spec apply(fun((A) -> B), option(A)) ->
-    maybe(B).
+    option(B).
 apply(_Fun, undefined) ->
     undefined;
 apply(Fun, Term) when is_function(Fun) ->

apps/emqx/src/emqx_message.erl

@@ -65,7 +65,7 @@
 ]).
 
 -export([
-    is_expired/1,
+    is_expired/2,
     update_expiry/1,
     timestamp_now/0
 ]).
@@ -186,7 +186,7 @@ estimate_size(#message{topic = Topic, payload = Payload}) ->
     TopicLengthSize = 2,
     FixedHeaderSize + VarLenSize + TopicLengthSize + TopicSize + PacketIdSize + PayloadSize.
 
--spec id(emqx_types:message()) -> maybe(binary()).
+-spec id(emqx_types:message()) -> option(binary()).
 id(#message{id = Id}) -> Id.
 
 -spec qos(emqx_types:message()) -> emqx_types:qos().
@@ -229,7 +229,7 @@ get_flag(Flag, Msg) ->
 get_flag(Flag, #message{flags = Flags}, Default) ->
     maps:get(Flag, Flags, Default).
 
--spec get_flags(emqx_types:message()) -> maybe(map()).
+-spec get_flags(emqx_types:message()) -> option(map()).
 get_flags(#message{flags = Flags}) -> Flags.
 
 -spec set_flag(emqx_types:flag(), emqx_types:message()) -> emqx_types:message().
@@ -252,7 +252,7 @@ unset_flag(Flag, Msg = #message{flags = Flags}) ->
 set_headers(New, Msg = #message{headers = Old}) when is_map(New) ->
     Msg#message{headers = maps:merge(Old, New)}.
 
--spec get_headers(emqx_types:message()) -> maybe(map()).
+-spec get_headers(emqx_types:message()) -> option(map()).
 get_headers(Msg) -> Msg#message.headers.
 
 -spec get_header(term(), emqx_types:message()) -> term().
@@ -273,14 +273,20 @@ remove_header(Hdr, Msg = #message{headers = Headers}) ->
         false -> Msg
     end.
 
--spec is_expired(emqx_types:message()) -> boolean().
+-spec is_expired(emqx_types:message(), atom()) -> boolean().
-is_expired(#message{
+is_expired(
+    #message{
         headers = #{properties := #{'Message-Expiry-Interval' := Interval}},
         timestamp = CreatedAt
-}) ->
+    },
+    _
+) ->
     elapsed(CreatedAt) > timer:seconds(Interval);
-is_expired(_Msg) ->
+is_expired(#message{timestamp = CreatedAt}, Zone) ->
-    false.
+    case emqx_config:get_zone_conf(Zone, [mqtt, message_expiry_interval], infinity) of
+        infinity -> false;
+        Interval -> elapsed(CreatedAt) > Interval
+    end.
 
 -spec update_expiry(emqx_types:message()) -> emqx_types:message().
 update_expiry(

apps/emqx/src/emqx_mountpoint.erl

@@ -32,7 +32,7 @@
 
 -type mountpoint() :: binary().
 
--spec mount(maybe(mountpoint()), Any) -> Any when
+-spec mount(option(mountpoint()), Any) -> Any when
     Any ::
         emqx_types:topic()
         | emqx_types:share()
@@ -47,7 +47,7 @@ mount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) ->
 mount(MountPoint, TopicFilters) when is_list(TopicFilters) ->
     [{prefix_maybe_share(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters].
 
--spec prefix_maybe_share(maybe(mountpoint()), Any) -> Any when
+-spec prefix_maybe_share(option(mountpoint()), Any) -> Any when
     Any ::
         emqx_types:topic()
         | emqx_types:share().
@@ -60,7 +60,7 @@ prefix_maybe_share(MountPoint, #share{group = Group, topic = Topic}) when
 ->
     #share{group = Group, topic = prefix_maybe_share(MountPoint, Topic)}.
 
--spec unmount(maybe(mountpoint()), Any) -> Any when
+-spec unmount(option(mountpoint()), Any) -> Any when
     Any ::
         emqx_types:topic()
         | emqx_types:share()
@@ -84,7 +84,7 @@ unmount_maybe_share(MountPoint, TopicFilter = #share{topic = Topic}) when
 ->
     TopicFilter#share{topic = unmount_maybe_share(MountPoint, Topic)}.
 
--spec replvar(maybe(mountpoint()), map()) -> maybe(mountpoint()).
+-spec replvar(option(mountpoint()), map()) -> option(mountpoint()).
 replvar(undefined, _Vars) ->
     undefined;
 replvar(MountPoint, Vars) ->

apps/emqx/src/emqx_mqueue.erl

@@ -189,7 +189,7 @@ stats(#mqueue{max_len = MaxLen, dropped = Dropped} = MQ) ->
     [{len, len(MQ)}, {max_len, MaxLen}, {dropped, Dropped}].
 
 %% @doc Enqueue a message.
--spec in(message(), mqueue()) -> {maybe(message()), mqueue()}.
+-spec in(message(), mqueue()) -> {option(message()), mqueue()}.
 in(Msg = #message{qos = ?QOS_0}, MQ = #mqueue{store_qos0 = false}) ->
     {_Dropped = Msg, MQ};
 in(

apps/emqx/src/emqx_packet.erl

@@ -493,8 +493,8 @@ format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, Pa
         "" -> [HeaderIO, ")"];
         VarIO -> [HeaderIO, ", ", VarIO, ")"]
     end;
-%% receive a frame error packet, such as {frame_error,frame_too_large} or
+%% receive a frame error packet, such as {frame_error,#{cause := frame_too_large}} or
-%% {frame_error,#{expected => <<"'MQTT' or 'MQIsdp'">>,hint => invalid_proto_name,received => <<"bad_name">>}}
+%% {frame_error,#{expected => <<"'MQTT' or 'MQIsdp'">>,cause => invalid_proto_name,received => <<"bad_name">>}}
 format(FrameError, _PayloadEncode) ->
     lists:flatten(io_lib:format("~tp", [FrameError])).
 

apps/emqx/src/emqx_pd.erl

@@ -48,7 +48,7 @@ get_counter(Key) ->
         Cnt -> Cnt
     end.
 
--spec inc_counter(key(), number()) -> maybe(number()).
+-spec inc_counter(key(), number()) -> option(number()).
 inc_counter(Key, Inc) ->
     put(Key, get_counter(Key) + Inc).
 

apps/emqx/src/emqx_persistent_message.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2021-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -61,7 +61,11 @@ force_ds() ->
     emqx_config:get([session_persistence, force_persistence]).
 
 storage_backend(#{
-    builtin := #{enable := true, n_shards := NShards, replication_factor := ReplicationFactor}
+    builtin := #{
+        enable := true,
+        n_shards := NShards,
+        replication_factor := ReplicationFactor
+    }
 }) ->
     #{
         backend => builtin,
@@ -93,7 +97,7 @@ needs_persistence(Msg) ->
 
 -spec store_message(emqx_types:message()) -> emqx_ds:store_batch_result().
 store_message(Msg) ->
-    emqx_ds:store_batch(?PERSISTENT_MESSAGE_DB, [Msg]).
+    emqx_ds:store_batch(?PERSISTENT_MESSAGE_DB, [Msg], #{sync => false}).
 
 has_subscribers(#message{topic = Topic}) ->
     emqx_persistent_session_ds_router:has_any_route(Topic).

apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl

@@ -0,0 +1,157 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_persistent_message_ds_gc_worker).
+
+-behaviour(gen_server).
+
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("stdlib/include/qlc.hrl").
+-include_lib("stdlib/include/ms_transform.hrl").
+
+-include("emqx_persistent_session_ds.hrl").
+
+%% API
+-export([
+    start_link/0,
+    gc/0
+]).
+
+%% `gen_server' API
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+%% call/cast/info records
+-record(gc, {}).
+
+%%--------------------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------------------
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+%% For testing or manual ops
+gc() ->
+    gen_server:call(?MODULE, #gc{}, infinity).
+
+%%--------------------------------------------------------------------------------
+%% `gen_server' API
+%%--------------------------------------------------------------------------------
+
+init(_Opts) ->
+    ensure_gc_timer(),
+    State = #{},
+    {ok, State}.
+
+handle_call(#gc{}, _From, State) ->
+    maybe_gc(),
+    {reply, ok, State};
+handle_call(_Call, _From, State) ->
+    {reply, error, State}.
+
+handle_cast(_Cast, State) ->
+    {noreply, State}.
+
+handle_info(#gc{}, State) ->
+    try_gc(),
+    ensure_gc_timer(),
+    {noreply, State};
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+%%--------------------------------------------------------------------------------
+%% Internal fns
+%%--------------------------------------------------------------------------------
+
+ensure_gc_timer() ->
+    Timeout = emqx_config:get([session_persistence, message_retention_period]),
+    _ = erlang:send_after(Timeout, self(), #gc{}),
+    ok.
+
+try_gc() ->
+    %% Only cores should run GC.
+    CoreNodes = mria_membership:running_core_nodelist(),
+    Res = global:trans(
+        {?MODULE, self()},
+        fun maybe_gc/0,
+        CoreNodes,
+        %% Note: we set retries to 1 here because, in rare occasions, GC might start at the
+        %% same time in more than one node, and each one will abort the other.  By allowing
+        %% one retry, at least one node will (hopefully) get to enter the transaction and
+        %% the other will abort.  If GC runs too fast, both nodes might run in sequence.
+        %% But, in that case, GC is clearly not too costly, and that shouldn't be a problem,
+        %% resource-wise.
+        _Retries = 1
+    ),
+    case Res of
+        aborted ->
+            ?tp(ds_message_gc_lock_taken, #{}),
+            ok;
+        ok ->
+            ok
+    end.
+
+now_ms() ->
+    erlang:system_time(millisecond).
+
+maybe_gc() ->
+    AllGens = emqx_ds:list_generations_with_lifetimes(?PERSISTENT_MESSAGE_DB),
+    NowMS = now_ms(),
+    RetentionPeriod = emqx_config:get([session_persistence, message_retention_period]),
+    TimeThreshold = NowMS - RetentionPeriod,
+    maybe_create_new_generation(AllGens, TimeThreshold),
+    ?tp_span(
+        ps_message_gc,
+        #{},
+        begin
+            ExpiredGens =
+                maps:filter(
+                    fun(_GenId, #{until := Until}) ->
+                        is_number(Until) andalso Until =< TimeThreshold
+                    end,
+                    AllGens
+                ),
+            ExpiredGenIds = maps:keys(ExpiredGens),
+            lists:foreach(
+                fun(GenId) ->
+                    ok = emqx_ds:drop_generation(?PERSISTENT_MESSAGE_DB, GenId),
+                    ?tp(message_gc_generation_dropped, #{gen_id => GenId})
+                end,
+                ExpiredGenIds
+            )
+        end
+    ).
+
+maybe_create_new_generation(AllGens, TimeThreshold) ->
+    NeedNewGen =
+        lists:all(
+            fun({_GenId, #{created_at := CreatedAt}}) ->
+                CreatedAt =< TimeThreshold
+            end,
+            maps:to_list(AllGens)
+        ),
+    case NeedNewGen of
+        false ->
+            ?tp(ps_message_gc_too_early, #{}),
+            ok;
+        true ->
+            ok = emqx_ds:add_generation(?PERSISTENT_MESSAGE_DB),
+            ?tp(ps_message_gc_added_gen, #{})
+    end.

apps/emqx/src/emqx_persistent_message_ds_replayer.erl

@@ -1,795 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
-%% @doc This module implements the routines for replaying streams of
-%% messages.
--module(emqx_persistent_message_ds_replayer).
-
-%% API:
--export([new/0, open/1, next_packet_id/1, n_inflight/1]).
-
--export([poll/4, replay/2, commit_offset/4]).
-
--export([seqno_to_packet_id/1, packet_id_to_seqno/2]).
-
--export([committed_until/2]).
-
-%% internal exports:
--export([]).
-
--export_type([inflight/0, seqno/0]).
-
--include_lib("emqx/include/logger.hrl").
--include_lib("emqx/include/emqx_mqtt.hrl").
--include_lib("emqx_utils/include/emqx_message.hrl").
--include("emqx_persistent_session_ds.hrl").
-
--ifdef(TEST).
--include_lib("proper/include/proper.hrl").
--include_lib("eunit/include/eunit.hrl").
--endif.
-
--define(EPOCH_SIZE, 16#10000).
-
--define(ACK, 0).
--define(COMP, 1).
-
--define(TRACK_FLAG(WHICH), (1 bsl WHICH)).
--define(TRACK_FLAGS_ALL, ?TRACK_FLAG(?ACK) bor ?TRACK_FLAG(?COMP)).
--define(TRACK_FLAGS_NONE, 0).
-
-%%================================================================================
-%% Type declarations
-%%================================================================================
-
-%% Note: sequence numbers are monotonic; they don't wrap around:
--type seqno() :: non_neg_integer().
-
--type track() :: ack | comp.
--type commit_type() :: rec.
-
--record(inflight, {
-    next_seqno = 1 :: seqno(),
-    commits = #{ack => 1, comp => 1, rec => 1} :: #{track() | commit_type() => seqno()},
-    %% Ranges are sorted in ascending order of their sequence numbers.
-    offset_ranges = [] :: [ds_pubrange()]
-}).
-
--opaque inflight() :: #inflight{}.
-
--type message() :: emqx_types:message().
--type replies() :: [emqx_session:reply()].
-
--type preproc_fun() :: fun((message()) -> message() | [message()]).
-
-%%================================================================================
-%% API funcions
-%%================================================================================
-
--spec new() -> inflight().
-new() ->
-    #inflight{}.
-
--spec open(emqx_persistent_session_ds:id()) -> inflight().
-open(SessionId) ->
-    {Ranges, RecUntil} = ro_transaction(
-        fun() -> {get_ranges(SessionId), get_committed_offset(SessionId, rec)} end
-    ),
-    {Commits, NextSeqno} = compute_inflight_range(Ranges),
-    #inflight{
-        commits = Commits#{rec => RecUntil},
-        next_seqno = NextSeqno,
-        offset_ranges = Ranges
-    }.
-
--spec next_packet_id(inflight()) -> {emqx_types:packet_id(), inflight()}.
-next_packet_id(Inflight0 = #inflight{next_seqno = LastSeqno}) ->
-    Inflight = Inflight0#inflight{next_seqno = next_seqno(LastSeqno)},
-    {seqno_to_packet_id(LastSeqno), Inflight}.
-
--spec n_inflight(inflight()) -> non_neg_integer().
-n_inflight(#inflight{offset_ranges = Ranges}) ->
-    %% TODO
-    %% This is not very efficient. Instead, we can take the maximum of
-    %% `range_size(AckedUntil, NextSeqno)` and `range_size(CompUntil, NextSeqno)`.
-    %% This won't be exact number but a pessimistic estimate, but this way we
-    %% will penalize clients that PUBACK QoS 1 messages but don't PUBCOMP QoS 2
-    %% messages for some reason. For that to work, we need to additionally track
-    %% actual `AckedUntil` / `CompUntil` during `commit_offset/4`.
-    lists:foldl(
-        fun
-            (#ds_pubrange{type = ?T_CHECKPOINT}, N) ->
-                N;
-            (#ds_pubrange{type = ?T_INFLIGHT} = Range, N) ->
-                N + range_size(Range)
-        end,
-        0,
-        Ranges
-    ).
-
--spec replay(preproc_fun(), inflight()) -> {emqx_session:replies(), inflight()}.
-replay(PreprocFunFun, Inflight0 = #inflight{offset_ranges = Ranges0, commits = Commits}) ->
-    {Ranges, Replies} = lists:mapfoldr(
-        fun(Range, Acc) ->
-            replay_range(PreprocFunFun, Commits, Range, Acc)
-        end,
-        [],
-        Ranges0
-    ),
-    Inflight = Inflight0#inflight{offset_ranges = Ranges},
-    {Replies, Inflight}.
-
--spec commit_offset(emqx_persistent_session_ds:id(), Offset, emqx_types:packet_id(), inflight()) ->
-    {_IsValidOffset :: boolean(), inflight()}
-when
-    Offset :: track() | commit_type().
-commit_offset(
-    SessionId,
-    Track,
-    PacketId,
-    Inflight0 = #inflight{commits = Commits}
-) when Track == ack orelse Track == comp ->
-    case validate_commit(Track, PacketId, Inflight0) of
-        CommitUntil when is_integer(CommitUntil) ->
-            %% TODO
-            %% We do not preserve `CommitUntil` in the database. Instead, we discard
-            %% fully acked ranges from the database. In effect, this means that the
-            %% most recent `CommitUntil` the client has sent may be lost in case of a
-            %% crash or client loss.
-            Inflight1 = Inflight0#inflight{commits = Commits#{Track := CommitUntil}},
-            Inflight = discard_committed(SessionId, Inflight1),
-            {true, Inflight};
-        false ->
-            {false, Inflight0}
-    end;
-commit_offset(
-    SessionId,
-    CommitType = rec,
-    PacketId,
-    Inflight0 = #inflight{commits = Commits}
-) ->
-    case validate_commit(CommitType, PacketId, Inflight0) of
-        CommitUntil when is_integer(CommitUntil) ->
-            update_committed_offset(SessionId, CommitType, CommitUntil),
-            Inflight = Inflight0#inflight{commits = Commits#{CommitType := CommitUntil}},
-            {true, Inflight};
-        false ->
-            {false, Inflight0}
-    end.
-
--spec poll(preproc_fun(), emqx_persistent_session_ds:id(), inflight(), pos_integer()) ->
-    {emqx_session:replies(), inflight()}.
-poll(PreprocFun, SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < ?EPOCH_SIZE ->
-    MinBatchSize = emqx_config:get([session_persistence, min_batch_size]),
-    FetchThreshold = min(MinBatchSize, ceil(WindowSize / 2)),
-    FreeSpace = WindowSize - n_inflight(Inflight0),
-    case FreeSpace >= FetchThreshold of
-        false ->
-            %% TODO: this branch is meant to avoid fetching data from
-            %% the DB in chunks that are too small. However, this
-            %% logic is not exactly good for the latency. Can the
-            %% client get stuck even?
-            {[], Inflight0};
-        true ->
-            %% TODO: Wrap this in `mria:async_dirty/2`?
-            Checkpoints = find_checkpoints(Inflight0#inflight.offset_ranges),
-            StreamGroups = group_streams(get_streams(SessionId)),
-            {Publihes, Inflight} =
-                fetch(PreprocFun, SessionId, Inflight0, Checkpoints, StreamGroups, FreeSpace, []),
-            %% Discard now irrelevant QoS0-only ranges, if any.
-            {Publihes, discard_committed(SessionId, Inflight)}
-    end.
-
-%% Which seqno this track is committed until.
-%% "Until" means this is first seqno that is _not yet committed_ for this track.
--spec committed_until(track() | commit_type(), inflight()) -> seqno().
-committed_until(Track, #inflight{commits = Commits}) ->
-    maps:get(Track, Commits).
-
--spec seqno_to_packet_id(seqno()) -> emqx_types:packet_id() | 0.
-seqno_to_packet_id(Seqno) ->
-    Seqno rem ?EPOCH_SIZE.
-
-%% Reconstruct session counter by adding most significant bits from
-%% the current counter to the packet id.
--spec packet_id_to_seqno(emqx_types:packet_id(), inflight()) -> seqno().
-packet_id_to_seqno(PacketId, #inflight{next_seqno = NextSeqno}) ->
-    packet_id_to_seqno_(NextSeqno, PacketId).
-
-%%================================================================================
-%% Internal exports
-%%================================================================================
-
-%%================================================================================
-%% Internal functions
-%%================================================================================
-
-compute_inflight_range([]) ->
-    {#{ack => 1, comp => 1}, 1};
-compute_inflight_range(Ranges) ->
-    _RangeLast = #ds_pubrange{until = LastSeqno} = lists:last(Ranges),
-    AckedUntil = find_committed_until(ack, Ranges),
-    CompUntil = find_committed_until(comp, Ranges),
-    Commits = #{
-        ack => emqx_maybe:define(AckedUntil, LastSeqno),
-        comp => emqx_maybe:define(CompUntil, LastSeqno)
-    },
-    {Commits, LastSeqno}.
-
-find_committed_until(Track, Ranges) ->
-    RangesUncommitted = lists:dropwhile(
-        fun(Range) ->
-            case Range of
-                #ds_pubrange{type = ?T_CHECKPOINT} ->
-                    true;
-                #ds_pubrange{type = ?T_INFLIGHT, tracks = Tracks} ->
-                    not has_track(Track, Tracks)
-            end
-        end,
-        Ranges
-    ),
-    case RangesUncommitted of
-        [#ds_pubrange{id = {_, CommittedUntil, _StreamRef}} | _] ->
-            CommittedUntil;
-        [] ->
-            undefined
-    end.
-
--spec get_ranges(emqx_persistent_session_ds:id()) -> [ds_pubrange()].
-get_ranges(SessionId) ->
-    Pat = erlang:make_tuple(
-        record_info(size, ds_pubrange),
-        '_',
-        [{1, ds_pubrange}, {#ds_pubrange.id, {SessionId, '_', '_'}}]
-    ),
-    mnesia:match_object(?SESSION_PUBRANGE_TAB, Pat, read).
-
-fetch(PreprocFun, SessionId, Inflight0, CPs, Groups, N, Acc) when N > 0, Groups =/= [] ->
-    #inflight{next_seqno = FirstSeqno, offset_ranges = Ranges} = Inflight0,
-    {Stream, Groups2} = get_the_first_stream(Groups),
-    case get_next_n_messages_from_stream(Stream, CPs, N) of
-        [] ->
-            fetch(PreprocFun, SessionId, Inflight0, CPs, Groups2, N, Acc);
-        {ItBegin, ItEnd, Messages} ->
-            %% We need to preserve the iterator pointing to the beginning of the
-            %% range, so that we can replay it if needed.
-            {Publishes, UntilSeqno} = publish_fetch(PreprocFun, FirstSeqno, Messages),
-            Size = range_size(FirstSeqno, UntilSeqno),
-            Range0 = #ds_pubrange{
-                id = {SessionId, FirstSeqno, Stream#ds_stream.ref},
-                type = ?T_INFLIGHT,
-                tracks = compute_pub_tracks(Publishes),
-                until = UntilSeqno,
-                iterator = ItBegin
-            },
-            ok = preserve_range(Range0),
-            %% ...Yet we need to keep the iterator pointing past the end of the
-            %% range, so that we can pick up where we left off: it will become
-            %% `ItBegin` of the next range for this stream.
-            Range = keep_next_iterator(ItEnd, Range0),
-            Inflight = Inflight0#inflight{
-                next_seqno = UntilSeqno,
-                offset_ranges = Ranges ++ [Range]
-            },
-            fetch(PreprocFun, SessionId, Inflight, CPs, Groups2, N - Size, [Publishes | Acc])
-    end;
-fetch(_ReplyFun, _SessionId, Inflight, _CPs, _Groups, _N, Acc) ->
-    Publishes = lists:append(lists:reverse(Acc)),
-    {Publishes, Inflight}.
-
-discard_committed(
-    SessionId,
-    Inflight0 = #inflight{commits = Commits, offset_ranges = Ranges0}
-) ->
-    %% TODO: This could be kept and incrementally updated in the inflight state.
-    Checkpoints = find_checkpoints(Ranges0),
-    %% TODO: Wrap this in `mria:async_dirty/2`?
-    Ranges = discard_committed_ranges(SessionId, Commits, Checkpoints, Ranges0),
-    Inflight0#inflight{offset_ranges = Ranges}.
-
-find_checkpoints(Ranges) ->
-    lists:foldl(
-        fun(#ds_pubrange{id = {_SessionId, _, StreamRef}} = Range, Acc) ->
-            %% For each stream, remember the last range over this stream.
-            Acc#{StreamRef => Range}
-        end,
-        #{},
-        Ranges
-    ).
-
-discard_committed_ranges(
-    SessionId,
-    Commits,
-    Checkpoints,
-    Ranges = [Range = #ds_pubrange{id = {_SessionId, _, StreamRef}} | Rest]
-) ->
-    case discard_committed_range(Commits, Range) of
-        discard ->
-            %% This range has been fully committed.
-            %% Either discard it completely, or preserve the iterator for the next range
-            %% over this stream (i.e. a checkpoint).
-            RangeKept =
-                case maps:get(StreamRef, Checkpoints) of
-                    Range ->
-                        [checkpoint_range(Range)];
-                    _Previous ->
-                        discard_range(Range),
-                        []
-                end,
-            %% Since we're (intentionally) not using transactions here, it's important to
-            %% issue database writes in the same order in which ranges are stored: from
-            %% the oldest to the newest. This is also why we need to compute which ranges
-            %% should become checkpoints before we start writing anything.
-            RangeKept ++ discard_committed_ranges(SessionId, Commits, Checkpoints, Rest);
-        keep ->
-            %% This range has not been fully committed.
-            [Range | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)];
-        keep_all ->
-            %% The rest of ranges (if any) still have uncommitted messages.
-            Ranges;
-        TracksLeft ->
-            %% Only some track has been committed.
-            %% Preserve the uncommitted tracks in the database.
-            RangeKept = Range#ds_pubrange{tracks = TracksLeft},
-            preserve_range(restore_first_iterator(RangeKept)),
-            [RangeKept | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)]
-    end;
-discard_committed_ranges(_SessionId, _Commits, _Checkpoints, []) ->
-    [].
-
-discard_committed_range(_Commits, #ds_pubrange{type = ?T_CHECKPOINT}) ->
-    discard;
-discard_committed_range(
-    #{ack := AckedUntil, comp := CompUntil},
-    #ds_pubrange{until = Until}
-) when Until > AckedUntil andalso Until > CompUntil ->
-    keep_all;
-discard_committed_range(Commits, #ds_pubrange{until = Until, tracks = Tracks}) ->
-    case discard_tracks(Commits, Until, Tracks) of
-        0 ->
-            discard;
-        Tracks ->
-            keep;
-        TracksLeft ->
-            TracksLeft
-    end.
-
-discard_tracks(#{ack := AckedUntil, comp := CompUntil}, Until, Tracks) ->
-    TAck =
-        case Until > AckedUntil of
-            true -> ?TRACK_FLAG(?ACK) band Tracks;
-            false -> 0
-        end,
-    TComp =
-        case Until > CompUntil of
-            true -> ?TRACK_FLAG(?COMP) band Tracks;
-            false -> 0
-        end,
-    TAck bor TComp.
-
-replay_range(
-    PreprocFun,
-    Commits,
-    Range0 = #ds_pubrange{
-        type = ?T_INFLIGHT, id = {_, First, _StreamRef}, until = Until, iterator = It
-    },
-    Acc
-) ->
-    Size = range_size(First, Until),
-    {ok, ItNext, MessagesUnacked} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, It, Size),
-    %% Asserting that range is consistent with the message storage state.
-    {Replies, Until} = publish_replay(PreprocFun, Commits, First, MessagesUnacked),
-    %% Again, we need to keep the iterator pointing past the end of the
-    %% range, so that we can pick up where we left off.
-    Range = keep_next_iterator(ItNext, Range0),
-    {Range, Replies ++ Acc};
-replay_range(_PreprocFun, _Commits, Range0 = #ds_pubrange{type = ?T_CHECKPOINT}, Acc) ->
-    {Range0, Acc}.
-
-validate_commit(
-    Track,
-    PacketId,
-    Inflight = #inflight{commits = Commits, next_seqno = NextSeqno}
-) ->
-    Seqno = packet_id_to_seqno_(NextSeqno, PacketId),
-    CommittedUntil = maps:get(Track, Commits),
-    CommitNext = get_commit_next(Track, Inflight),
-    case Seqno >= CommittedUntil andalso Seqno < CommitNext of
-        true ->
-            next_seqno(Seqno);
-        false ->
-            ?SLOG(warning, #{
-                msg => "out-of-order_commit",
-                track => Track,
-                packet_id => PacketId,
-                commit_seqno => Seqno,
-                committed_until => CommittedUntil,
-                commit_next => CommitNext
-            }),
-            false
-    end.
-
-get_commit_next(ack, #inflight{next_seqno = NextSeqno}) ->
-    NextSeqno;
-get_commit_next(rec, #inflight{next_seqno = NextSeqno}) ->
-    NextSeqno;
-get_commit_next(comp, #inflight{commits = Commits}) ->
-    maps:get(rec, Commits).
-
-publish_fetch(PreprocFun, FirstSeqno, Messages) ->
-    flatmapfoldl(
-        fun({_DSKey, MessageIn}, Acc) ->
-            Message = PreprocFun(MessageIn),
-            publish_fetch(Message, Acc)
-        end,
-        FirstSeqno,
-        Messages
-    ).
-
-publish_fetch(#message{qos = ?QOS_0} = Message, Seqno) ->
-    {{undefined, Message}, Seqno};
-publish_fetch(#message{} = Message, Seqno) ->
-    PacketId = seqno_to_packet_id(Seqno),
-    {{PacketId, Message}, next_seqno(Seqno)};
-publish_fetch(Messages, Seqno) ->
-    flatmapfoldl(fun publish_fetch/2, Seqno, Messages).
-
-publish_replay(PreprocFun, Commits, FirstSeqno, Messages) ->
-    #{ack := AckedUntil, comp := CompUntil, rec := RecUntil} = Commits,
-    flatmapfoldl(
-        fun({_DSKey, MessageIn}, Acc) ->
-            Message = PreprocFun(MessageIn),
-            publish_replay(Message, AckedUntil, CompUntil, RecUntil, Acc)
-        end,
-        FirstSeqno,
-        Messages
-    ).
-
-publish_replay(#message{qos = ?QOS_0}, _, _, _, Seqno) ->
-    %% QoS 0 (at most once) messages should not be replayed.
-    {[], Seqno};
-publish_replay(#message{qos = Qos} = Message, AckedUntil, CompUntil, RecUntil, Seqno) ->
-    case Qos of
-        ?QOS_1 when Seqno < AckedUntil ->
-            %% This message has already been acked, so we can skip it.
-            %% We still need to advance seqno, because previously we assigned this message
-            %% a unique Packet Id.
-            {[], next_seqno(Seqno)};
-        ?QOS_2 when Seqno < CompUntil ->
-            %% This message's flow has already been fully completed, so we can skip it.
-            %% We still need to advance seqno, because previously we assigned this message
-            %% a unique Packet Id.
-            {[], next_seqno(Seqno)};
-        ?QOS_2 when Seqno < RecUntil ->
-            %% This message's flow has been partially completed, we need to resend a PUBREL.
-            PacketId = seqno_to_packet_id(Seqno),
-            Pub = {pubrel, PacketId},
-            {Pub, next_seqno(Seqno)};
-        _ ->
-            %% This message flow hasn't been acked and/or received, we need to resend it.
-            PacketId = seqno_to_packet_id(Seqno),
-            Pub = {PacketId, emqx_message:set_flag(dup, true, Message)},
-            {Pub, next_seqno(Seqno)}
-    end;
-publish_replay([], _, _, _, Seqno) ->
-    {[], Seqno};
-publish_replay(Messages, AckedUntil, CompUntil, RecUntil, Seqno) ->
-    flatmapfoldl(
-        fun(Message, Acc) ->
-            publish_replay(Message, AckedUntil, CompUntil, RecUntil, Acc)
-        end,
-        Seqno,
-        Messages
-    ).
-
--spec compute_pub_tracks(replies()) -> non_neg_integer().
-compute_pub_tracks(Pubs) ->
-    compute_pub_tracks(Pubs, ?TRACK_FLAGS_NONE).
-
-compute_pub_tracks(_Pubs, Tracks = ?TRACK_FLAGS_ALL) ->
-    Tracks;
-compute_pub_tracks([Pub | Rest], Tracks) ->
-    Track =
-        case Pub of
-            {_PacketId, #message{qos = ?QOS_1}} -> ?TRACK_FLAG(?ACK);
-            {_PacketId, #message{qos = ?QOS_2}} -> ?TRACK_FLAG(?COMP);
-            {pubrel, _PacketId} -> ?TRACK_FLAG(?COMP);
-            _ -> ?TRACK_FLAGS_NONE
-        end,
-    compute_pub_tracks(Rest, Track bor Tracks);
-compute_pub_tracks([], Tracks) ->
-    Tracks.
-
-keep_next_iterator(ItNext, Range = #ds_pubrange{iterator = ItFirst, misc = Misc}) ->
-    Range#ds_pubrange{
-        iterator = ItNext,
-        %% We need to keep the first iterator around, in case we need to preserve
-        %% this range again, updating still uncommitted tracks it's part of.
-        misc = Misc#{iterator_first => ItFirst}
-    }.
-
-restore_first_iterator(Range = #ds_pubrange{misc = Misc = #{iterator_first := ItFirst}}) ->
-    Range#ds_pubrange{
-        iterator = ItFirst,
-        misc = maps:remove(iterator_first, Misc)
-    }.
-
--spec preserve_range(ds_pubrange()) -> ok.
-preserve_range(Range = #ds_pubrange{type = ?T_INFLIGHT}) ->
-    mria:dirty_write(?SESSION_PUBRANGE_TAB, Range).
-
-has_track(ack, Tracks) ->
-    (?TRACK_FLAG(?ACK) band Tracks) > 0;
-has_track(comp, Tracks) ->
-    (?TRACK_FLAG(?COMP) band Tracks) > 0.
-
--spec discard_range(ds_pubrange()) -> ok.
-discard_range(#ds_pubrange{id = RangeId}) ->
-    mria:dirty_delete(?SESSION_PUBRANGE_TAB, RangeId).
-
--spec checkpoint_range(ds_pubrange()) -> ds_pubrange().
-checkpoint_range(Range0 = #ds_pubrange{type = ?T_INFLIGHT}) ->
-    Range = Range0#ds_pubrange{type = ?T_CHECKPOINT, misc = #{}},
-    ok = mria:dirty_write(?SESSION_PUBRANGE_TAB, Range),
-    Range;
-checkpoint_range(Range = #ds_pubrange{type = ?T_CHECKPOINT}) ->
-    %% This range should have been checkpointed already.
-    Range.
-
-get_last_iterator(Stream = #ds_stream{ref = StreamRef}, Checkpoints) ->
-    case maps:get(StreamRef, Checkpoints, none) of
-        none ->
-            Stream#ds_stream.beginning;
-        #ds_pubrange{iterator = ItNext} ->
-            ItNext
-    end.
-
--spec get_streams(emqx_persistent_session_ds:id()) -> [ds_stream()].
-get_streams(SessionId) ->
-    mnesia:dirty_read(?SESSION_STREAM_TAB, SessionId).
-
--spec get_committed_offset(emqx_persistent_session_ds:id(), _Name) -> seqno().
-get_committed_offset(SessionId, Name) ->
-    case mnesia:read(?SESSION_COMMITTED_OFFSET_TAB, {SessionId, Name}) of
-        [] ->
-            1;
-        [#ds_committed_offset{until = Seqno}] ->
-            Seqno
-    end.
-
--spec update_committed_offset(emqx_persistent_session_ds:id(), _Name, seqno()) -> ok.
-update_committed_offset(SessionId, Name, Until) ->
-    mria:dirty_write(?SESSION_COMMITTED_OFFSET_TAB, #ds_committed_offset{
-        id = {SessionId, Name}, until = Until
-    }).
-
-next_seqno(Seqno) ->
-    NextSeqno = Seqno + 1,
-    case seqno_to_packet_id(NextSeqno) of
-        0 ->
-            %% We skip sequence numbers that lead to PacketId = 0 to
-            %% simplify math. Note: it leads to occasional gaps in the
-            %% sequence numbers.
-            NextSeqno + 1;
-        _ ->
-            NextSeqno
-    end.
-
-packet_id_to_seqno_(NextSeqno, PacketId) ->
-    Epoch = NextSeqno bsr 16,
-    case (Epoch bsl 16) + PacketId of
-        N when N =< NextSeqno ->
-            N;
-        N ->
-            N - ?EPOCH_SIZE
-    end.
-
-range_size(#ds_pubrange{id = {_, First, _StreamRef}, until = Until}) ->
-    range_size(First, Until).
-
-range_size(FirstSeqno, UntilSeqno) ->
-    %% This function assumes that gaps in the sequence ID occur _only_ when the
-    %% packet ID wraps.
-    Size = UntilSeqno - FirstSeqno,
-    Size + (FirstSeqno bsr 16) - (UntilSeqno bsr 16).
-
-%%================================================================================
-%% stream scheduler
-
-%% group streams by the first position in the rank
--spec group_streams(list(ds_stream())) -> list(list(ds_stream())).
-group_streams(Streams) ->
-    Groups = maps:groups_from_list(
-        fun(#ds_stream{rank = {RankX, _}}) -> RankX end,
-        Streams
-    ),
-    shuffle(maps:values(Groups)).
-
--spec shuffle([A]) -> [A].
-shuffle(L0) ->
-    L1 = lists:map(
-        fun(A) ->
-            %% maybe topic/stream prioritization could be introduced here?
-            {rand:uniform(), A}
-        end,
-        L0
-    ),
-    L2 = lists:sort(L1),
-    {_, L} = lists:unzip(L2),
-    L.
-
-get_the_first_stream([Group | Groups]) ->
-    case get_next_stream_from_group(Group) of
-        {Stream, {sorted, []}} ->
-            {Stream, Groups};
-        {Stream, Group2} ->
-            {Stream, [Group2 | Groups]};
-        undefined ->
-            get_the_first_stream(Groups)
-    end;
-get_the_first_stream([]) ->
-    %% how this possible ?
-    throw(#{reason => no_valid_stream}).
-
-%% the scheduler is simple, try to get messages from the same shard, but it's okay to take turns
-get_next_stream_from_group({sorted, [H | T]}) ->
-    {H, {sorted, T}};
-get_next_stream_from_group({sorted, []}) ->
-    undefined;
-get_next_stream_from_group(Streams) ->
-    [Stream | T] = lists:sort(
-        fun(#ds_stream{rank = {_, RankA}}, #ds_stream{rank = {_, RankB}}) ->
-            RankA < RankB
-        end,
-        Streams
-    ),
-    {Stream, {sorted, T}}.
-
-get_next_n_messages_from_stream(Stream, CPs, N) ->
-    ItBegin = get_last_iterator(Stream, CPs),
-    case emqx_ds:next(?PERSISTENT_MESSAGE_DB, ItBegin, N) of
-        {ok, _ItEnd, []} ->
-            [];
-        {ok, ItEnd, Messages} ->
-            {ItBegin, ItEnd, Messages};
-        {ok, end_of_stream} ->
-            %% TODO: how to skip this closed stream or it should be taken over by lower level layer
-            []
-    end.
-
-%%================================================================================
-
--spec flatmapfoldl(fun((X, Acc) -> {Y | [Y], Acc}), Acc, [X]) -> {[Y], Acc}.
-flatmapfoldl(_Fun, Acc, []) ->
-    {[], Acc};
-flatmapfoldl(Fun, Acc, [X | Xs]) ->
-    {Ys, NAcc} = Fun(X, Acc),
-    {Zs, FAcc} = flatmapfoldl(Fun, NAcc, Xs),
-    case is_list(Ys) of
-        true ->
-            {Ys ++ Zs, FAcc};
-        _ ->
-            {[Ys | Zs], FAcc}
-    end.
-
-ro_transaction(Fun) ->
-    {atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun),
-    Res.
-
--ifdef(TEST).
-
-%% This test only tests boundary conditions (to make sure property-based test didn't skip them):
-packet_id_to_seqno_test() ->
-    %% Packet ID = 1; first epoch:
-    ?assertEqual(1, packet_id_to_seqno_(1, 1)),
-    ?assertEqual(1, packet_id_to_seqno_(10, 1)),
-    ?assertEqual(1, packet_id_to_seqno_(1 bsl 16 - 1, 1)),
-    ?assertEqual(1, packet_id_to_seqno_(1 bsl 16, 1)),
-    %% Packet ID = 1; second and 3rd epochs:
-    ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(1 bsl 16 + 1, 1)),
-    ?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16, 1)),
-    ?assertEqual(2 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16 + 1, 1)),
-    %% Packet ID = 16#ffff:
-    PID = 1 bsl 16 - 1,
-    ?assertEqual(PID, packet_id_to_seqno_(PID, PID)),
-    ?assertEqual(PID, packet_id_to_seqno_(1 bsl 16, PID)),
-    ?assertEqual(1 bsl 16 + PID, packet_id_to_seqno_(2 bsl 16, PID)),
-    ok.
-
-packet_id_to_seqno_test_() ->
-    Opts = [{numtests, 1000}, {to_file, user}],
-    {timeout, 30, fun() -> ?assert(proper:quickcheck(packet_id_to_seqno_prop(), Opts)) end}.
-
-packet_id_to_seqno_prop() ->
-    ?FORALL(
-        NextSeqNo,
-        next_seqno_gen(),
-        ?FORALL(
-            SeqNo,
-            seqno_gen(NextSeqNo),
-            begin
-                PacketId = seqno_to_packet_id(SeqNo),
-                ?assertEqual(SeqNo, packet_id_to_seqno_(NextSeqNo, PacketId)),
-                true
-            end
-        )
-    ).
-
-next_seqno_gen() ->
-    ?LET(
-        {Epoch, Offset},
-        {non_neg_integer(), non_neg_integer()},
-        Epoch bsl 16 + Offset
-    ).
-
-seqno_gen(NextSeqNo) ->
-    WindowSize = 1 bsl 16 - 1,
-    Min = max(0, NextSeqNo - WindowSize),
-    Max = max(0, NextSeqNo - 1),
-    range(Min, Max).
-
-range_size_test_() ->
-    [
-        ?_assertEqual(0, range_size(42, 42)),
-        ?_assertEqual(1, range_size(42, 43)),
-        ?_assertEqual(1, range_size(16#ffff, 16#10001)),
-        ?_assertEqual(16#ffff - 456 + 123, range_size(16#1f0000 + 456, 16#200000 + 123))
-    ].
-
-compute_inflight_range_test_() ->
-    [
-        ?_assertEqual(
-            {#{ack => 1, comp => 1}, 1},
-            compute_inflight_range([])
-        ),
-        ?_assertEqual(
-            {#{ack => 12, comp => 13}, 42},
-            compute_inflight_range([
-                #ds_pubrange{id = {<<>>, 1, 0}, until = 2, type = ?T_CHECKPOINT},
-                #ds_pubrange{id = {<<>>, 4, 0}, until = 8, type = ?T_CHECKPOINT},
-                #ds_pubrange{id = {<<>>, 11, 0}, until = 12, type = ?T_CHECKPOINT},
-                #ds_pubrange{
-                    id = {<<>>, 12, 0},
-                    until = 13,
-                    type = ?T_INFLIGHT,
-                    tracks = ?TRACK_FLAG(?ACK)
-                },
-                #ds_pubrange{
-                    id = {<<>>, 13, 0},
-                    until = 20,
-                    type = ?T_INFLIGHT,
-                    tracks = ?TRACK_FLAG(?COMP)
-                },
-                #ds_pubrange{
-                    id = {<<>>, 20, 0},
-                    until = 42,
-                    type = ?T_INFLIGHT,
-                    tracks = ?TRACK_FLAG(?ACK) bor ?TRACK_FLAG(?COMP)
-                }
-            ])
-        ),
-        ?_assertEqual(
-            {#{ack => 13, comp => 13}, 13},
-            compute_inflight_range([
-                #ds_pubrange{id = {<<>>, 1, 0}, until = 2, type = ?T_CHECKPOINT},
-                #ds_pubrange{id = {<<>>, 4, 0}, until = 8, type = ?T_CHECKPOINT},
-                #ds_pubrange{id = {<<>>, 11, 0}, until = 12, type = ?T_CHECKPOINT},
-                #ds_pubrange{id = {<<>>, 12, 0}, until = 13, type = ?T_CHECKPOINT}
-            ])
-        )
-    ].
-
--endif.

apps/emqx/src/emqx_persistent_session_ds.hrl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -25,75 +25,54 @@
 -define(SESSION_COMMITTED_OFFSET_TAB, emqx_ds_committed_offset_tab).
 -define(DS_MRIA_SHARD, emqx_ds_session_shard).
 
--define(T_INFLIGHT, 1).
+%%%%% Session sequence numbers:
--define(T_CHECKPOINT, 2).
 
--record(ds_sub, {
+%%
-    id :: emqx_persistent_session_ds:subscription_id(),
+%%   -----|----------|-----|-----|------> seqno
-    start_time :: emqx_ds:time(),
+%%        |          |     |     |
-    props = #{} :: map(),
+%%   committed      dup   rec   next
-    extra = #{} :: map()
+%%                       (Qos2)
-}).
--type ds_sub() :: #ds_sub{}.
 
--record(ds_stream, {
+%% Seqno becomes committed after receiving PUBACK for QoS1 or PUBCOMP
-    session :: emqx_persistent_session_ds:id(),
+%% for QoS2.
-    ref :: _StreamRef,
+-define(committed(QOS), QOS).
-    stream :: emqx_ds:stream(),
+%% Seqno becomes dup after broker sends QoS1 or QoS2 message to the
-    rank :: emqx_ds:stream_rank(),
+%% client. Upon session reconnect, messages with seqno in the
-    beginning :: emqx_ds:iterator()
+%% committed..dup range are retransmitted with DUP flag.
-}).
+%%
--type ds_stream() :: #ds_stream{}.
+-define(dup(QOS), (10 + QOS)).
+%% Rec flag is specific for the QoS2. It contains seqno of the last
+%% PUBREC received from the client. When the session reconnects,
+%% PUBREL packages for the dup..rec range are retransmitted.
+-define(rec, 22).
+%% Last seqno assigned to a message (it may not be sent yet).
+-define(next(QOS), (30 + QOS)).
 
--record(ds_pubrange, {
+%%%%% Stream Replay State:
-    id :: {
+-record(srs, {
-        %% What session this range belongs to.
+    rank_x :: emqx_ds:rank_x(),
-        _Session :: emqx_persistent_session_ds:id(),
+    rank_y :: emqx_ds:rank_y(),
-        %% Where this range starts.
+    %% Iterators at the beginning and the end of the last batch:
-        _First :: emqx_persistent_message_ds_replayer:seqno(),
+    it_begin :: emqx_ds:iterator() | undefined,
-        %% Which stream this range is over.
+    it_end :: emqx_ds:iterator() | end_of_stream,
-        _StreamRef
+    %% Size of the last batch:
-    },
+    batch_size = 0 :: non_neg_integer(),
-    %% Where this range ends: the first seqno that is not included in the range.
+    %% Session sequence numbers at the time when the batch was fetched:
-    until :: emqx_persistent_message_ds_replayer:seqno(),
+    first_seqno_qos1 = 0 :: emqx_persistent_session_ds:seqno(),
-    %% Type of a range:
+    first_seqno_qos2 = 0 :: emqx_persistent_session_ds:seqno(),
-    %% * Inflight range is a range of yet unacked messages from this stream.
+    %% Sequence numbers that have to be committed for the batch:
-    %% * Checkpoint range was already acked, its purpose is to keep track of the
+    last_seqno_qos1 = 0 :: emqx_persistent_session_ds:seqno(),
-    %%   very last iterator for this stream.
+    last_seqno_qos2 = 0 :: emqx_persistent_session_ds:seqno(),
-    type :: ?T_INFLIGHT | ?T_CHECKPOINT,
+    %% This stream belongs to an unsubscribed topic-filter, and is
-    %% What commit tracks this range is part of.
+    %% marked for deletion:
-    tracks = 0 :: non_neg_integer(),
+    unsubscribed = false :: boolean()
-    %% Meaning of this depends on the type of the range:
-    %% * For inflight range, this is the iterator pointing to the first message in
-    %%   the range.
-    %% * For checkpoint range, this is the iterator pointing right past the last
-    %%   message in the range.
-    iterator :: emqx_ds:iterator(),
-    %% Reserved for future use.
-    misc = #{} :: map()
-}).
--type ds_pubrange() :: #ds_pubrange{}.
-
--record(ds_committed_offset, {
-    id :: {
-        %% What session this marker belongs to.
-        _Session :: emqx_persistent_session_ds:id(),
-        %% Marker name.
-        _CommitType
-    },
-    %% Where this marker is pointing to: the first seqno that is not marked.
-    until :: emqx_persistent_message_ds_replayer:seqno()
 }).
 
--record(session, {
+%% Session metadata keys:
-    %% same as clientid
+-define(created_at, created_at).
-    id :: emqx_persistent_session_ds:id(),
+-define(last_alive_at, last_alive_at).
-    %% creation time
+-define(expiry_interval, expiry_interval).
-    created_at :: _Millisecond :: non_neg_integer(),
+%% Unique integer used to create unique identities
-    last_alive_at :: _Millisecond :: non_neg_integer(),
+-define(last_id, last_id).
-    conninfo :: emqx_types:conninfo(),
-    %% for future usage
-    props = #{} :: map()
-}).
 
 -endif.

apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -69,7 +69,7 @@ handle_info(_Info, State) ->
     {noreply, State}.
 
 %%--------------------------------------------------------------------------------
-%% Internal fns
+%% Internal functions
 %%--------------------------------------------------------------------------------
 
 ensure_gc_timer() ->
@@ -104,58 +104,33 @@ now_ms() ->
     erlang:system_time(millisecond).
 
 start_gc() ->
-    do_gc(more).
-
-zombie_session_ms() ->
-    NowMS = now_ms(),
     GCInterval = emqx_config:get([session_persistence, session_gc_interval]),
     BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
     TimeThreshold = max(GCInterval, BumpInterval) * 3,
-    ets:fun2ms(
+    MinLastAlive = now_ms() - TimeThreshold,
-        fun(
+    gc_loop(MinLastAlive, emqx_persistent_session_ds_state:make_session_iterator()).
-            #session{
-                id = DSSessionId,
-                last_alive_at = LastAliveAt,
-                conninfo = #{expiry_interval := EI}
-            }
-        ) when
-            LastAliveAt + EI + TimeThreshold =< NowMS
-        ->
-            DSSessionId
-        end
-    ).
 
-do_gc(more) ->
+gc_loop(MinLastAlive, It0) ->
     GCBatchSize = emqx_config:get([session_persistence, session_gc_batch_size]),
-    MS = zombie_session_ms(),
+    case emqx_persistent_session_ds_state:session_iterator_next(It0, GCBatchSize) of
-    {atomic, Next} = mria:transaction(?DS_MRIA_SHARD, fun() ->
+        {[], _It} ->
-        Res = mnesia:select(?SESSION_TAB, MS, GCBatchSize, write),
+            ok;
-        case Res of
+        {Sessions, It} ->
-            '$end_of_table' ->
+            [do_gc(SessionId, MinLastAlive, Metadata) || {SessionId, Metadata} <- Sessions],
-                done;
+            gc_loop(MinLastAlive, It)
-            {[], Cont} ->
+    end.
-                %% since `GCBatchsize' is just a "recommendation" for `select', we try only
-                %% _once_ the continuation and then stop if it yields nothing, to avoid a
-                %% dead loop.
-                case mnesia:select(Cont) of
-                    '$end_of_table' ->
-                        done;
-                    {[], _Cont} ->
-                        done;
-                    {DSSessionIds0, _Cont} ->
-                        do_gc_(DSSessionIds0),
-                        more
-                end;
-            {DSSessionIds0, _Cont} ->
-                do_gc_(DSSessionIds0),
-                more
-        end
-    end),
-    do_gc(Next);
-do_gc(done) ->
-    ok.
 
-do_gc_(DSSessionIds) ->
+do_gc(SessionId, MinLastAlive, Metadata) ->
-    lists:foreach(fun emqx_persistent_session_ds:destroy_session/1, DSSessionIds),
+    #{?last_alive_at := LastAliveAt, ?expiry_interval := EI} = Metadata,
-    ?tp(ds_session_gc_cleaned, #{session_ids => DSSessionIds}),
+    case LastAliveAt + EI < MinLastAlive of
-    ok.
+        true ->
+            emqx_persistent_session_ds:destroy_session(SessionId),
+            ?tp(debug, ds_session_gc_cleaned, #{
+                session_id => SessionId,
+                last_alive_at => LastAliveAt,
+                expiry_interval => EI,
+                min_last_alive => MinLastAlive
+            });
+        false ->
+            ok
+    end.

apps/emqx/src/emqx_persistent_session_ds_inflight.erl

@@ -0,0 +1,347 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_persistent_session_ds_inflight).
+
+%% API:
+-export([
+    new/1,
+    push/2,
+    pop/1,
+    n_buffered/2,
+    n_inflight/1,
+    puback/2,
+    pubrec/2,
+    pubcomp/2,
+    receive_maximum/1
+]).
+
+%% internal exports:
+-export([]).
+
+-export_type([t/0]).
+
+-include("emqx.hrl").
+-include("emqx_mqtt.hrl").
+
+-ifdef(TEST).
+-include_lib("proper/include/proper.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-endif.
+
+%%================================================================================
+%% Type declarations
+%%================================================================================
+
+-type payload() ::
+    {emqx_persistent_session_ds:seqno() | undefined, emqx_types:message()}
+    | {pubrel, emqx_persistent_session_ds:seqno()}.
+
+-record(inflight, {
+    receive_maximum :: pos_integer(),
+    %% Main queue:
+    queue :: queue:queue(payload()),
+    %% Queues that are used to track sequence numbers of ack tracks:
+    puback_queue :: iqueue(),
+    pubrec_queue :: iqueue(),
+    pubcomp_queue :: iqueue(),
+    %% Counters:
+    n_inflight = 0 :: non_neg_integer(),
+    n_qos0 = 0 :: non_neg_integer(),
+    n_qos1 = 0 :: non_neg_integer(),
+    n_qos2 = 0 :: non_neg_integer()
+}).
+
+-type t() :: #inflight{}.
+
+%%================================================================================
+%% API funcions
+%%================================================================================
+
+-spec new(non_neg_integer()) -> t().
+new(ReceiveMaximum) when ReceiveMaximum > 0 ->
+    #inflight{
+        receive_maximum = ReceiveMaximum,
+        queue = queue:new(),
+        puback_queue = iqueue_new(),
+        pubrec_queue = iqueue_new(),
+        pubcomp_queue = iqueue_new()
+    }.
+
+-spec receive_maximum(t()) -> pos_integer().
+receive_maximum(#inflight{receive_maximum = ReceiveMaximum}) ->
+    ReceiveMaximum.
+
+-spec push(payload(), t()) -> t().
+push(Payload = {pubrel, _SeqNo}, Rec = #inflight{queue = Q}) ->
+    Rec#inflight{queue = queue:in(Payload, Q)};
+push(Payload = {_, Msg}, Rec) ->
+    #inflight{queue = Q0, n_qos0 = NQos0, n_qos1 = NQos1, n_qos2 = NQos2} = Rec,
+    Q = queue:in(Payload, Q0),
+    case Msg#message.qos of
+        ?QOS_0 ->
+            Rec#inflight{queue = Q, n_qos0 = NQos0 + 1};
+        ?QOS_1 ->
+            Rec#inflight{queue = Q, n_qos1 = NQos1 + 1};
+        ?QOS_2 ->
+            Rec#inflight{queue = Q, n_qos2 = NQos2 + 1}
+    end.
+
+-spec pop(t()) -> {payload(), t()} | undefined.
+pop(Rec0) ->
+    #inflight{
+        receive_maximum = ReceiveMaximum,
+        n_inflight = NInflight,
+        queue = Q0,
+        puback_queue = QAck,
+        pubrec_queue = QRec,
+        pubcomp_queue = QComp,
+        n_qos0 = NQos0,
+        n_qos1 = NQos1,
+        n_qos2 = NQos2
+    } = Rec0,
+    case NInflight < ReceiveMaximum andalso queue:out(Q0) of
+        {{value, Payload}, Q} ->
+            Rec =
+                case Payload of
+                    {pubrel, _} ->
+                        Rec0#inflight{queue = Q};
+                    {SeqNo, #message{qos = Qos}} ->
+                        case Qos of
+                            ?QOS_0 ->
+                                Rec0#inflight{queue = Q, n_qos0 = NQos0 - 1};
+                            ?QOS_1 ->
+                                Rec0#inflight{
+                                    queue = Q,
+                                    n_qos1 = NQos1 - 1,
+                                    n_inflight = NInflight + 1,
+                                    puback_queue = ipush(SeqNo, QAck)
+                                };
+                            ?QOS_2 ->
+                                Rec0#inflight{
+                                    queue = Q,
+                                    n_qos2 = NQos2 - 1,
+                                    n_inflight = NInflight + 1,
+                                    pubrec_queue = ipush(SeqNo, QRec),
+                                    pubcomp_queue = ipush(SeqNo, QComp)
+                                }
+                        end
+                end,
+            {Payload, Rec};
+        _ ->
+            undefined
+    end.
+
+-spec n_buffered(?QOS_0..?QOS_2 | all, t()) -> non_neg_integer().
+n_buffered(?QOS_0, #inflight{n_qos0 = NQos0}) ->
+    NQos0;
+n_buffered(?QOS_1, #inflight{n_qos1 = NQos1}) ->
+    NQos1;
+n_buffered(?QOS_2, #inflight{n_qos2 = NQos2}) ->
+    NQos2;
+n_buffered(all, #inflight{n_qos0 = NQos0, n_qos1 = NQos1, n_qos2 = NQos2}) ->
+    NQos0 + NQos1 + NQos2.
+
+-spec n_inflight(t()) -> non_neg_integer().
+n_inflight(#inflight{n_inflight = NInflight}) ->
+    NInflight.
+
+-spec puback(emqx_persistent_session_ds:seqno(), t()) -> {ok, t()} | {error, Expected} when
+    Expected :: emqx_persistent_session_ds:seqno() | undefined.
+puback(SeqNo, Rec = #inflight{puback_queue = Q0, n_inflight = N}) ->
+    case ipop(Q0) of
+        {{value, SeqNo}, Q} ->
+            {ok, Rec#inflight{
+                puback_queue = Q,
+                n_inflight = max(0, N - 1)
+            }};
+        {{value, Expected}, _} ->
+            {error, Expected};
+        _ ->
+            {error, undefined}
+    end.
+
+-spec pubcomp(emqx_persistent_session_ds:seqno(), t()) -> {ok, t()} | {error, Expected} when
+    Expected :: emqx_persistent_session_ds:seqno() | undefined.
+pubcomp(SeqNo, Rec = #inflight{pubcomp_queue = Q0, n_inflight = N}) ->
+    case ipop(Q0) of
+        {{value, SeqNo}, Q} ->
+            {ok, Rec#inflight{
+                pubcomp_queue = Q,
+                n_inflight = max(0, N - 1)
+            }};
+        {{value, Expected}, _} ->
+            {error, Expected};
+        _ ->
+            {error, undefined}
+    end.
+
+%% PUBREC doesn't affect inflight window:
+%% https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Flow_Control
+-spec pubrec(emqx_persistent_session_ds:seqno(), t()) -> {ok, t()} | {error, Expected} when
+    Expected :: emqx_persistent_session_ds:seqno() | undefined.
+pubrec(SeqNo, Rec = #inflight{pubrec_queue = Q0}) ->
+    case ipop(Q0) of
+        {{value, SeqNo}, Q} ->
+            {ok, Rec#inflight{
+                pubrec_queue = Q
+            }};
+        {{value, Expected}, _} ->
+            {error, Expected};
+        _ ->
+            {error, undefined}
+    end.
+
+%%================================================================================
+%% Internal functions
+%%================================================================================
+
+%%%% Interval queue:
+
+%% "Interval queue": a data structure that represents a queue of
+%% monotonically increasing non-negative integers in a compact manner.
+%% It is functionally equivalent to a `queue:queue(integer())'.
+-record(iqueue, {
+    %% Head interval:
+    head = 0 :: integer(),
+    head_end = 0 :: integer(),
+    %% Intermediate ranges:
+    queue :: queue:queue({integer(), integer()}),
+    %% End interval:
+    tail = 0 :: integer(),
+    tail_end = 0 :: integer()
+}).
+
+-type iqueue() :: #iqueue{}.
+
+iqueue_new() ->
+    #iqueue{
+        queue = queue:new()
+    }.
+
+%% @doc Push a value into the interval queue:
+-spec ipush(integer(), iqueue()) -> iqueue().
+ipush(Val, Q = #iqueue{tail_end = Val, head_end = Val}) ->
+    %% Optimization: head and tail intervals overlap, and the newly
+    %% inserted value extends both. Attach it to both intervals, to
+    %% avoid `queue:out' in `ipop':
+    Q#iqueue{
+        tail_end = Val + 1,
+        head_end = Val + 1
+    };
+ipush(Val, Q = #iqueue{tail_end = Val}) ->
+    %% Extend tail interval:
+    Q#iqueue{
+        tail_end = Val + 1
+    };
+ipush(Val, Q = #iqueue{tail = Tl, tail_end = End, queue = IQ0}) when is_number(Val), Val > End ->
+    IQ = queue:in({Tl, End}, IQ0),
+    %% Begin a new interval:
+    Q#iqueue{
+        queue = IQ,
+        tail = Val,
+        tail_end = Val + 1
+    }.
+
+-spec ipop(iqueue()) -> {{value, integer()}, iqueue()} | {empty, iqueue()}.
+ipop(Q = #iqueue{head = Hd, head_end = HdEnd}) when Hd < HdEnd ->
+    %% Head interval is not empty. Consume a value from it:
+    {{value, Hd}, Q#iqueue{head = Hd + 1}};
+ipop(Q = #iqueue{head_end = End, tail_end = End}) ->
+    %% Head interval is fully consumed, and it's overlaps with the
+    %% tail interval. It means the queue is empty:
+    {empty, Q};
+ipop(Q = #iqueue{head = Hd0, tail = Tl, tail_end = TlEnd, queue = IQ0}) ->
+    %% Head interval is fully consumed, and it doesn't overlap with
+    %% the tail interval. Replace the head interval with the next
+    %% interval from the queue or with the tail interval:
+    case queue:out(IQ0) of
+        {{value, {Hd, HdEnd}}, IQ} ->
+            ipop(Q#iqueue{head = max(Hd0, Hd), head_end = HdEnd, queue = IQ});
+        {empty, _} ->
+            ipop(Q#iqueue{head = max(Hd0, Tl), head_end = TlEnd})
+    end.
+
+-ifdef(TEST).
+
+%% Test that behavior of iqueue is identical to that of a regular queue of integers:
+iqueue_compat_test_() ->
+    Props = [iqueue_compat()],
+    Opts = [{numtests, 1000}, {to_file, user}, {max_size, 100}],
+    {timeout, 30, [?_assert(proper:quickcheck(Prop, Opts)) || Prop <- Props]}.
+
+%% Generate a sequence of pops and pushes with monotonically
+%% increasing arguments, and verify replaying produces equivalent
+%% results for the optimized and the reference implementation:
+iqueue_compat() ->
+    ?FORALL(
+        Cmds,
+        iqueue_commands(),
+        begin
+            lists:foldl(
+                fun
+                    ({push, N}, {IQ, Q, Acc}) ->
+                        {ipush(N, IQ), queue:in(N, Q), [N | Acc]};
+                    (pop, {IQ0, Q0, Acc}) ->
+                        {Ret, IQ} = ipop(IQ0),
+                        {Expected, Q} = queue:out(Q0),
+                        ?assertEqual(
+                            Expected,
+                            Ret,
+                            #{
+                                sequence => lists:reverse(Acc),
+                                q => queue:to_list(Q0),
+                                iq0 => iqueue_print(IQ0),
+                                iq => iqueue_print(IQ)
+                            }
+                        ),
+                        {IQ, Q, [pop | Acc]}
+                end,
+                {iqueue_new(), queue:new(), []},
+                Cmds
+            ),
+            true
+        end
+    ).
+
+iqueue_cmd() ->
+    oneof([
+        pop,
+        {push, range(1, 3)}
+    ]).
+
+iqueue_commands() ->
+    ?LET(
+        Cmds,
+        list(iqueue_cmd()),
+        process_test_cmds(Cmds, 0)
+    ).
+
+process_test_cmds([], _) ->
+    [];
+process_test_cmds([pop | Tl], Cnt) ->
+    [pop | process_test_cmds(Tl, Cnt)];
+process_test_cmds([{push, N} | Tl], Cnt0) ->
+    Cnt = Cnt0 + N,
+    [{push, Cnt} | process_test_cmds(Tl, Cnt)].
+
+iqueue_print(I = #iqueue{head = Hd, head_end = HdEnd, queue = Q, tail = Tl, tail_end = TlEnd}) ->
+    #{
+        hd => {Hd, HdEnd},
+        tl => {Tl, TlEnd},
+        q => queue:to_list(Q)
+    }.
+
+-endif.

apps/emqx/src/emqx_persistent_session_ds_state.erl

@@ -0,0 +1,586 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc CRUD interface for the persistent session
+%%
+%% This module encapsulates the data related to the state of the
+%% inflight messages for the persistent session based on DS.
+%%
+%% It is responsible for saving, caching, and restoring session state.
+%% It is completely devoid of business logic. Not even the default
+%% values should be set in this module.
+-module(emqx_persistent_session_ds_state).
+
+-export([create_tables/0]).
+
+-export([open/1, create_new/1, delete/1, commit/1, format/1, print_session/1, list_sessions/0]).
+-export([get_created_at/1, set_created_at/2]).
+-export([get_last_alive_at/1, set_last_alive_at/2]).
+-export([get_expiry_interval/1, set_expiry_interval/2]).
+-export([new_id/1]).
+-export([get_stream/2, put_stream/3, del_stream/2, fold_streams/3]).
+-export([get_seqno/2, put_seqno/3]).
+-export([get_rank/2, put_rank/3, del_rank/2, fold_ranks/3]).
+-export([get_subscriptions/1, put_subscription/4, del_subscription/3]).
+
+-export([make_session_iterator/0, session_iterator_next/2]).
+
+-export_type([
+    t/0, metadata/0, subscriptions/0, seqno_type/0, stream_key/0, rank_key/0, session_iterator/0
+]).
+
+-include("emqx_mqtt.hrl").
+-include("emqx_persistent_session_ds.hrl").
+-include_lib("snabbkaffe/include/trace.hrl").
+-include_lib("stdlib/include/qlc.hrl").
+
+%%================================================================================
+%% Type declarations
+%%================================================================================
+
+-type subscriptions() :: emqx_topic_gbt:t(_SubId, emqx_persistent_session_ds:subscription()).
+
+-opaque session_iterator() :: emqx_persistent_session_ds:id() | '$end_of_table'.
+
+%% Generic key-value wrapper that is used for exporting arbitrary
+%% terms to mnesia:
+-record(kv, {k, v}).
+
+%% Persistent map.
+%%
+%% Pmap accumulates the updates in a term stored in the heap of a
+%% process, so they can be committed all at once in a single
+%% transaction.
+%%
+%% It should be possible to make frequent changes to the pmap without
+%% stressing Mria.
+%%
+%% It's implemented as two maps: `cache', and `dirty'. `cache' stores
+%% the data, and `dirty' contains information about dirty and deleted
+%% keys. When `commit/1' is called, dirty keys are dumped to the
+%% tables, and deleted keys are removed from the tables.
+-record(pmap, {table, cache, dirty}).
+
+-type pmap(K, V) ::
+    #pmap{
+        table :: atom(),
+        cache :: #{K => V},
+        dirty :: #{K => dirty | del}
+    }.
+
+-type metadata() ::
+    #{
+        ?created_at => emqx_persistent_session_ds:timestamp(),
+        ?last_alive_at => emqx_persistent_session_ds:timestamp(),
+        ?expiry_interval => non_neg_integer(),
+        ?last_id => integer()
+    }.
+
+-type seqno_type() ::
+    ?next(?QOS_1)
+    | ?dup(?QOS_1)
+    | ?committed(?QOS_1)
+    | ?next(?QOS_2)
+    | ?dup(?QOS_2)
+    | ?rec
+    | ?committed(?QOS_2).
+
+-opaque t() :: #{
+    id := emqx_persistent_session_ds:id(),
+    dirty := boolean(),
+    metadata := metadata(),
+    subscriptions := subscriptions(),
+    seqnos := pmap(seqno_type(), emqx_persistent_session_ds:seqno()),
+    streams := pmap(emqx_ds:stream(), emqx_persistent_session_ds:stream_state()),
+    ranks := pmap(term(), integer())
+}.
+
+-define(session_tab, emqx_ds_session_tab).
+-define(subscription_tab, emqx_ds_session_subscriptions).
+-define(stream_tab, emqx_ds_session_streams).
+-define(seqno_tab, emqx_ds_session_seqnos).
+-define(rank_tab, emqx_ds_session_ranks).
+-define(pmap_tables, [?stream_tab, ?seqno_tab, ?rank_tab, ?subscription_tab]).
+
+%% Enable this flag if you suspect some code breaks the sequence:
+-ifndef(CHECK_SEQNO).
+-define(set_dirty, dirty => true).
+-define(unset_dirty, dirty => false).
+-else.
+-define(set_dirty, dirty => true, '_' => do_seqno()).
+-define(unset_dirty, dirty => false, '_' => do_seqno()).
+-endif.
+
+%%================================================================================
+%% API funcions
+%%================================================================================
+
+-spec create_tables() -> ok.
+create_tables() ->
+    ok = mria:create_table(
+        ?session_tab,
+        [
+            {rlog_shard, ?DS_MRIA_SHARD},
+            {type, ordered_set},
+            {storage, rocksdb_copies},
+            {record_name, kv},
+            {attributes, record_info(fields, kv)}
+        ]
+    ),
+    [create_kv_pmap_table(Table) || Table <- ?pmap_tables],
+    mria:wait_for_tables([?session_tab | ?pmap_tables]).
+
+-spec open(emqx_persistent_session_ds:id()) -> {ok, t()} | undefined.
+open(SessionId) ->
+    ro_transaction(fun() ->
+        case kv_restore(?session_tab, SessionId) of
+            [Metadata] ->
+                Rec = #{
+                    id => SessionId,
+                    metadata => Metadata,
+                    subscriptions => read_subscriptions(SessionId),
+                    streams => pmap_open(?stream_tab, SessionId),
+                    seqnos => pmap_open(?seqno_tab, SessionId),
+                    ranks => pmap_open(?rank_tab, SessionId),
+                    ?unset_dirty
+                },
+                {ok, Rec};
+            [] ->
+                undefined
+        end
+    end).
+
+-spec print_session(emqx_persistent_session_ds:id()) -> map() | undefined.
+print_session(SessionId) ->
+    case open(SessionId) of
+        undefined ->
+            undefined;
+        {ok, Session} ->
+            format(Session)
+    end.
+
+-spec format(t()) -> map().
+format(#{
+    metadata := Metadata,
+    subscriptions := SubsGBT,
+    streams := Streams,
+    seqnos := Seqnos,
+    ranks := Ranks
+}) ->
+    Subs = emqx_topic_gbt:fold(
+        fun(Key, Sub, Acc) ->
+            maps:put(emqx_topic_gbt:get_topic(Key), Sub, Acc)
+        end,
+        #{},
+        SubsGBT
+    ),
+    #{
+        metadata => Metadata,
+        subscriptions => Subs,
+        streams => pmap_format(Streams),
+        seqnos => pmap_format(Seqnos),
+        ranks => pmap_format(Ranks)
+    }.
+
+-spec list_sessions() -> [emqx_persistent_session_ds:id()].
+list_sessions() ->
+    mnesia:dirty_all_keys(?session_tab).
+
+-spec delete(emqx_persistent_session_ds:id()) -> ok.
+delete(Id) ->
+    transaction(
+        fun() ->
+            [kv_pmap_delete(Table, Id) || Table <- ?pmap_tables],
+            mnesia:delete(?session_tab, Id, write)
+        end
+    ).
+
+-spec commit(t()) -> t().
+commit(Rec = #{dirty := false}) ->
+    Rec;
+commit(
+    Rec = #{
+        id := SessionId,
+        metadata := Metadata,
+        streams := Streams,
+        seqnos := SeqNos,
+        ranks := Ranks
+    }
+) ->
+    check_sequence(Rec),
+    transaction(fun() ->
+        kv_persist(?session_tab, SessionId, Metadata),
+        Rec#{
+            streams => pmap_commit(SessionId, Streams),
+            seqnos => pmap_commit(SessionId, SeqNos),
+            ranks => pmap_commit(SessionId, Ranks),
+            ?unset_dirty
+        }
+    end).
+
+-spec create_new(emqx_persistent_session_ds:id()) -> t().
+create_new(SessionId) ->
+    transaction(fun() ->
+        delete(SessionId),
+        #{
+            id => SessionId,
+            metadata => #{},
+            subscriptions => emqx_topic_gbt:new(),
+            streams => pmap_open(?stream_tab, SessionId),
+            seqnos => pmap_open(?seqno_tab, SessionId),
+            ranks => pmap_open(?rank_tab, SessionId),
+            ?set_dirty
+        }
+    end).
+
+%%
+
+-spec get_created_at(t()) -> emqx_persistent_session_ds:timestamp() | undefined.
+get_created_at(Rec) ->
+    get_meta(?created_at, Rec).
+
+-spec set_created_at(emqx_persistent_session_ds:timestamp(), t()) -> t().
+set_created_at(Val, Rec) ->
+    set_meta(?created_at, Val, Rec).
+
+-spec get_last_alive_at(t()) -> emqx_persistent_session_ds:timestamp() | undefined.
+get_last_alive_at(Rec) ->
+    get_meta(?last_alive_at, Rec).
+
+-spec set_last_alive_at(emqx_persistent_session_ds:timestamp(), t()) -> t().
+set_last_alive_at(Val, Rec) ->
+    set_meta(?last_alive_at, Val, Rec).
+
+-spec get_expiry_interval(t()) -> non_neg_integer() | undefined.
+get_expiry_interval(Rec) ->
+    get_meta(?expiry_interval, Rec).
+
+-spec set_expiry_interval(non_neg_integer(), t()) -> t().
+set_expiry_interval(Val, Rec) ->
+    set_meta(?expiry_interval, Val, Rec).
+
+-spec new_id(t()) -> {emqx_persistent_session_ds:subscription_id(), t()}.
+new_id(Rec) ->
+    LastId =
+        case get_meta(?last_id, Rec) of
+            undefined -> 0;
+            N when is_integer(N) -> N
+        end,
+    {LastId, set_meta(?last_id, LastId + 1, Rec)}.
+
+%%
+
+-spec get_subscriptions(t()) -> subscriptions().
+get_subscriptions(#{subscriptions := Subs}) ->
+    Subs.
+
+-spec put_subscription(
+    emqx_persistent_session_ds:topic_filter(),
+    _SubId,
+    emqx_persistent_session_ds:subscription(),
+    t()
+) -> t().
+put_subscription(TopicFilter, SubId, Subscription, Rec = #{id := Id, subscriptions := Subs0}) ->
+    %% Note: currently changes to the subscriptions are persisted immediately.
+    Key = {TopicFilter, SubId},
+    transaction(fun() -> kv_pmap_persist(?subscription_tab, Id, Key, Subscription) end),
+    Subs = emqx_topic_gbt:insert(TopicFilter, SubId, Subscription, Subs0),
+    Rec#{subscriptions => Subs}.
+
+-spec del_subscription(emqx_persistent_session_ds:topic_filter(), _SubId, t()) -> t().
+del_subscription(TopicFilter, SubId, Rec = #{id := Id, subscriptions := Subs0}) ->
+    %% Note: currently the subscriptions are persisted immediately.
+    Key = {TopicFilter, SubId},
+    transaction(fun() -> kv_pmap_delete(?subscription_tab, Id, Key) end),
+    Subs = emqx_topic_gbt:delete(TopicFilter, SubId, Subs0),
+    Rec#{subscriptions => Subs}.
+
+%%
+
+-type stream_key() :: {emqx_persistent_session_ds:subscription_id(), _StreamId}.
+
+-spec get_stream(stream_key(), t()) ->
+    emqx_persistent_session_ds:stream_state() | undefined.
+get_stream(Key, Rec) ->
+    gen_get(streams, Key, Rec).
+
+-spec put_stream(stream_key(), emqx_persistent_session_ds:stream_state(), t()) -> t().
+put_stream(Key, Val, Rec) ->
+    gen_put(streams, Key, Val, Rec).
+
+-spec del_stream(stream_key(), t()) -> t().
+del_stream(Key, Rec) ->
+    gen_del(streams, Key, Rec).
+
+-spec fold_streams(fun(), Acc, t()) -> Acc.
+fold_streams(Fun, Acc, Rec) ->
+    gen_fold(streams, Fun, Acc, Rec).
+
+%%
+
+-spec get_seqno(seqno_type(), t()) -> emqx_persistent_session_ds:seqno() | undefined.
+get_seqno(Key, Rec) ->
+    gen_get(seqnos, Key, Rec).
+
+-spec put_seqno(seqno_type(), emqx_persistent_session_ds:seqno(), t()) -> t().
+put_seqno(Key, Val, Rec) ->
+    gen_put(seqnos, Key, Val, Rec).
+
+%%
+
+-type rank_key() :: {emqx_persistent_session_ds:subscription_id(), emqx_ds:rank_x()}.
+
+-spec get_rank(rank_key(), t()) -> integer() | undefined.
+get_rank(Key, Rec) ->
+    gen_get(ranks, Key, Rec).
+
+-spec put_rank(rank_key(), integer(), t()) -> t().
+put_rank(Key, Val, Rec) ->
+    gen_put(ranks, Key, Val, Rec).
+
+-spec del_rank(rank_key(), t()) -> t().
+del_rank(Key, Rec) ->
+    gen_del(ranks, Key, Rec).
+
+-spec fold_ranks(fun(), Acc, t()) -> Acc.
+fold_ranks(Fun, Acc, Rec) ->
+    gen_fold(ranks, Fun, Acc, Rec).
+
+-spec make_session_iterator() -> session_iterator().
+make_session_iterator() ->
+    case mnesia:dirty_first(?session_tab) of
+        '$end_of_table' ->
+            '$end_of_table';
+        Key ->
+            Key
+    end.
+
+-spec session_iterator_next(session_iterator(), pos_integer()) ->
+    {[{emqx_persistent_session_ds:id(), metadata()}], session_iterator()}.
+session_iterator_next(Cursor, 0) ->
+    {[], Cursor};
+session_iterator_next('$end_of_table', _N) ->
+    {[], '$end_of_table'};
+session_iterator_next(Cursor0, N) ->
+    ThisVal = [
+        {Cursor0, Metadata}
+     || #kv{v = Metadata} <- mnesia:dirty_read(?session_tab, Cursor0)
+    ],
+    {NextVals, Cursor} = session_iterator_next(mnesia:dirty_next(?session_tab, Cursor0), N - 1),
+    {ThisVal ++ NextVals, Cursor}.
+
+%%================================================================================
+%% Internal functions
+%%================================================================================
+
+%% All mnesia reads and writes are passed through this function.
+%% Backward compatiblity issues can be handled here.
+encoder(encode, _Table, Term) ->
+    Term;
+encoder(decode, _Table, Term) ->
+    Term.
+
+%%
+
+get_meta(K, #{metadata := Meta}) ->
+    maps:get(K, Meta, undefined).
+
+set_meta(K, V, Rec = #{metadata := Meta}) ->
+    check_sequence(Rec#{metadata => maps:put(K, V, Meta), ?set_dirty}).
+
+%%
+
+gen_get(Field, Key, Rec) ->
+    check_sequence(Rec),
+    pmap_get(Key, maps:get(Field, Rec)).
+
+gen_fold(Field, Fun, Acc, Rec) ->
+    check_sequence(Rec),
+    pmap_fold(Fun, Acc, maps:get(Field, Rec)).
+
+gen_put(Field, Key, Val, Rec) ->
+    check_sequence(Rec),
+    maps:update_with(
+        Field,
+        fun(PMap) -> pmap_put(Key, Val, PMap) end,
+        Rec#{?set_dirty}
+    ).
+
+gen_del(Field, Key, Rec) ->
+    check_sequence(Rec),
+    maps:update_with(
+        Field,
+        fun(PMap) -> pmap_del(Key, PMap) end,
+        Rec#{?set_dirty}
+    ).
+
+%%
+
+read_subscriptions(SessionId) ->
+    Records = kv_pmap_restore(?subscription_tab, SessionId),
+    lists:foldl(
+        fun({{TopicFilter, SubId}, Subscription}, Acc) ->
+            emqx_topic_gbt:insert(TopicFilter, SubId, Subscription, Acc)
+        end,
+        emqx_topic_gbt:new(),
+        Records
+    ).
+
+%%
+
+%% @doc Open a PMAP and fill the clean area with the data from DB.
+%% This functtion should be ran in a transaction.
+-spec pmap_open(atom(), emqx_persistent_session_ds:id()) -> pmap(_K, _V).
+pmap_open(Table, SessionId) ->
+    Clean = maps:from_list(kv_pmap_restore(Table, SessionId)),
+    #pmap{
+        table = Table,
+        cache = Clean,
+        dirty = #{}
+    }.
+
+-spec pmap_get(K, pmap(K, V)) -> V | undefined.
+pmap_get(K, #pmap{cache = Cache}) ->
+    maps:get(K, Cache, undefined).
+
+-spec pmap_put(K, V, pmap(K, V)) -> pmap(K, V).
+pmap_put(K, V, Pmap = #pmap{dirty = Dirty, cache = Cache}) ->
+    Pmap#pmap{
+        cache = maps:put(K, V, Cache),
+        dirty = Dirty#{K => dirty}
+    }.
+
+-spec pmap_del(K, pmap(K, V)) -> pmap(K, V).
+pmap_del(
+    Key,
+    Pmap = #pmap{dirty = Dirty, cache = Cache}
+) ->
+    Pmap#pmap{
+        cache = maps:remove(Key, Cache),
+        dirty = Dirty#{Key => del}
+    }.
+
+-spec pmap_fold(fun((K, V, A) -> A), A, pmap(K, V)) -> A.
+pmap_fold(Fun, Acc, #pmap{cache = Cache}) ->
+    maps:fold(Fun, Acc, Cache).
+
+-spec pmap_commit(emqx_persistent_session_ds:id(), pmap(K, V)) -> pmap(K, V).
+pmap_commit(
+    SessionId, Pmap = #pmap{table = Tab, dirty = Dirty, cache = Cache}
+) ->
+    maps:foreach(
+        fun
+            (K, del) ->
+                kv_pmap_delete(Tab, SessionId, K);
+            (K, dirty) ->
+                V = maps:get(K, Cache),
+                kv_pmap_persist(Tab, SessionId, K, V)
+        end,
+        Dirty
+    ),
+    Pmap#pmap{
+        dirty = #{}
+    }.
+
+-spec pmap_format(pmap(_K, _V)) -> map().
+pmap_format(#pmap{cache = Cache}) ->
+    Cache.
+
+%% Functions dealing with set tables:
+
+kv_persist(Tab, SessionId, Val0) ->
+    Val = encoder(encode, Tab, Val0),
+    mnesia:write(Tab, #kv{k = SessionId, v = Val}, write).
+
+kv_restore(Tab, SessionId) ->
+    [encoder(decode, Tab, V) || #kv{v = V} <- mnesia:read(Tab, SessionId)].
+
+%% Functions dealing with bags:
+
+%% @doc Create a mnesia table for the PMAP:
+-spec create_kv_pmap_table(atom()) -> ok.
+create_kv_pmap_table(Table) ->
+    mria:create_table(Table, [
+        {type, ordered_set},
+        {rlog_shard, ?DS_MRIA_SHARD},
+        {storage, rocksdb_copies},
+        {record_name, kv},
+        {attributes, record_info(fields, kv)}
+    ]).
+
+kv_pmap_persist(Tab, SessionId, Key, Val0) ->
+    %% Write data to mnesia:
+    Val = encoder(encode, Tab, Val0),
+    mnesia:write(Tab, #kv{k = {SessionId, Key}, v = Val}, write).
+
+kv_pmap_restore(Table, SessionId) ->
+    MS = [{#kv{k = {SessionId, '$1'}, v = '$2'}, [], [{{'$1', '$2'}}]}],
+    Objs = mnesia:select(Table, MS, read),
+    [{K, encoder(decode, Table, V)} || {K, V} <- Objs].
+
+kv_pmap_delete(Table, SessionId) ->
+    MS = [{#kv{k = {SessionId, '$1'}, _ = '_'}, [], ['$1']}],
+    Keys = mnesia:select(Table, MS, read),
+    [mnesia:delete(Table, {SessionId, K}, write) || K <- Keys],
+    ok.
+
+kv_pmap_delete(Table, SessionId, Key) ->
+    %% Note: this match spec uses a fixed primary key, so it doesn't
+    %% require a table scan, and the transaction doesn't grab the
+    %% whole table lock:
+    mnesia:delete(Table, {SessionId, Key}, write).
+
+%%
+
+transaction(Fun) ->
+    mria:async_dirty(?DS_MRIA_SHARD, Fun).
+
+ro_transaction(Fun) ->
+    mria:async_dirty(?DS_MRIA_SHARD, Fun).
+
+%% transaction(Fun) ->
+%%     case mnesia:is_transaction() of
+%%         true ->
+%%             Fun();
+%%         false ->
+%%             {atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun),
+%%             Res
+%%     end.
+
+%% ro_transaction(Fun) ->
+%%     {atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun),
+%%     Res.
+
+-compile({inline, check_sequence/1}).
+
+-ifdef(CHECK_SEQNO).
+do_seqno() ->
+    case erlang:get(?MODULE) of
+        undefined ->
+            put(?MODULE, 0),
+            0;
+        N ->
+            put(?MODULE, N + 1),
+            N + 1
+    end.
+
+check_sequence(A = #{'_' := N}) ->
+    N = erlang:get(?MODULE),
+    A.
+-else.
+check_sequence(A) ->
+    A.
+-endif.

apps/emqx/src/emqx_persistent_session_ds_stream_scheduler.erl

@@ -0,0 +1,380 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_persistent_session_ds_stream_scheduler).
+
+%% API:
+-export([find_new_streams/1, find_replay_streams/1, is_fully_acked/2]).
+-export([renew_streams/1, on_unsubscribe/2]).
+
+%% behavior callbacks:
+-export([]).
+
+%% internal exports:
+-export([]).
+
+-export_type([]).
+
+-include_lib("emqx/include/logger.hrl").
+-include("emqx_mqtt.hrl").
+-include("emqx_persistent_session_ds.hrl").
+
+%%================================================================================
+%% Type declarations
+%%================================================================================
+
+%%================================================================================
+%% API functions
+%%================================================================================
+
+%% @doc Find the streams that have uncommitted (in-flight) messages.
+%% Return them in the order they were previously replayed.
+-spec find_replay_streams(emqx_persistent_session_ds_state:t()) ->
+    [{emqx_persistent_session_ds_state:stream_key(), emqx_persistent_session_ds:stream_state()}].
+find_replay_streams(S) ->
+    Comm1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S),
+    Comm2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S),
+    %% 1. Find the streams that aren't fully acked
+    Streams = emqx_persistent_session_ds_state:fold_streams(
+        fun(Key, Stream, Acc) ->
+            case is_fully_acked(Comm1, Comm2, Stream) of
+                false ->
+                    [{Key, Stream} | Acc];
+                true ->
+                    Acc
+            end
+        end,
+        [],
+        S
+    ),
+    lists:sort(fun compare_streams/2, Streams).
+
+%% @doc Find streams from which the new messages can be fetched.
+%%
+%% Currently it amounts to the streams that don't have any inflight
+%% messages, since for performance reasons we keep only one record of
+%% in-flight messages per stream, and we don't want to overwrite these
+%% records prematurely.
+%%
+%% This function is non-detereministic: it randomizes the order of
+%% streams to ensure fair replay of different topics.
+-spec find_new_streams(emqx_persistent_session_ds_state:t()) ->
+    [{emqx_persistent_session_ds_state:stream_key(), emqx_persistent_session_ds:stream_state()}].
+find_new_streams(S) ->
+    %% FIXME: this function is currently very sensitive to the
+    %% consistency of the packet IDs on both broker and client side.
+    %%
+    %% If the client fails to properly ack packets due to a bug, or a
+    %% network issue, or if the state of streams and seqno tables ever
+    %% become de-synced, then this function will return an empty list,
+    %% and the replay cannot progress.
+    %%
+    %% In other words, this function is not robust, and we should find
+    %% some way to get the replays un-stuck at the cost of potentially
+    %% losing messages during replay (or just kill the stuck channel
+    %% after timeout?)
+    Comm1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S),
+    Comm2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S),
+    shuffle(
+        emqx_persistent_session_ds_state:fold_streams(
+            fun
+                (_Key, #srs{it_end = end_of_stream}, Acc) ->
+                    Acc;
+                (Key, Stream, Acc) ->
+                    case is_fully_acked(Comm1, Comm2, Stream) andalso not Stream#srs.unsubscribed of
+                        true ->
+                            [{Key, Stream} | Acc];
+                        false ->
+                            Acc
+                    end
+            end,
+            [],
+            S
+        )
+    ).
+
+%% @doc This function makes the session aware of the new streams.
+%%
+%% It has the following properties:
+%%
+%% 1. For each RankX, it keeps only the streams with the same RankY.
+%%
+%% 2. For each RankX, it never advances RankY until _all_ streams with
+%% the same RankX are replayed.
+%%
+%% 3. Once all streams with the given rank are replayed, it advances
+%% the RankY to the smallest known RankY that is greater than replayed
+%% RankY.
+%%
+%% 4. If the RankX has never been replayed, it selects the streams
+%% with the smallest RankY.
+%%
+%% This way, messages from the same topic/shard are never reordered.
+-spec renew_streams(emqx_persistent_session_ds_state:t()) -> emqx_persistent_session_ds_state:t().
+renew_streams(S0) ->
+    S1 = remove_unsubscribed_streams(S0),
+    S2 = remove_fully_replayed_streams(S1),
+    emqx_persistent_session_ds_subs:fold(
+        fun
+            (Key, #{start_time := StartTime, id := SubId, deleted := false}, Acc) ->
+                TopicFilter = emqx_topic:words(Key),
+                Streams = select_streams(
+                    SubId,
+                    emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime),
+                    Acc
+                ),
+                lists:foldl(
+                    fun(I, Acc1) ->
+                        ensure_iterator(TopicFilter, StartTime, SubId, I, Acc1)
+                    end,
+                    Acc,
+                    Streams
+                );
+            (_Key, _DeletedSubscription, Acc) ->
+                Acc
+        end,
+        S2,
+        S2
+    ).
+
+-spec on_unsubscribe(
+    emqx_persistent_session_ds:subscription_id(), emqx_persistent_session_ds_state:t()
+) ->
+    emqx_persistent_session_ds_state:t().
+on_unsubscribe(SubId, S0) ->
+    %% NOTE: this function only marks the streams for deletion,
+    %% instead of outright deleting them.
+    %%
+    %% It's done for two reasons:
+    %%
+    %% - MQTT standard states that the broker MUST process acks for
+    %% all sent messages, and it MAY keep on sending buffered
+    %% messages:
+    %% https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Toc3901186
+    %%
+    %% - Deleting the streams may lead to gaps in the sequence number
+    %% series, and lead to problems with acknowledgement tracking, we
+    %% avoid that by delaying the deletion.
+    %%
+    %% When the stream is marked for deletion, the session won't fetch
+    %% _new_ batches from it. Actual deletion is done by
+    %% `renew_streams', when it detects that all in-flight messages
+    %% from the stream have been acked by the client.
+    emqx_persistent_session_ds_state:fold_streams(
+        fun(Key, Srs, Acc) ->
+            case Key of
+                {SubId, _Stream} ->
+                    %% This stream belongs to a deleted subscription.
+                    %% Mark for deletion:
+                    emqx_persistent_session_ds_state:put_stream(
+                        Key, Srs#srs{unsubscribed = true}, Acc
+                    );
+                _ ->
+                    Acc
+            end
+        end,
+        S0,
+        S0
+    ).
+
+-spec is_fully_acked(
+    emqx_persistent_session_ds:stream_state(), emqx_persistent_session_ds_state:t()
+) -> boolean().
+is_fully_acked(Srs, S) ->
+    CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S),
+    CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S),
+    is_fully_acked(CommQos1, CommQos2, Srs).
+
+%%================================================================================
+%% Internal functions
+%%================================================================================
+
+ensure_iterator(TopicFilter, StartTime, SubId, {{RankX, RankY}, Stream}, S) ->
+    Key = {SubId, Stream},
+    case emqx_persistent_session_ds_state:get_stream(Key, S) of
+        undefined ->
+            ?SLOG(debug, #{
+                msg => new_stream, key => Key, stream => Stream
+            }),
+            {ok, Iterator} = emqx_ds:make_iterator(
+                ?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime
+            ),
+            NewStreamState = #srs{
+                rank_x = RankX,
+                rank_y = RankY,
+                it_begin = Iterator,
+                it_end = Iterator
+            },
+            emqx_persistent_session_ds_state:put_stream(Key, NewStreamState, S);
+        #srs{} ->
+            S
+    end.
+
+select_streams(SubId, Streams0, S) ->
+    TopicStreamGroups = maps:groups_from_list(fun({{X, _}, _}) -> X end, Streams0),
+    maps:fold(
+        fun(RankX, Streams, Acc) ->
+            select_streams(SubId, RankX, Streams, S) ++ Acc
+        end,
+        [],
+        TopicStreamGroups
+    ).
+
+select_streams(SubId, RankX, Streams0, S) ->
+    %% 1. Find the streams with the rank Y greater than the recorded one:
+    Streams1 =
+        case emqx_persistent_session_ds_state:get_rank({SubId, RankX}, S) of
+            undefined ->
+                Streams0;
+            ReplayedY ->
+                [I || I = {{_, Y}, _} <- Streams0, Y > ReplayedY]
+        end,
+    %% 2. Sort streams by rank Y:
+    Streams = lists:sort(
+        fun({{_, Y1}, _}, {{_, Y2}, _}) ->
+            Y1 =< Y2
+        end,
+        Streams1
+    ),
+    %% 3. Select streams with the least rank Y:
+    case Streams of
+        [] ->
+            [];
+        [{{_, MinRankY}, _} | _] ->
+            lists:takewhile(fun({{_, Y}, _}) -> Y =:= MinRankY end, Streams)
+    end.
+
+%% @doc Remove fully acked streams for the deleted subscriptions.
+-spec remove_unsubscribed_streams(emqx_persistent_session_ds_state:t()) ->
+    emqx_persistent_session_ds_state:t().
+remove_unsubscribed_streams(S0) ->
+    CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S0),
+    CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S0),
+    emqx_persistent_session_ds_state:fold_streams(
+        fun(Key, ReplayState, S1) ->
+            case
+                ReplayState#srs.unsubscribed andalso is_fully_acked(CommQos1, CommQos2, ReplayState)
+            of
+                true ->
+                    emqx_persistent_session_ds_state:del_stream(Key, S1);
+                false ->
+                    S1
+            end
+        end,
+        S0,
+        S0
+    ).
+
+%% @doc Advance RankY for each RankX that doesn't have any unreplayed
+%% streams.
+%%
+%% Drop streams with the fully replayed rank. This function relies on
+%% the fact that all streams with the same RankX have also the same
+%% RankY.
+-spec remove_fully_replayed_streams(emqx_persistent_session_ds_state:t()) ->
+    emqx_persistent_session_ds_state:t().
+remove_fully_replayed_streams(S0) ->
+    CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S0),
+    CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S0),
+    %% 1. For each subscription, find the X ranks that were fully replayed:
+    Groups = emqx_persistent_session_ds_state:fold_streams(
+        fun({SubId, _Stream}, StreamState = #srs{rank_x = RankX, rank_y = RankY}, Acc) ->
+            Key = {SubId, RankX},
+            case {is_fully_replayed(CommQos1, CommQos2, StreamState), Acc} of
+                {_, #{Key := false}} ->
+                    Acc;
+                {true, #{Key := {true, RankY}}} ->
+                    Acc;
+                {true, #{Key := {true, _RankYOther}}} ->
+                    %% assert, should never happen
+                    error(multiple_rank_y_for_rank_x);
+                {true, #{}} ->
+                    Acc#{Key => {true, RankY}};
+                {false, #{}} ->
+                    Acc#{Key => false}
+            end
+        end,
+        #{},
+        S0
+    ),
+    %% 2. Advance rank y for each fully replayed set of streams:
+    S1 = maps:fold(
+        fun
+            (Key, {true, RankY}, Acc) ->
+                emqx_persistent_session_ds_state:put_rank(Key, RankY, Acc);
+            (_, _, Acc) ->
+                Acc
+        end,
+        S0,
+        Groups
+    ),
+    %% 3. Remove the fully replayed streams:
+    emqx_persistent_session_ds_state:fold_streams(
+        fun(Key = {SubId, _Stream}, #srs{rank_x = RankX, rank_y = RankY}, Acc) ->
+            case emqx_persistent_session_ds_state:get_rank({SubId, RankX}, Acc) of
+                undefined ->
+                    Acc;
+                MinRankY when RankY =< MinRankY ->
+                    ?SLOG(debug, #{
+                        msg => del_fully_preplayed_stream,
+                        key => Key,
+                        rank => {RankX, RankY},
+                        min => MinRankY
+                    }),
+                    emqx_persistent_session_ds_state:del_stream(Key, Acc);
+                _ ->
+                    Acc
+            end
+        end,
+        S1,
+        S1
+    ).
+
+%% @doc Compare the streams by the order in which they were replayed.
+compare_streams(
+    {_KeyA, #srs{first_seqno_qos1 = A1, first_seqno_qos2 = A2}},
+    {_KeyB, #srs{first_seqno_qos1 = B1, first_seqno_qos2 = B2}}
+) ->
+    case A1 =:= B1 of
+        true ->
+            A2 =< B2;
+        false ->
+            A1 < B1
+    end.
+
+is_fully_replayed(Comm1, Comm2, S = #srs{it_end = It}) ->
+    It =:= end_of_stream andalso is_fully_acked(Comm1, Comm2, S).
+
+is_fully_acked(_, _, #srs{
+    first_seqno_qos1 = Q1, last_seqno_qos1 = Q1, first_seqno_qos2 = Q2, last_seqno_qos2 = Q2
+}) ->
+    %% Streams where the last chunk doesn't contain any QoS1 and 2
+    %% messages are considered fully acked:
+    true;
+is_fully_acked(Comm1, Comm2, #srs{last_seqno_qos1 = S1, last_seqno_qos2 = S2}) ->
+    (Comm1 >= S1) andalso (Comm2 >= S2).
+
+-spec shuffle([A]) -> [A].
+shuffle(L0) ->
+    L1 = lists:map(
+        fun(A) ->
+            %% maybe topic/stream prioritization could be introduced here?
+            {rand:uniform(), A}
+        end,
+        L0
+    ),
+    L2 = lists:sort(L1),
+    {_, L} = lists:unzip(L2),
+    L.

apps/emqx/src/emqx_persistent_session_ds_subs.erl

@@ -0,0 +1,154 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc This module encapsulates the data related to the client's
+%% subscriptions. It tries to reppresent the subscriptions as if they
+%% were a simple key-value map.
+%%
+%% In reality, however, the session has to retain old the
+%% subscriptions for longer to ensure the consistency of message
+%% replay.
+-module(emqx_persistent_session_ds_subs).
+
+%% API:
+-export([on_subscribe/3, on_unsubscribe/3, gc/1, lookup/2, to_map/1, fold/3, fold_all/3]).
+
+-export_type([]).
+
+%%================================================================================
+%% Type declarations
+%%================================================================================
+
+%%================================================================================
+%% API functions
+%%================================================================================
+
+%% @doc Process a new subscription
+-spec on_subscribe(
+    emqx_persistent_session_ds:topic_filter(),
+    emqx_persistent_session_ds:subscription(),
+    emqx_persistent_session_ds_state:t()
+) ->
+    emqx_persistent_session_ds_state:t().
+on_subscribe(TopicFilter, Subscription, S) ->
+    emqx_persistent_session_ds_state:put_subscription(TopicFilter, [], Subscription, S).
+
+%% @doc Process UNSUBSCRIBE
+-spec on_unsubscribe(
+    emqx_persistent_session_ds:topic_filter(),
+    emqx_persistent_session_ds:subscription(),
+    emqx_persistent_session_ds_state:t()
+) ->
+    emqx_persistent_session_ds_state:t().
+on_unsubscribe(TopicFilter, Subscription0, S0) ->
+    %% Note: we cannot delete the subscription immediately, since its
+    %% metadata can be used during replay (see `process_batch'). We
+    %% instead mark it as deleted, and let `subscription_gc' function
+    %% dispatch it later:
+    Subscription = Subscription0#{deleted => true},
+    emqx_persistent_session_ds_state:put_subscription(TopicFilter, [], Subscription, S0).
+
+%% @doc Remove subscriptions that have been marked for deletion, and
+%% that don't have any unacked messages:
+-spec gc(emqx_persistent_session_ds_state:t()) -> emqx_persistent_session_ds_state:t().
+gc(S0) ->
+    fold_all(
+        fun(TopicFilter, #{id := SubId, deleted := Deleted}, Acc) ->
+            case Deleted andalso has_no_unacked_streams(SubId, S0) of
+                true ->
+                    emqx_persistent_session_ds_state:del_subscription(TopicFilter, [], Acc);
+                false ->
+                    Acc
+            end
+        end,
+        S0,
+        S0
+    ).
+
+%% @doc Fold over active subscriptions:
+-spec lookup(emqx_persistent_session_ds:topic_filter(), emqx_persistent_session_ds_state:t()) ->
+    emqx_persistent_session_ds:subscription() | undefined.
+lookup(TopicFilter, S) ->
+    Subs = emqx_persistent_session_ds_state:get_subscriptions(S),
+    case emqx_topic_gbt:lookup(TopicFilter, [], Subs, undefined) of
+        #{deleted := true} ->
+            undefined;
+        Sub ->
+            Sub
+    end.
+
+%% @doc Convert active subscriptions to a map, for information
+%% purpose:
+-spec to_map(emqx_persistent_session_ds_state:t()) -> map().
+to_map(S) ->
+    fold(
+        fun(TopicFilter, #{props := Props}, Acc) -> Acc#{TopicFilter => Props} end,
+        #{},
+        S
+    ).
+
+%% @doc Fold over active subscriptions:
+-spec fold(
+    fun((emqx_types:topic(), emqx_persistent_session_ds:subscription(), Acc) -> Acc),
+    Acc,
+    emqx_persistent_session_ds_state:t()
+) ->
+    Acc.
+fold(Fun, AccIn, S) ->
+    fold_all(
+        fun(TopicFilter, Sub = #{deleted := Deleted}, Acc) ->
+            case Deleted of
+                true -> Acc;
+                false -> Fun(TopicFilter, Sub, Acc)
+            end
+        end,
+        AccIn,
+        S
+    ).
+
+%% @doc Fold over all subscriptions, including inactive ones:
+-spec fold_all(
+    fun((emqx_types:topic(), emqx_persistent_session_ds:subscription(), Acc) -> Acc),
+    Acc,
+    emqx_persistent_session_ds_state:t()
+) ->
+    Acc.
+fold_all(Fun, AccIn, S) ->
+    Subs = emqx_persistent_session_ds_state:get_subscriptions(S),
+    emqx_topic_gbt:fold(
+        fun(Key, Sub, Acc) -> Fun(emqx_topic_gbt:get_topic(Key), Sub, Acc) end,
+        AccIn,
+        Subs
+    ).
+
+%%================================================================================
+%% Internal functions
+%%================================================================================
+
+-spec has_no_unacked_streams(
+    emqx_persistent_session_ds:subscription_id(), emqx_persistent_session_ds_state:t()
+) -> boolean().
+has_no_unacked_streams(SubId, S) ->
+    emqx_persistent_session_ds_state:fold_streams(
+        fun
+            ({SID, _Stream}, Srs, Acc) when SID =:= SubId ->
+                emqx_persistent_session_ds_stream_scheduler:is_fully_acked(Srs, S) andalso Acc;
+            (_StreamKey, _Srs, Acc) ->
+                Acc
+        end,
+        true,
+        S
+    ).

apps/emqx/src/emqx_persistent_session_ds_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -48,13 +48,14 @@ init(Opts) ->
 
 do_init(_Opts) ->
     SupFlags = #{
-        strategy => rest_for_one,
+        strategy => one_for_one,
         intensity => 10,
         period => 2,
         auto_shutdown => never
     },
     CoreChildren = [
-        worker(gc_worker, emqx_persistent_session_ds_gc_worker, [])
+        worker(session_gc_worker, emqx_persistent_session_ds_gc_worker, []),
+        worker(message_gc_worker, emqx_persistent_message_ds_gc_worker, [])
     ],
     Children =
         case mria_rlog:role() of

apps/emqx/src/emqx_pool.erl

@@ -28,11 +28,15 @@
     submit/1,
     submit/2,
     async_submit/1,
-    async_submit/2
+    async_submit/2,
+    submit_to_pool/2,
+    submit_to_pool/3,
+    async_submit_to_pool/2,
+    async_submit_to_pool/3
 ]).
 
 -ifdef(TEST).
--export([worker/0, flush_async_tasks/0]).
+-export([worker/0, flush_async_tasks/0, flush_async_tasks/1]).
 -endif.
 
 %% gen_server callbacks
@@ -57,7 +61,7 @@
 -spec start_link(atom(), pos_integer()) -> startlink_ret().
 start_link(Pool, Id) ->
     gen_server:start_link(
-        {local, emqx_utils:proc_name(?MODULE, Id)},
+        {local, emqx_utils:proc_name(Pool, Id)},
         ?MODULE,
         [Pool, Id],
         [{hibernate_after, 1000}]
@@ -66,32 +70,48 @@ start_link(Pool, Id) ->
 %% @doc Submit work to the pool.
 -spec submit(task()) -> any().
 submit(Task) ->
-    call({submit, Task}).
+    submit_to_pool(?POOL, Task).
 
 -spec submit(fun(), list(any())) -> any().
 submit(Fun, Args) ->
-    call({submit, {Fun, Args}}).
+    submit_to_pool(?POOL, Fun, Args).
-
-%% @private
-call(Req) ->
-    gen_server:call(worker(), Req, infinity).
 
 %% @doc Submit work to the pool asynchronously.
 -spec async_submit(task()) -> ok.
 async_submit(Task) ->
-    cast({async_submit, Task}).
+    async_submit_to_pool(?POOL, Task).
 
 -spec async_submit(fun(), list(any())) -> ok.
 async_submit(Fun, Args) ->
-    cast({async_submit, {Fun, Args}}).
+    async_submit_to_pool(?POOL, Fun, Args).
+
+-spec submit_to_pool(any(), task()) -> any().
+submit_to_pool(Pool, Task) ->
+    call(Pool, {submit, Task}).
+
+-spec submit_to_pool(any(), fun(), list(any())) -> any().
+submit_to_pool(Pool, Fun, Args) ->
+    call(Pool, {submit, {Fun, Args}}).
+
+-spec async_submit_to_pool(any(), task()) -> ok.
+async_submit_to_pool(Pool, Task) ->
+    cast(Pool, {async_submit, Task}).
+
+-spec async_submit_to_pool(any(), fun(), list(any())) -> ok.
+async_submit_to_pool(Pool, Fun, Args) ->
+    cast(Pool, {async_submit, {Fun, Args}}).
 
 %% @private
-cast(Msg) ->
+call(Pool, Req) ->
-    gen_server:cast(worker(), Msg).
+    gen_server:call(worker(Pool), Req, infinity).
 
 %% @private
-worker() ->
+cast(Pool, Msg) ->
-    gproc_pool:pick_worker(?POOL).
+    gen_server:cast(worker(Pool), Msg).
+
+%% @private
+worker(Pool) ->
+    gproc_pool:pick_worker(Pool).
 
 %%--------------------------------------------------------------------
 %% gen_server callbacks
@@ -146,15 +166,25 @@ run(Fun) when is_function(Fun) ->
     Fun().
 
 -ifdef(TEST).
+
+worker() ->
+    worker(?POOL).
+
+flush_async_tasks() ->
+    flush_async_tasks(?POOL).
+
 %% This help function creates a large enough number of async tasks
 %% to force flush the pool workers.
 %% The number of tasks should be large enough to ensure all workers have
 %% the chance to work on at least one of the tasks.
-flush_async_tasks() ->
+flush_async_tasks(Pool) ->
     Ref = make_ref(),
     Self = self(),
     L = lists:seq(1, 997),
-    lists:foreach(fun(I) -> emqx_pool:async_submit(fun() -> Self ! {done, Ref, I} end, []) end, L),
+    lists:foreach(
+        fun(I) -> emqx_pool:async_submit_to_pool(Pool, fun() -> Self ! {done, Ref, I} end, []) end,
+        L
+    ),
     lists:foreach(
         fun(I) ->
             receive

apps/emqx/src/emqx_router.erl

@@ -24,9 +24,7 @@
 -include_lib("emqx/include/emqx_router.hrl").
 
 %% Mnesia bootstrap
--export([mnesia/1]).
+-export([create_tables/0]).
-
--boot_mnesia({mnesia, [boot]}).
 
 -export([start_link/2]).
 
@@ -123,7 +121,7 @@
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------
 
-mnesia(boot) ->
+create_tables() ->
     mria_config:set_dirty_shard(?ROUTE_SHARD, true),
     ok = mria:create_table(?ROUTE_TAB, [
         {type, bag},
@@ -151,7 +149,8 @@ mnesia(boot) ->
                 {decentralized_counters, true}
             ]}
         ]}
-    ]).
+    ]),
+    [?ROUTE_TAB, ?ROUTE_TAB_FILTERS].
 
 %%--------------------------------------------------------------------
 %% Start a router

apps/emqx/src/emqx_router_helper.erl

@@ -25,9 +25,7 @@
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 %% Mnesia bootstrap
--export([mnesia/1]).
+-export([create_tables/0]).
-
--boot_mnesia({mnesia, [boot]}).
 
 %% API
 -export([
@@ -63,7 +61,7 @@
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------
 
-mnesia(boot) ->
+create_tables() ->
     ok = mria:create_table(?ROUTING_NODE, [
         {type, set},
         {rlog_shard, ?ROUTE_SHARD},
@@ -71,7 +69,8 @@ mnesia(boot) ->
         {record_name, routing_node},
         {attributes, record_info(fields, routing_node)},
         {storage_properties, [{ets, [{read_concurrency, true}]}]}
-    ]).
+    ]),
+    [?ROUTING_NODE].
 
 %%--------------------------------------------------------------------
 %% API

apps/emqx/src/emqx_router_sup.erl

@@ -24,6 +24,11 @@
 
 start_link() ->
     %% Init and log routing table type
+    ok = mria:wait_for_tables(
+        emqx_trie:create_trie() ++
+            emqx_router:create_tables() ++
+            emqx_router_helper:create_tables()
+    ),
     ok = emqx_router:init_schema(),
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 

apps/emqx/src/emqx_schema.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -94,6 +94,7 @@
     non_empty_string/1,
     validations/0,
     naive_env_interpolation/1,
+    ensure_unicode_path/2,
     validate_server_ssl_opts/1,
     validate_tcp_keepalive/1,
     parse_tcp_keepalive/1
@@ -181,7 +182,7 @@
 -define(DEFAULT_MULTIPLIER, 1.5).
 -define(DEFAULT_BACKOFF, 0.75).
 
-namespace() -> broker.
+namespace() -> emqx.
 
 tags() ->
     [<<"EMQX">>].
@@ -229,7 +230,7 @@ roots(high) ->
         );
 roots(medium) ->
     [
-        {"broker",
+        {broker,
             sc(
                 ref("broker"),
                 #{
@@ -1103,6 +1104,14 @@ fields("ws_opts") ->
             sc(
                 ref("deflate_opts"),
                 #{}
+            )},
+        {"validate_utf8",
+            sc(
+                boolean(),
+                #{
+                    default => true,
+                    desc => ?DESC(fields_ws_opts_validate_utf8)
+                }
             )}
     ];
 fields("tcp_opts") ->
@@ -1338,24 +1347,43 @@ fields("deflate_opts") ->
     ];
 fields("broker") ->
     [
-        {"enable_session_registry",
+        {enable_session_registry,
             sc(
                 boolean(),
                 #{
                     default => true,
+                    importance => ?IMPORTANCE_HIGH,
                     desc => ?DESC(broker_enable_session_registry)
                 }
             )},
-        {"session_locking_strategy",
+        {session_history_retain,
+            sc(
+                duration_s(),
+                #{
+                    default => <<"0s">>,
+                    importance => ?IMPORTANCE_LOW,
+                    desc => ?DESC("broker_session_history_retain")
+                }
+            )},
+        {session_locking_strategy,
             sc(
                 hoconsc:enum([local, leader, quorum, all]),
                 #{
                     default => quorum,
+                    importance => ?IMPORTANCE_HIDDEN,
                     desc => ?DESC(broker_session_locking_strategy)
                 }
             )},
-        shared_subscription_strategy(),
+        %% moved to under mqtt root
-        {"shared_dispatch_ack_enabled",
+        {shared_subscription_strategy,
+            sc(
+                string(),
+                #{
+                    deprecated => {since, "5.1.0"},
+                    importance => ?IMPORTANCE_HIDDEN
+                }
+            )},
+        {shared_dispatch_ack_enabled,
             sc(
                 boolean(),
                 #{
@@ -1365,7 +1393,7 @@ fields("broker") ->
                     desc => ?DESC(broker_shared_dispatch_ack_enabled)
                 }
             )},
-        {"route_batch_clean",
+        {route_batch_clean,
             sc(
                 boolean(),
                 #{
@@ -1374,18 +1402,18 @@ fields("broker") ->
                     importance => ?IMPORTANCE_HIDDEN
                 }
             )},
-        {"perf",
+        {perf,
             sc(
                 ref("broker_perf"),
                 #{importance => ?IMPORTANCE_HIDDEN}
             )},
-        {"routing",
+        {routing,
             sc(
                 ref("broker_routing"),
                 #{importance => ?IMPORTANCE_HIDDEN}
             )},
         %% FIXME: Need new design for shared subscription group
-        {"shared_subscription_group",
+        {shared_subscription_group,
             sc(
                 map(name, ref("shared_subscription_group")),
                 #{
@@ -1801,7 +1829,7 @@ fields("session_persistence") ->
             sc(
                 pos_integer(),
                 #{
-                    default => 1000,
+                    default => 100,
                     desc => ?DESC(session_ds_max_batch_size)
                 }
             )},
@@ -1854,6 +1882,14 @@ fields("session_persistence") ->
                     desc => ?DESC(session_ds_session_gc_batch_size)
                 }
             )},
+        {"message_retention_period",
+            sc(
+                timeout_duration(),
+                #{
+                    default => <<"1d">>,
+                    desc => ?DESC(session_ds_message_retention_period)
+                }
+            )},
         {"force_persistence",
             sc(
                 boolean(),
@@ -1882,6 +1918,16 @@ fields("session_storage_backend_builtin") ->
                     default => true
                 }
             )},
+        {"data_dir",
+            sc(
+                string(),
+                #{
+                    desc => ?DESC(session_builtin_data_dir),
+                    mapping => "emqx_durable_storage.db_data_dir",
+                    required => false,
+                    importance => ?IMPORTANCE_LOW
+                }
+            )},
         {"n_shards",
             sc(
                 pos_integer(),
@@ -1897,6 +1943,24 @@ fields("session_storage_backend_builtin") ->
                     default => 3,
                     importance => ?IMPORTANCE_HIDDEN
                 }
+            )},
+        {"egress_batch_size",
+            sc(
+                pos_integer(),
+                #{
+                    default => 1000,
+                    mapping => "emqx_durable_storage.egress_batch_size",
+                    importance => ?IMPORTANCE_HIDDEN
+                }
+            )},
+        {"egress_flush_interval",
+            sc(
+                timeout_duration_ms(),
+                #{
+                    default => 100,
+                    mapping => "emqx_durable_storage.egress_flush_interval",
+                    importance => ?IMPORTANCE_HIDDEN
+                }
             )}
     ].
 
@@ -3595,7 +3659,22 @@ mqtt_general() ->
                     desc => ?DESC(mqtt_shared_subscription)
                 }
             )},
-        shared_subscription_strategy(),
+        {"shared_subscription_strategy",
+            sc(
+                hoconsc:enum([
+                    random,
+                    round_robin,
+                    round_robin_per_group,
+                    sticky,
+                    local,
+                    hash_topic,
+                    hash_clientid
+                ]),
+                #{
+                    default => round_robin,
+                    desc => ?DESC(mqtt_shared_subscription_strategy)
+                }
+            )},
         {"exclusive_subscription",
             sc(
                 boolean(),
@@ -3700,6 +3779,15 @@ mqtt_session() ->
                     importance => ?IMPORTANCE_LOW
                 }
             )},
+        {"message_expiry_interval",
+            sc(
+                hoconsc:union([duration(), infinity]),
+                #{
+                    default => infinity,
+                    desc => ?DESC(mqtt_message_expiry_interval),
+                    importance => ?IMPORTANCE_LOW
+                }
+            )},
         {"max_awaiting_rel",
             sc(
                 hoconsc:union([non_neg_integer(), infinity]),
@@ -3792,24 +3880,6 @@ mqtt_session() ->
             )}
     ].
 
-shared_subscription_strategy() ->
-    {"shared_subscription_strategy",
-        sc(
-            hoconsc:enum([
-                random,
-                round_robin,
-                round_robin_per_group,
-                sticky,
-                local,
-                hash_topic,
-                hash_clientid
-            ]),
-            #{
-                default => round_robin,
-                desc => ?DESC(broker_shared_subscription_strategy)
-            }
-        )}.
-
 default_mem_check_interval() ->
     case emqx_os_mon:is_os_check_supported() of
         true -> <<"60s">>;
@@ -3836,3 +3906,20 @@ tags_schema() ->
             importance => ?IMPORTANCE_LOW
         }
     ).
+
+ensure_unicode_path(undefined, _) ->
+    undefined;
+ensure_unicode_path(Path, #{make_serializable := true}) ->
+    %% format back to serializable string
+    unicode:characters_to_binary(Path, utf8);
+ensure_unicode_path(Path, Opts) when is_binary(Path) ->
+    case unicode:characters_to_list(Path, utf8) of
+        {R, _, _} when R =:= error orelse R =:= incomplete ->
+            throw({"bad_file_path_string", Path});
+        PathStr ->
+            ensure_unicode_path(PathStr, Opts)
+    end;
+ensure_unicode_path(Path, _) when is_list(Path) ->
+    Path;
+ensure_unicode_path(Path, _) ->
+    throw({"not_string", Path}).

apps/emqx/src/emqx_session.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -135,7 +135,7 @@
 -type custom_timer_name() :: atom().
 
 -type message() :: emqx_types:message().
--type publish() :: {maybe(emqx_types:packet_id()), emqx_types:message()}.
+-type publish() :: {option(emqx_types:packet_id()), emqx_types:message()}.
 -type pubrel() :: {pubrel, emqx_types:packet_id()}.
 -type reply() :: publish() | pubrel().
 -type replies() :: [reply()] | reply().
@@ -409,12 +409,8 @@ enrich_delivers(ClientInfo, Delivers, Session) ->
 enrich_delivers(_ClientInfo, [], _UpgradeQoS, _Session) ->
     [];
 enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) ->
-    case enrich_deliver(ClientInfo, D, UpgradeQoS, Session) of
+    enrich_deliver(ClientInfo, D, UpgradeQoS, Session) ++
-        [] ->
+        enrich_delivers(ClientInfo, Rest, UpgradeQoS, Session).
-            enrich_delivers(ClientInfo, Rest, UpgradeQoS, Session);
-        Msg ->
-            [Msg | enrich_delivers(ClientInfo, Rest, UpgradeQoS, Session)]
-    end.
 
 enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) ->
     SubOpts =
@@ -435,13 +431,15 @@ enrich_message(
     _ = emqx_session_events:handle_event(ClientInfo, {dropped, Msg, no_local}),
     [];
 enrich_message(_ClientInfo, MsgIn, SubOpts = #{}, UpgradeQoS) ->
+    [
         maps:fold(
             fun(SubOpt, V, Msg) -> enrich_subopts(SubOpt, V, Msg, UpgradeQoS) end,
             MsgIn,
             SubOpts
-    );
+        )
+    ];
 enrich_message(_ClientInfo, Msg, undefined, _UpgradeQoS) ->
-    Msg.
+    [Msg].
 
 enrich_subopts(nl, 1, Msg, _) ->
     emqx_message:set_flag(nl, Msg);

apps/emqx/src/emqx_session_events.erl

@@ -62,10 +62,10 @@ handle_event(ClientInfo, {dropped, Msg, #{reason := queue_full, logctx := Ctx}})
     ok = emqx_metrics:inc('delivery.dropped.queue_full'),
     ok = inc_pd('send_msg.dropped', 1),
     ok = inc_pd('send_msg.dropped.queue_full', 1),
-    ?SLOG(
+    ?SLOG_THROTTLE(
         warning,
         Ctx#{
-            msg => "dropped_msg_due_to_mqueue_is_full",
+            msg => dropped_msg_due_to_mqueue_is_full,
             payload => Msg#message.payload
         },
         #{topic => Msg#message.topic}

apps/emqx/src/emqx_session_mem.erl

@@ -468,12 +468,12 @@ dequeue(ClientInfo, Session = #session{inflight = Inflight, mqueue = Q}) ->
 
 dequeue(_ClientInfo, 0, Msgs, Q) ->
     {lists:reverse(Msgs), Q};
-dequeue(ClientInfo, Cnt, Msgs, Q) ->
+dequeue(ClientInfo = #{zone := Zone}, Cnt, Msgs, Q) ->
     case emqx_mqueue:out(Q) of
         {empty, _Q} ->
             dequeue(ClientInfo, 0, Msgs, Q);
         {{value, Msg}, Q1} ->
-            case emqx_message:is_expired(Msg) of
+            case emqx_message:is_expired(Msg, Zone) of
                 true ->
                     _ = emqx_session_events:handle_event(ClientInfo, {expired, Msg}),
                     dequeue(ClientInfo, Cnt, Msgs, Q1);
@@ -619,14 +619,14 @@ retry_delivery(
     end.
 
 do_retry_delivery(
-    ClientInfo,
+    ClientInfo = #{zone := Zone},
     PacketId,
     #inflight_data{phase = wait_ack, message = Msg} = Data,
     Now,
     Acc,
     Inflight
 ) ->
-    case emqx_message:is_expired(Msg) of
+    case emqx_message:is_expired(Msg, Zone) of
         true ->
             _ = emqx_session_events:handle_event(ClientInfo, {expired, Msg}),
             {Acc, emqx_inflight:delete(PacketId, Inflight)};

apps/emqx/src/emqx_shared_sub.erl

@@ -25,9 +25,7 @@
 -include("types.hrl").
 
 %% Mnesia bootstrap
--export([mnesia/1]).
+-export([create_tables/0]).
-
--boot_mnesia({mnesia, [boot]}).
 
 %% APIs
 -export([start_link/0]).
@@ -107,14 +105,15 @@
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------
 
-mnesia(boot) ->
+create_tables() ->
     ok = mria:create_table(?TAB, [
         {type, bag},
         {rlog_shard, ?SHARED_SUB_SHARD},
         {storage, ram_copies},
         {record_name, emqx_shared_subscription},
         {attributes, record_info(fields, emqx_shared_subscription)}
-    ]).
+    ]),
+    [?TAB].
 
 %%--------------------------------------------------------------------
 %% API

apps/emqx/src/emqx_stats.erl

@@ -62,7 +62,7 @@
 -record(update, {name, countdown, interval, func}).
 
 -record(state, {
-    timer :: maybe(reference()),
+    timer :: option(reference()),
     updates :: [#update{}],
     tick_ms :: timeout()
 }).
@@ -99,7 +99,11 @@
     [
         'sessions.count',
         %% Maximum Number of Concurrent Sessions
-        'sessions.max'
+        'sessions.max',
+        %% Count of Sessions in the cluster
+        'cluster_sessions.count',
+        %% Maximum Number of Sessions in the cluster
+        'cluster_sessions.max'
     ]
 ).
 
@@ -164,6 +168,8 @@ names() ->
         emqx_connections_max,
         emqx_live_connections_count,
         emqx_live_connections_max,
+        emqx_cluster_sessions_count,
+        emqx_cluster_sessions_max,
         emqx_sessions_count,
         emqx_sessions_max,
         emqx_channels_count,

apps/emqx/src/emqx_sys.erl

@@ -65,8 +65,8 @@
 -import(emqx_utils, [start_timer/2]).
 
 -record(state, {
-    heartbeat :: maybe(reference()),
+    heartbeat :: option(reference()),
-    ticker :: maybe(reference()),
+    ticker :: option(reference()),
     sysdescr :: binary()
 }).
 

apps/emqx/src/emqx_sys_sup.erl

@@ -22,6 +22,7 @@
 -export([init/1]).
 
 start_link() ->
+    _ = mria:wait_for_tables(emqx_alarm:create_tables()),
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 init([]) ->

apps/emqx/src/emqx_topic_gbt.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -39,11 +39,11 @@
 -type match(ID) :: key(ID).
 
 -opaque t(ID, Value) :: gb_trees:tree(key(ID), Value).
--opaque t() :: t(_ID, _Value).
+-type t() :: t(_ID, _Value).
 
 %% @doc Create a new gb_tree and store it in the persitent_term with the
 %% given name.
--spec new() -> t().
+-spec new() -> t(_ID, _Value).
 new() ->
     gb_trees:empty().
 
@@ -54,19 +54,19 @@ size(Gbt) ->
 %% @doc Insert a new entry into the index that associates given topic filter to given
 %% record ID, and attaches arbitrary record to the entry. This allows users to choose
 %% between regular and "materialized" indexes, for example.
--spec insert(emqx_types:topic() | words(), _ID, _Record, t()) -> t().
+-spec insert(emqx_types:topic() | words(), ID, Record, t(ID, Record)) -> t(ID, Record).
 insert(Filter, ID, Record, Gbt) ->
     Key = key(Filter, ID),
     gb_trees:enter(Key, Record, Gbt).
 
 %% @doc Delete an entry from the index that associates given topic filter to given
 %% record ID. Deleting non-existing entry is not an error.
--spec delete(emqx_types:topic() | words(), _ID, t()) -> t().
+-spec delete(emqx_types:topic() | words(), ID, t(ID, Record)) -> t(ID, Record).
 delete(Filter, ID, Gbt) ->
     Key = key(Filter, ID),
     gb_trees:delete_any(Key, Gbt).
 
--spec lookup(emqx_types:topic() | words(), _ID, t(), Default) -> _Record | Default.
+-spec lookup(emqx_types:topic() | words(), ID, t(ID, Record), Default) -> Record | Default.
 lookup(Filter, ID, Gbt, Default) ->
     Key = key(Filter, ID),
     case gb_trees:lookup(Key, Gbt) of
@@ -76,7 +76,7 @@ lookup(Filter, ID, Gbt, Default) ->
             Default
     end.
 
--spec fold(fun((key(_ID), _Record, Acc) -> Acc), Acc, t()) -> Acc.
+-spec fold(fun((key(ID), Record, Acc) -> Acc), Acc, t(ID, Record)) -> Acc.
 fold(Fun, Acc, Gbt) ->
     Iter = gb_trees:iterator(Gbt),
     fold_iter(Fun, Acc, Iter).
@@ -91,13 +91,13 @@ fold_iter(Fun, Acc, Iter) ->
 
 %% @doc Match given topic against the index and return the first match, or `false` if
 %% no match is found.
--spec match(emqx_types:topic(), t()) -> match(_ID) | false.
+-spec match(emqx_types:topic(), t(ID, _Record)) -> match(ID) | false.
 match(Topic, Gbt) ->
     emqx_trie_search:match(Topic, make_nextf(Gbt)).
 
 %% @doc Match given topic against the index and return _all_ matches.
 %% If `unique` option is given, return only unique matches by record ID.
--spec matches(emqx_types:topic(), t(), emqx_trie_search:opts()) -> [match(_ID)].
+-spec matches(emqx_types:topic(), t(ID, _Record), emqx_trie_search:opts()) -> [match(ID)].
 matches(Topic, Gbt, Opts) ->
     emqx_trie_search:matches(Topic, make_nextf(Gbt), Opts).
 
@@ -112,7 +112,7 @@ get_topic(Key) ->
     emqx_trie_search:get_topic(Key).
 
 %% @doc Fetch the record associated with the match.
--spec get_record(match(_ID), t()) -> _Record.
+-spec get_record(match(ID), t(ID, Record)) -> Record.
 get_record(Key, Gbt) ->
     gb_trees:get(Key, Gbt).
 

apps/emqx/src/emqx_trie.erl

@@ -20,13 +20,11 @@
 
 %% Mnesia bootstrap
 -export([
-    mnesia/1,
+    create_trie/0,
     wait_for_tables/0,
     create_session_trie/1
 ]).
 
--boot_mnesia({mnesia, [boot]}).
-
 %% Trie APIs
 -export([
     insert/1,
@@ -65,8 +63,8 @@
 %%--------------------------------------------------------------------
 
 %% @doc Create or replicate topics table.
--spec mnesia(boot | copy) -> ok.
+-spec create_trie() -> [mria:table()].
-mnesia(boot) ->
+create_trie() ->
     %% Optimize storage
     StoreProps = [
         {ets, [
@@ -80,7 +78,8 @@ mnesia(boot) ->
         {attributes, record_info(fields, ?TRIE)},
         {type, ordered_set},
         {storage_properties, StoreProps}
-    ]).
+    ]),
+    [?TRIE].
 
 create_session_trie(Type) ->
     Storage =

apps/emqx/src/emqx_types.erl

@@ -100,6 +100,7 @@
 
 -export_type([
     banned/0,
+    banned_who/0,
     command/0
 ]).
 
@@ -173,7 +174,7 @@
     atom() => term()
 }.
 -type clientinfo() :: #{
-    zone := maybe(zone()),
+    zone := option(zone()),
     protocol := protocol(),
     peerhost := peerhost(),
     sockport := non_neg_integer(),
@@ -181,9 +182,9 @@
     username := username(),
     is_bridge := boolean(),
     is_superuser := boolean(),
-    mountpoint := maybe(binary()),
+    mountpoint := option(binary()),
-    ws_cookie => maybe(list()),
+    ws_cookie => option(list()),
-    password => maybe(binary()),
+    password => option(binary()),
     auth_result => auth_result(),
     anonymous => boolean(),
     cn => binary(),
@@ -191,8 +192,8 @@
     atom() => term()
 }.
 -type clientid() :: binary() | atom().
--type username() :: maybe(binary()).
+-type username() :: option(binary()).
--type password() :: maybe(binary()).
+-type password() :: option(binary()).
 -type peerhost() :: inet:ip_address().
 -type peername() ::
     {inet:ip_address(), inet:port_number()}
@@ -222,8 +223,8 @@
 -type packet_id() :: 1..16#FFFF.
 -type alias_id() :: 0..16#FFFF.
 -type topic_aliases() :: #{
-    inbound => maybe(map()),
+    inbound => option(map()),
-    outbound => maybe(map())
+    outbound => option(map())
 }.
 -type properties() :: #{atom() => term()}.
 -type topic_filters() :: list({topic(), subopts()}).
@@ -246,6 +247,14 @@
 }.
 
 -type banned() :: #banned{}.
+-type banned_who() ::
+    {clientid, binary()}
+    | {peerhost, inet:ip_address()}
+    | {username, binary()}
+    | {clientid_re, {_RE :: tuple(), binary()}}
+    | {username_re, {_RE :: tuple(), binary()}}
+    | {peerhost_net, esockd_cidr:cidr()}.
+
 -type deliver() :: {deliver, topic(), message()}.
 -type delivery() :: #delivery{}.
 -type deliver_result() :: ok | {ok, non_neg_integer()} | {error, term()}.

apps/emqx/src/emqx_ws_connection.erl

@@ -76,15 +76,15 @@
     %% Channel
     channel :: emqx_channel:channel(),
     %% GC State
-    gc_state :: maybe(emqx_gc:gc_state()),
+    gc_state :: option(emqx_gc:gc_state()),
     %% Postponed Packets|Cmds|Events
     postponed :: list(emqx_types:packet() | ws_cmd() | tuple()),
     %% Stats Timer
-    stats_timer :: disabled | maybe(reference()),
+    stats_timer :: disabled | option(reference()),
     %% Idle Timeout
     idle_timeout :: timeout(),
     %% Idle Timer
-    idle_timer :: maybe(reference()),
+    idle_timer :: option(reference()),
     %% Zone name
     zone :: atom(),
     %% Listener Type and Name
@@ -205,7 +205,8 @@ init(Req, #{listener := {Type, Listener}} = Opts) ->
         compress => get_ws_opts(Type, Listener, compress),
         deflate_opts => get_ws_opts(Type, Listener, deflate_opts),
         max_frame_size => get_ws_opts(Type, Listener, max_frame_size),
-        idle_timeout => get_ws_opts(Type, Listener, idle_timeout)
+        idle_timeout => get_ws_opts(Type, Listener, idle_timeout),
+        validate_utf8 => get_ws_opts(Type, Listener, validate_utf8)
     },
     case check_origin_header(Req, Opts) of
         {error, Reason} ->

apps/emqx/test/emqx_banned_SUITE.erl

@@ -34,7 +34,7 @@ end_per_suite(Config) ->
 
 t_add_delete(_) ->
     Banned = #banned{
-        who = {clientid, <<"TestClient">>},
+        who = emqx_banned:who(clientid, <<"TestClient">>),
         by = <<"banned suite">>,
         reason = <<"test">>,
         at = erlang:system_time(second),
@@ -47,54 +47,91 @@ t_add_delete(_) ->
         emqx_banned:create(Banned#banned{until = erlang:system_time(second) + 100}),
     ?assertEqual(1, emqx_banned:info(size)),
 
-    ok = emqx_banned:delete({clientid, <<"TestClient">>}),
+    ok = emqx_banned:delete(emqx_banned:who(clientid, <<"TestClient">>)),
     ?assertEqual(0, emqx_banned:info(size)).
 
 t_check(_) ->
-    {ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient">>}}),
+    {ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(clientid, <<"BannedClient">>)}),
-    {ok, _} = emqx_banned:create(#banned{who = {username, <<"BannedUser">>}}),
+    {ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(username, <<"BannedUser">>)}),
-    {ok, _} = emqx_banned:create(#banned{who = {peerhost, {192, 168, 0, 1}}}),
+    {ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(peerhost, {192, 168, 0, 1})}),
-    ?assertEqual(3, emqx_banned:info(size)),
+    {ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(peerhost, <<"192.168.0.2">>)}),
-    ClientInfo1 = #{
+    {ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(clientid_re, <<"BannedClientRE.*">>)}),
+    {ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(username_re, <<"BannedUserRE.*">>)}),
+    {ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(peerhost_net, <<"192.168.3.0/24">>)}),
+
+    ?assertEqual(7, emqx_banned:info(size)),
+    ClientInfoBannedClientId = #{
         clientid => <<"BannedClient">>,
         username => <<"user">>,
         peerhost => {127, 0, 0, 1}
     },
-    ClientInfo2 = #{
+    ClientInfoBannedUsername = #{
         clientid => <<"client">>,
         username => <<"BannedUser">>,
         peerhost => {127, 0, 0, 1}
     },
-    ClientInfo3 = #{
+    ClientInfoBannedAddr1 = #{
         clientid => <<"client">>,
         username => <<"user">>,
         peerhost => {192, 168, 0, 1}
     },
-    ClientInfo4 = #{
+    ClientInfoBannedAddr2 = #{
+        clientid => <<"client">>,
+        username => <<"user">>,
+        peerhost => {192, 168, 0, 2}
+    },
+    ClientInfoBannedClientIdRE = #{
+        clientid => <<"BannedClientRE1">>,
+        username => <<"user">>,
+        peerhost => {127, 0, 0, 1}
+    },
+    ClientInfoBannedUsernameRE = #{
+        clientid => <<"client">>,
+        username => <<"BannedUserRE1">>,
+        peerhost => {127, 0, 0, 1}
+    },
+    ClientInfoBannedAddrNet = #{
+        clientid => <<"client">>,
+        username => <<"user">>,
+        peerhost => {192, 168, 3, 1}
+    },
+    ClientInfoValidFull = #{
         clientid => <<"client">>,
         username => <<"user">>,
         peerhost => {127, 0, 0, 1}
     },
-    ClientInfo5 = #{},
+    ClientInfoValidEmpty = #{},
-    ClientInfo6 = #{clientid => <<"client1">>},
+    ClientInfoValidOnlyClientId = #{clientid => <<"client1">>},
-    ?assert(emqx_banned:check(ClientInfo1)),
+    ?assert(emqx_banned:check(ClientInfoBannedClientId)),
-    ?assert(emqx_banned:check(ClientInfo2)),
+    ?assert(emqx_banned:check(ClientInfoBannedUsername)),
-    ?assert(emqx_banned:check(ClientInfo3)),
+    ?assert(emqx_banned:check(ClientInfoBannedAddr1)),
-    ?assertNot(emqx_banned:check(ClientInfo4)),
+    ?assert(emqx_banned:check(ClientInfoBannedAddr2)),
-    ?assertNot(emqx_banned:check(ClientInfo5)),
+    ?assert(emqx_banned:check(ClientInfoBannedClientIdRE)),
-    ?assertNot(emqx_banned:check(ClientInfo6)),
+    ?assert(emqx_banned:check(ClientInfoBannedUsernameRE)),
-    ok = emqx_banned:delete({clientid, <<"BannedClient">>}),
+    ?assert(emqx_banned:check(ClientInfoBannedAddrNet)),
-    ok = emqx_banned:delete({username, <<"BannedUser">>}),
+    ?assertNot(emqx_banned:check(ClientInfoValidFull)),
-    ok = emqx_banned:delete({peerhost, {192, 168, 0, 1}}),
+    ?assertNot(emqx_banned:check(ClientInfoValidEmpty)),
-    ?assertNot(emqx_banned:check(ClientInfo1)),
+    ?assertNot(emqx_banned:check(ClientInfoValidOnlyClientId)),
-    ?assertNot(emqx_banned:check(ClientInfo2)),
+    ok = emqx_banned:delete(emqx_banned:who(clientid, <<"BannedClient">>)),
-    ?assertNot(emqx_banned:check(ClientInfo3)),
+    ok = emqx_banned:delete(emqx_banned:who(username, <<"BannedUser">>)),
-    ?assertNot(emqx_banned:check(ClientInfo4)),
+    ok = emqx_banned:delete(emqx_banned:who(peerhost, {192, 168, 0, 1})),
+    ok = emqx_banned:delete(emqx_banned:who(peerhost, <<"192.168.0.2">>)),
+    ok = emqx_banned:delete(emqx_banned:who(clientid_re, <<"BannedClientRE.*">>)),
+    ok = emqx_banned:delete(emqx_banned:who(username_re, <<"BannedUserRE.*">>)),
+    ok = emqx_banned:delete(emqx_banned:who(peerhost_net, <<"192.168.3.0/24">>)),
+    ?assertNot(emqx_banned:check(ClientInfoBannedClientId)),
+    ?assertNot(emqx_banned:check(ClientInfoBannedUsername)),
+    ?assertNot(emqx_banned:check(ClientInfoBannedAddr1)),
+    ?assertNot(emqx_banned:check(ClientInfoBannedAddr2)),
+    ?assertNot(emqx_banned:check(ClientInfoBannedClientIdRE)),
+    ?assertNot(emqx_banned:check(ClientInfoBannedUsernameRE)),
+    ?assertNot(emqx_banned:check(ClientInfoBannedAddrNet)),
+    ?assertNot(emqx_banned:check(ClientInfoValidFull)),
     ?assertEqual(0, emqx_banned:info(size)).
 
 t_unused(_) ->
-    Who1 = {clientid, <<"BannedClient1">>},
+    Who1 = emqx_banned:who(clientid, <<"BannedClient1">>),
-    Who2 = {clientid, <<"BannedClient2">>},
+    Who2 = emqx_banned:who(clientid, <<"BannedClient2">>),
 
     ?assertMatch(
         {ok, _},
@@ -123,7 +160,7 @@ t_kick(_) ->
     snabbkaffe:start_trace(),
 
     Now = erlang:system_time(second),
-    Who = {clientid, ClientId},
+    Who = emqx_banned:who(clientid, ClientId),
 
     emqx_banned:create(#{
         who => Who,
@@ -194,7 +231,7 @@ t_session_taken(_) ->
     Publish(),
 
     Now = erlang:system_time(second),
-    Who = {clientid, ClientId2},
+    Who = emqx_banned:who(clientid, ClientId2),
     emqx_banned:create(#{
         who => Who,
         by => <<"test">>,

apps/emqx/test/emqx_channel_SUITE.erl

@@ -427,19 +427,32 @@ t_handle_in_auth(_) ->
 
 t_handle_in_frame_error(_) ->
     IdleChannel = channel(#{conn_state => idle}),
-    {shutdown, #{shutdown_count := frame_error, reason := frame_too_large}, _Chan} =
+    {shutdown, #{shutdown_count := frame_too_large, cause := frame_too_large}, _Chan} =
-        emqx_channel:handle_in({frame_error, frame_too_large}, IdleChannel),
+        emqx_channel:handle_in({frame_error, #{cause => frame_too_large}}, IdleChannel),
     ConnectingChan = channel(#{conn_state => connecting}),
     ConnackPacket = ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE),
-    {shutdown, #{shutdown_count := frame_error, reason := frame_too_large}, ConnackPacket, _} =
+    {shutdown,
-        emqx_channel:handle_in({frame_error, frame_too_large}, ConnectingChan),
+        #{
+            shutdown_count := frame_too_large,
+            cause := frame_too_large,
+            limit := 100,
+            received := 101
+        },
+        ConnackPacket,
+        _} =
+        emqx_channel:handle_in(
+            {frame_error, #{cause => frame_too_large, received => 101, limit => 100}},
+            ConnectingChan
+        ),
     DisconnectPacket = ?DISCONNECT_PACKET(?RC_PACKET_TOO_LARGE),
     ConnectedChan = channel(#{conn_state => connected}),
-    {ok, [{outgoing, DisconnectPacket}, {close, frame_too_large}], _} =
+    ?assertMatch(
-        emqx_channel:handle_in({frame_error, frame_too_large}, ConnectedChan),
+        {ok, [{outgoing, DisconnectPacket}, {close, frame_too_large}], _},
+        emqx_channel:handle_in({frame_error, #{cause => frame_too_large}}, ConnectedChan)
+    ),
     DisconnectedChan = channel(#{conn_state => disconnected}),
     {ok, DisconnectedChan} =
-        emqx_channel:handle_in({frame_error, frame_too_large}, DisconnectedChan).
+        emqx_channel:handle_in({frame_error, #{cause => frame_too_large}}, DisconnectedChan).
 
 t_handle_in_expected_packet(_) ->
     Packet = ?DISCONNECT_PACKET(?RC_PROTOCOL_ERROR),

apps/emqx/test/emqx_client_SUITE.erl

@@ -72,7 +72,7 @@ groups() ->
             t_dollar_topics,
             t_sub_non_utf8_topic
         ]},
-        {mqttv5, [non_parallel_tests], [t_basic_with_props_v5]},
+        {mqttv5, [non_parallel_tests], [t_basic_with_props_v5, t_v5_receive_maximim_in_connack]},
         {others, [non_parallel_tests], [
             t_username_as_clientid,
             t_certcn_as_clientid_default_config_tls,
@@ -103,14 +103,14 @@ end_per_testcase(_Case, _Config) ->
 %%--------------------------------------------------------------------
 
 t_basic_v3(_) ->
-    t_basic([{proto_ver, v3}]).
+    run_basic([{proto_ver, v3}]).
 
 %%--------------------------------------------------------------------
 %% Test cases for MQTT v4
 %%--------------------------------------------------------------------
 
 t_basic_v4(_Config) ->
-    t_basic([{proto_ver, v4}]).
+    run_basic([{proto_ver, v4}]).
 
 t_cm(_) ->
     emqx_config:put_zone_conf(default, [mqtt, idle_timeout], 1000),
@@ -335,19 +335,30 @@ t_sub_non_utf8_topic(_) ->
 %% Test cases for MQTT v5
 %%--------------------------------------------------------------------
 
-t_basic_with_props_v5(_) ->
+v5_conn_props(ReceiveMaximum) ->
-    t_basic([
+    [
         {proto_ver, v5},
-        {properties, #{'Receive-Maximum' => 4}}
+        {properties, #{'Receive-Maximum' => ReceiveMaximum}}
-    ]).
+    ].
+
+t_basic_with_props_v5(_) ->
+    run_basic(v5_conn_props(4)).
+
+t_v5_receive_maximim_in_connack(_) ->
+    ReceiveMaximum = 7,
+    {ok, C} = emqtt:start_link(v5_conn_props(ReceiveMaximum)),
+    {ok, Props} = emqtt:connect(C),
+    ?assertMatch(#{'Receive-Maximum' := ReceiveMaximum}, Props),
+    ok = emqtt:disconnect(C),
+    ok.
 
 %%--------------------------------------------------------------------
 %% General test cases.
 %%--------------------------------------------------------------------
 
-t_basic(_Opts) ->
+run_basic(Opts) ->
     Topic = nth(1, ?TOPICS),
-    {ok, C} = emqtt:start_link([{proto_ver, v4}]),
+    {ok, C} = emqtt:start_link(Opts),
     {ok, _} = emqtt:connect(C),
     {ok, _, [1]} = emqtt:subscribe(C, Topic, qos1),
     {ok, _, [2]} = emqtt:subscribe(C, Topic, qos2),

apps/emqx/test/emqx_cm_SUITE.erl

@@ -221,7 +221,7 @@ t_open_session_race_condition(_) ->
     end,
     %% sync
     ignored = gen_server:call(?CM, ignore, infinity),
-    ok = emqx_pool:flush_async_tasks(),
+    ok = emqx_pool:flush_async_tasks(?CM_POOL),
     ?assertEqual([], emqx_cm:lookup_channels(ClientId)).
 
 t_kick_session_discard_normal(_) ->
@@ -343,29 +343,9 @@ test_stepdown_session(Action, Reason) ->
     end,
     % sync
     ignored = gen_server:call(?CM, ignore, infinity),
-    ok = flush_emqx_pool(),
+    ok = emqx_pool:flush_async_tasks(?CM_POOL),
     ?assertEqual([], emqx_cm:lookup_channels(ClientId)).
 
-%% Channel deregistration is delegated to emqx_pool as a sync tasks.
-%% The emqx_pool is pool of workers, and there is no way to know
-%% which worker was picked for the last deregistration task.
-%% This help function creates a large enough number of async tasks
-%% to sync with the pool workers.
-%% The number of tasks should be large enough to ensure all workers have
-%% the chance to work on at least one of the tasks.
-flush_emqx_pool() ->
-    Self = self(),
-    L = lists:seq(1, 1000),
-    lists:foreach(fun(I) -> emqx_pool:async_submit(fun() -> Self ! {done, I} end, []) end, L),
-    lists:foreach(
-        fun(I) ->
-            receive
-                {done, I} -> ok
-            end
-        end,
-        L
-    ).
-
 t_discard_session_race(_) ->
     ClientId = rand_client_id(),
     ?check_trace(

apps/emqx/test/emqx_cm_registry_keeper_SUITE.erl

@@ -0,0 +1,100 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_cm_registry_keeper_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include("emqx_cm.hrl").
+
+%%--------------------------------------------------------------------
+%% CT callbacks
+%%--------------------------------------------------------------------
+
+all() -> emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    AppConfig = "broker.session_history_retain = 2s",
+    Apps = emqx_cth_suite:start(
+        [{emqx, #{config => AppConfig}}],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{apps, Apps} | Config].
+
+end_per_suite(Config) ->
+    emqx_cth_suite:stop(proplists:get_value(apps, Config)).
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_TestCase, Config) ->
+    Config.
+
+t_cleanup_after_retain(_) ->
+    Pid = spawn(fun() ->
+        receive
+            stop -> ok
+        end
+    end),
+    ClientId = <<"clientid">>,
+    ClientId2 = <<"clientid2">>,
+    emqx_cm_registry:register_channel({ClientId, Pid}),
+    emqx_cm_registry:register_channel({ClientId2, Pid}),
+    ?assertEqual([Pid], emqx_cm_registry:lookup_channels(ClientId)),
+    ?assertEqual([Pid], emqx_cm_registry:lookup_channels(ClientId2)),
+    ?assertEqual(2, emqx_cm_registry_keeper:count(0)),
+    T0 = erlang:system_time(seconds),
+    exit(Pid, kill),
+    %% lookup_channel chesk if the channel is still alive
+    ?assertEqual([], emqx_cm_registry:lookup_channels(ClientId)),
+    ?assertEqual([], emqx_cm_registry:lookup_channels(ClientId2)),
+    %% simulate a DOWN message which causes emqx_cm to call clean_down
+    %% to clean the channels for real
+    ok = emqx_cm:clean_down({Pid, ClientId}),
+    ok = emqx_cm:clean_down({Pid, ClientId2}),
+    ?assertEqual(2, emqx_cm_registry_keeper:count(T0)),
+    ?assertEqual(2, emqx_cm_registry_keeper:count(0)),
+    ?retry(_Interval = 1000, _Attempts = 4, begin
+        ?assertEqual(0, emqx_cm_registry_keeper:count(T0)),
+        ?assertEqual(0, emqx_cm_registry_keeper:count(0))
+    end),
+    ok.
+
+%% count is cached when the number of entries is greater than 1000
+t_count_cache(_) ->
+    Pid = self(),
+    ClientsCount = 999,
+    ClientIds = lists:map(fun erlang:integer_to_binary/1, lists:seq(1, ClientsCount)),
+    Channels = lists:map(fun(ClientId) -> {ClientId, Pid} end, ClientIds),
+    lists:foreach(
+        fun emqx_cm_registry:register_channel/1,
+        Channels
+    ),
+    T0 = erlang:system_time(seconds),
+    ?assertEqual(ClientsCount, emqx_cm_registry_keeper:count(0)),
+    ?assertEqual(ClientsCount, emqx_cm_registry_keeper:count(T0)),
+    %% insert another one to trigger the cache threshold
+    emqx_cm_registry:register_channel({<<"-1">>, Pid}),
+    ?assertEqual(ClientsCount + 1, emqx_cm_registry_keeper:count(0)),
+    ?assertEqual(ClientsCount, emqx_cm_registry_keeper:count(T0)),
+    mnesia:clear_table(?CHAN_REG_TAB),
+    ok.
+
+channel(Id, Pid) ->
+    #channel{chid = Id, pid = Pid}.