yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into 0221-sync-release-55

This commit is contained in:
Zaiming (Stone) Shi 2024-02-21 09:11:12 +01:00
parent 80be84c401 8e44c80dd0
commit da340c92a1
562 changed files with 18206 additions and 8308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.ci/docker-compose-file/.env
View File

@ -10,7 +10,7 @@ CASSANDRA_TAG=3.11
MINIO_TAG=RELEASE.2023-03-20T20-16-18Z
OPENTS_TAG=9aa7f88
KINESIS_TAG=2.1
HSTREAMDB_TAG=v0.16.1
HSTREAMDB_TAG=v0.19.3
HSTREAMDB_ZK_TAG=3.8.1
MS_IMAGE_ADDR=mcr.microsoft.com/mssql/server

4
.ci/docker-compose-file/docker-compose-rabbitmq.yaml
View File

@ -9,10 +9,12 @@ services:
expose:
- "15672"
- "5672"
- "5671"
# We don't want to take ports from the host
# ports:
#ports:
# - "15672:15672"
# - "5672:5672"
# - "5671:5671"
volumes:
- ./certs/ca.crt:/opt/certs/ca.crt
- ./certs/server.crt:/opt/certs/server.crt

4
.ci/docker-compose-file/docker-compose-toxiproxy.yaml
View File

@ -39,6 +39,10 @@ services:
- 19042:9042
# Cassandra TLS
- 19142:9142
# Cassandra No Auth
- 19043:9043
# Cassandra TLS No Auth
- 19143:9143
# S3
- 19000:19000
# S3 TLS

12
.ci/docker-compose-file/toxiproxy.json
View File

@ -96,6 +96,18 @@
"upstream": "cassandra:9142",
"enabled": true
},
{
"name": "cassa_no_auth_tcp",
"listen": "0.0.0.0:9043",
"upstream": "cassandra_noauth:9042",
"enabled": true
},
{
"name": "cassa_no_auth_tls",
"listen": "0.0.0.0:9143",
"upstream": "cassandra_noauth:9142",
"enabled": true
},
{
"name": "sqlserver",
"listen": "0.0.0.0:1433",

2
.github/actions/package-macos/action.yaml vendored
View File

@ -51,7 +51,7 @@ runs:
echo "SELF_HOSTED=false" >> $GITHUB_OUTPUT
;;
esac
- uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
- uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
id: cache
if: steps.prepare.outputs.SELF_HOSTED != 'true'
with:

4
.github/actions/prepare-jmeter/action.yaml vendored
View File

@ -8,7 +8,7 @@ inputs:
runs:
using: composite
steps:
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: emqx-docker
path: /tmp
@ -31,7 +31,7 @@ runs:
architecture: x64 # (x64 or x86) - defaults to x64
# https://github.com/actions/setup-java/blob/main/docs/switching-to-v2.md
distribution: 'zulu'
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: apache-jmeter.tgz
- name: install jmeter

4
.github/workflows/_pr_entrypoint.yaml vendored
View File

@ -144,11 +144,11 @@ jobs:
echo "PROFILE=${PROFILE}" | tee -a .env
echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env
zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip .
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: ${{ matrix.profile }}
path: ${{ matrix.profile }}.zip
retention-days: 1
retention-days: 7
run_emqx_app_tests:
needs:

12
.github/workflows/_push-entrypoint.yaml vendored
View File

@ -28,7 +28,6 @@ jobs:
profile: ${{ steps.parse-git-ref.outputs.profile }}
release: ${{ steps.parse-git-ref.outputs.release }}
latest: ${{ steps.parse-git-ref.outputs.latest }}
version: ${{ steps.parse-git-ref.outputs.version }}
ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
ct-host: ${{ steps.matrix.outputs.ct-host }}
ct-docker: ${{ steps.matrix.outputs.ct-docker }}
@ -46,18 +45,16 @@ jobs:
shell: bash
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Detect emqx profile and version
- name: Detect emqx profile
id: parse-git-ref
run: |
JSON="$(./scripts/parse-git-ref.sh $GITHUB_REF)"
PROFILE=$(echo "$JSON" | jq -cr '.profile')
RELEASE=$(echo "$JSON" | jq -cr '.release')
LATEST=$(echo "$JSON" | jq -cr '.latest')
VERSION="$(./pkg-vsn.sh "$PROFILE")"
echo "profile=$PROFILE" | tee -a $GITHUB_OUTPUT
echo "release=$RELEASE" | tee -a $GITHUB_OUTPUT
echo "latest=$LATEST" | tee -a $GITHUB_OUTPUT
echo "version=$VERSION" | tee -a $GITHUB_OUTPUT
- name: Build matrix
id: matrix
run: |
@ -91,7 +88,7 @@ jobs:
uses: ./.github/workflows/build_packages.yaml
with:
profile: ${{ needs.prepare.outputs.profile }}
publish: ${{ needs.prepare.outputs.release }}
publish: true
otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
@ -104,8 +101,7 @@ jobs:
uses: ./.github/workflows/build_and_push_docker_images.yaml
with:
profile: ${{ needs.prepare.outputs.profile }}
version: ${{ needs.prepare.outputs.version }}
publish: ${{ needs.prepare.outputs.release }}
publish: true
latest: ${{ needs.prepare.outputs.latest }}
# TODO: revert this back to needs.prepare.outputs.otp_vsn when OTP 26 bug is fixed
otp_vsn: 25.3.2-2
@ -153,7 +149,7 @@ jobs:
echo "PROFILE=${PROFILE}" | tee -a .env
echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env
zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip .
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: ${{ matrix.profile }}
path: ${{ matrix.profile }}.zip

180
.github/workflows/build_and_push_docker_images.yaml vendored
View File

@ -10,15 +10,12 @@ on:
profile:
required: true
type: string
version:
required: true
type: string
latest:
required: true
type: string
publish:
required: true
type: string
type: boolean
otp_vsn:
required: true
type: string
@ -45,8 +42,6 @@ on:
required: false
type: string
default: 'emqx'
version:
required: true
latest:
required: false
type: boolean
@ -72,8 +67,11 @@ permissions:
contents: read
jobs:
docker:
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
build:
runs-on: ${{ github.repository_owner == 'emqx' && fromJSON(format('["self-hosted","ephemeral","linux","{0}"]', matrix.arch)) || 'ubuntu-22.04' }}
container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ inputs.otp_vsn }}-debian11"
outputs:
PKG_VSN: ${{ steps.build.outputs.PKG_VSN }}
strategy:
fail-fast: false
@ -81,54 +79,130 @@ jobs:
profile:
- ${{ inputs.profile }}
- ${{ inputs.profile }}-elixir
registry:
- 'docker.io'
- 'public.ecr.aws'
exclude:
- profile: emqx-enterprise
registry: 'public.ecr.aws'
- profile: emqx-enterprise-elixir
registry: 'public.ecr.aws'
arch:
- x64
- arm64
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ github.event.inputs.ref }}
fetch-depth: 0
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ github.event.inputs.ref }}
- run: git config --global --add safe.directory "$PWD"
- name: build release tarball
id: build
run: |
make ${{ matrix.profile }}-tgz
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: "${{ matrix.profile }}-${{ matrix.arch }}.tar.gz"
path: "_packages/emqx*/emqx-*.tar.gz"
retention-days: 7
overwrite: true
if-no-files-found: error
- uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
- uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
docker:
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
needs:
- build
defaults:
run:
shell: bash
- name: Login to hub.docker.com
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
if: matrix.registry == 'docker.io'
with:
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
strategy:
fail-fast: false
matrix:
profile:
- ${{ inputs.profile }}
- ${{ inputs.profile }}-elixir
- name: Login to AWS ECR
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
if: matrix.registry == 'public.ecr.aws'
with:
registry: public.ecr.aws
username: ${{ secrets.AWS_ACCESS_KEY_ID }}
password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
ecr: true
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ github.event.inputs.ref }}
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
pattern: "${{ matrix.profile }}-*.tar.gz"
path: _packages
merge-multiple: true
- name: Build docker image
env:
PROFILE: ${{ matrix.profile }}
DOCKER_REGISTRY: ${{ matrix.registry }}
DOCKER_ORG: ${{ github.repository_owner }}
DOCKER_LATEST: ${{ inputs.latest }}
DOCKER_PUSH: ${{ inputs.publish == 'true' || inputs.publish || github.repository_owner != 'emqx' }}
DOCKER_BUILD_NOCACHE: true
DOCKER_PLATFORMS: linux/amd64,linux/arm64
EMQX_RUNNER: 'debian:11-slim'
EMQX_DOCKERFILE: 'deploy/docker/Dockerfile'
PKG_VSN: ${{ inputs.version }}
EMQX_BUILDER_VERSION: ${{ inputs.builder_vsn }}
EMQX_BUILDER_OTP: ${{ inputs.otp_vsn }}
EMQX_BUILDER_ELIXIR: ${{ inputs.elixir_vsn }}
run: |
./build ${PROFILE} docker
- name: Move artifacts to root directory
env:
PROFILE: ${{ inputs.profile }}
run: |
ls -lR _packages/$PROFILE
mv _packages/$PROFILE/*.tar.gz ./
- name: Enable containerd image store on Docker Engine
run: |
echo "$(jq '. += {"features": {"containerd-snapshotter": true}}' /etc/docker/daemon.json)" > daemon.json
sudo mv daemon.json /etc/docker/daemon.json
sudo systemctl restart docker
- uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
- uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
- name: Login to hub.docker.com
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
if: inputs.publish || github.repository_owner != 'emqx'
with:
username: ${{ secrets.DOCKER_HUB_USER }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}
- name: Login to AWS ECR
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
if: inputs.publish || github.repository_owner != 'emqx'
with:
registry: public.ecr.aws
username: ${{ secrets.AWS_ACCESS_KEY_ID }}
password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
ecr: true
- name: Build docker image
env:
PROFILE: ${{ matrix.profile }}
DOCKER_REGISTRY: 'docker.io,public.ecr.aws'
DOCKER_ORG: ${{ github.repository_owner }}
DOCKER_LATEST: ${{ inputs.latest }}
DOCKER_PUSH: false
DOCKER_BUILD_NOCACHE: true
DOCKER_PLATFORMS: linux/amd64,linux/arm64
DOCKER_LOAD: true
EMQX_RUNNER: 'public.ecr.aws/debian/debian:11-slim@sha256:22cfb3c06a7dd5e18d86123a73405664475b9d9fa209cbedcf4c50a25649cc74'
EMQX_DOCKERFILE: 'deploy/docker/Dockerfile'
PKG_VSN: ${{ needs.build.outputs.PKG_VSN }}
EMQX_BUILDER_VERSION: ${{ inputs.builder_vsn }}
EMQX_BUILDER_OTP: ${{ inputs.otp_vsn }}
EMQX_BUILDER_ELIXIR: ${{ inputs.elixir_vsn }}
EMQX_SOURCE_TYPE: tgz
run: |
./build ${PROFILE} docker
cat .emqx_docker_image_tags
echo "_EMQX_DOCKER_IMAGE_TAG=$(head -n 1 .emqx_docker_image_tags)" >> $GITHUB_ENV
- name: smoke test
timeout-minutes: 1
run: |
for tag in $(cat .emqx_docker_image_tags); do
CID=$(docker run -d -P $tag)
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
docker rm -f $CID
done
- name: dashboard tests
working-directory: ./scripts/ui-tests
timeout-minutes: 5
run: |
set -eu
docker compose up --abort-on-container-exit --exit-code-from selenium
docker compose rm -fsv
- name: test node_dump
run: |
CID=$(docker run -d -P $_EMQX_DOCKER_IMAGE_TAG)
docker exec -t -u root -w /root $CID bash -c 'apt-get -y update && apt-get -y install net-tools'
docker exec -t -u root $CID node_dump
docker rm -f $CID
- name: push images
if: inputs.publish || github.repository_owner != 'emqx'
run: |
for tag in $(cat .emqx_docker_image_tags); do
docker push $tag
done

8
.github/workflows/build_docker_for_test.yaml vendored
View File

@ -47,17 +47,17 @@ jobs:
id: build
run: |
make ${EMQX_NAME}-docker
echo "EMQX_IMAGE_TAG=$(cat .docker_image_tag)" >> $GITHUB_ENV
echo "_EMQX_DOCKER_IMAGE_TAG=$(head -n 1 .emqx_docker_image_tags)" >> $GITHUB_ENV
- name: smoke test
run: |
CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
CID=$(docker run -d --rm -P $_EMQX_DOCKER_IMAGE_TAG)
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
docker stop $CID
- name: export docker image
run: |
docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
docker save $_EMQX_DOCKER_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: "${{ env.EMQX_NAME }}-docker"
path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz"

94
.github/workflows/build_packages.yaml vendored
View File

@ -12,7 +12,7 @@ on:
type: string
publish:
required: true
type: string
type: boolean
otp_vsn:
required: true
type: string
@ -74,12 +74,12 @@ jobs:
matrix:
profile:
- ${{ inputs.profile }}
otp:
- ${{ inputs.otp_vsn }}
os:
- macos-12
- macos-12-arm64
- macos-13
otp:
- ${{ inputs.otp_vsn }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@ -95,30 +95,20 @@ jobs:
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: success()
with:
name: ${{ matrix.profile }}
name: ${{ matrix.profile }}-${{ matrix.os }}-${{ matrix.otp }}
path: _packages/${{ matrix.profile }}/
retention-days: 7
linux:
runs-on: [self-hosted, ephemeral, linux, "${{ matrix.arch }}"]
# always run in builder container because the host might have the wrong OTP version etc.
# otherwise buildx.sh does not run docker if arch and os matches the target arch and os.
container:
image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
runs-on: [self-hosted, ephemeral, linux, "${{ matrix.arch == 'arm64' && 'arm64' || 'x64' }}"]
strategy:
fail-fast: false
matrix:
profile:
- ${{ inputs.profile }}
otp:
- ${{ inputs.otp_vsn }}
arch:
- x64
- arm64
os:
- ubuntu22.04
- ubuntu20.04
@ -131,70 +121,53 @@ jobs:
- el7
- amzn2
- amzn2023
arch:
- amd64
- arm64
with_elixir:
- 'no'
otp:
- ${{ inputs.otp_vsn }}
builder:
- ${{ inputs.builder_vsn }}
elixir:
- ${{ inputs.elixir_vsn }}
with_elixir:
- 'no'
include:
- profile: ${{ inputs.profile }}
otp: ${{ inputs.otp_vsn }}
arch: x64
os: ubuntu22.04
arch: amd64
with_elixir: 'yes'
otp: ${{ inputs.otp_vsn }}
builder: ${{ inputs.builder_vsn }}
elixir: ${{ inputs.elixir_vsn }}
with_elixir: 'yes'
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ github.event.inputs.ref }}
fetch-depth: 0
- name: fix workdir
run: |
set -eu
git config --global --add safe.directory "$GITHUB_WORKSPACE"
# Align path for CMake caches
if [ ! "$PWD" = "/emqx" ]; then
ln -s $PWD /emqx
cd /emqx
fi
echo "pwd is $PWD"
- name: build emqx packages
env:
PROFILE: ${{ matrix.profile }}
ARCH: ${{ matrix.arch }}
OS: ${{ matrix.os }}
IS_ELIXIR: ${{ matrix.with_elixir }}
ACLOCAL_PATH: "/usr/share/aclocal:/usr/local/share/aclocal"
BUILDER: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
BUILDER_SYSTEM: force_docker
run: |
set -eu
if [ "${IS_ELIXIR:-}" == 'yes' ]; then
make "${PROFILE}-elixir-tgz"
else
make "${PROFILE}-tgz"
make "${PROFILE}-pkg"
fi
- name: test emqx packages
env:
PROFILE: ${{ matrix.profile }}
IS_ELIXIR: ${{ matrix.with_elixir }}
run: |
set -eu
if [ "${IS_ELIXIR:-}" == 'yes' ]; then
./scripts/pkg-tests.sh "${PROFILE}-elixir-tgz"
else
./scripts/pkg-tests.sh "${PROFILE}-tgz"
./scripts/pkg-tests.sh "${PROFILE}-pkg"
fi
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
./scripts/buildx.sh \
--profile $PROFILE \
--arch $ARCH \
--builder $BUILDER \
--elixir $IS_ELIXIR \
--pkgtype pkg
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: ${{ matrix.profile }}
name: ${{ matrix.profile }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.with_elixir == 'yes' && '-elixir' || '' }}-${{ matrix.builder }}-${{ matrix.otp }}-${{ matrix.elixir }}
path: _packages/${{ matrix.profile }}/
retention-days: 7
@ -203,17 +176,18 @@ jobs:
needs:
- mac
- linux
if: inputs.publish == 'true' || inputs.publish
if: inputs.publish
strategy:
fail-fast: false
matrix:
profile:
- ${{ inputs.profile }}
steps:
- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: ${{ matrix.profile }}
pattern: "${{ matrix.profile }}-*"
path: packages/${{ matrix.profile }}
merge-multiple: true
- name: install dos2unix
run: sudo apt-get update -y && sudo apt install -y dos2unix
- name: get packages
@ -226,7 +200,7 @@ jobs:
echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
done
cd -
- uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
- uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

8
.github/workflows/build_packages_cron.yaml vendored
View File

@ -66,14 +66,14 @@ jobs:
set -eu
./scripts/pkg-tests.sh "${PROFILE}-tgz"
./scripts/pkg-tests.sh "${PROFILE}-pkg"
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: success()
with:
name: ${{ matrix.profile[0] }}-${{ matrix.os }}
path: _packages/${{ matrix.profile[0] }}/
retention-days: 7
- name: Send notification to Slack
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
if: failure()
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
@ -111,14 +111,14 @@ jobs:
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: success()
with:
name: ${{ matrix.profile }}-${{ matrix.os }}
path: _packages/${{ matrix.profile }}/
retention-days: 7
- name: Send notification to Slack
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
if: failure()
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

6
.github/workflows/build_slim_packages.yaml vendored
View File

@ -88,13 +88,13 @@ jobs:
run: |
make ${EMQX_NAME}-elixir-pkg
./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}-${{ matrix.profile[3] }}-${{ matrix.profile[4] }}"
path: _packages/${{ matrix.profile[0] }}/*
retention-days: 7
compression-level: 0
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: "${{ matrix.profile[0] }}-schema-dump-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}-${{ matrix.profile[3] }}-${{ matrix.profile[4] }}"
path: |
@ -128,7 +128,7 @@ jobs:
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: ${{ matrix.os }}
path: _packages/**/*

2
.github/workflows/check_deps_integrity.yaml vendored
View File

@ -36,7 +36,7 @@ jobs:
MIX_ENV: emqx-enterprise
PROFILE: emqx-enterprise
- name: Upload produced lock files
uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: produced_lock_files

42
.github/workflows/performance_test.yaml vendored
View File

@ -52,7 +52,7 @@ jobs:
id: package_file
run: |
echo "PACKAGE_FILE=$(find _packages/emqx -name 'emqx-*.deb' | head -n 1 | xargs basename)" >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: emqx-ubuntu20.04
path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }}
@ -66,7 +66,7 @@ jobs:
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
@ -77,7 +77,7 @@ jobs:
repository: emqx/tf-emqx-performance-test
path: tf-emqx-performance-test
ref: v0.2.3
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: emqx-ubuntu20.04
path: tf-emqx-performance-test/
@ -105,7 +105,7 @@ jobs:
terraform destroy -auto-approve
aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
- name: Send notification to Slack
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
with:
payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
- name: terraform destroy
@ -113,13 +113,13 @@ jobs:
working-directory: ./tf-emqx-performance-test
run: |
terraform destroy -auto-approve
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: success()
with:
name: metrics
path: |
"./tf-emqx-performance-test/*.tar.gz"
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: terraform
@ -137,7 +137,7 @@ jobs:
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
@ -148,7 +148,7 @@ jobs:
repository: emqx/tf-emqx-performance-test
path: tf-emqx-performance-test
ref: v0.2.3
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: emqx-ubuntu20.04
path: tf-emqx-performance-test/
@ -176,7 +176,7 @@ jobs:
terraform destroy -auto-approve
aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
- name: Send notification to Slack
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
with:
payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
- name: terraform destroy
@ -184,13 +184,13 @@ jobs:
working-directory: ./tf-emqx-performance-test
run: |
terraform destroy -auto-approve
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: success()
with:
name: metrics
path: |
"./tf-emqx-performance-test/*.tar.gz"
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: terraform
@ -209,7 +209,7 @@ jobs:
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
@ -220,7 +220,7 @@ jobs:
repository: emqx/tf-emqx-performance-test
path: tf-emqx-performance-test
ref: v0.2.3
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: emqx-ubuntu20.04
path: tf-emqx-performance-test/
@ -249,7 +249,7 @@ jobs:
terraform destroy -auto-approve
aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
- name: Send notification to Slack
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
with:
payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
- name: terraform destroy
@ -257,13 +257,13 @@ jobs:
working-directory: ./tf-emqx-performance-test
run: |
terraform destroy -auto-approve
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: success()
with:
name: metrics
path: |
"./tf-emqx-performance-test/*.tar.gz"
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: terraform
@ -283,7 +283,7 @@ jobs:
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
@ -294,7 +294,7 @@ jobs:
repository: emqx/tf-emqx-performance-test
path: tf-emqx-performance-test
ref: v0.2.3
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: emqx-ubuntu20.04
path: tf-emqx-performance-test/
@ -322,7 +322,7 @@ jobs:
terraform destroy -auto-approve
aws s3 sync --exclude '*' --include '*.tar.gz' s3://$TF_VAR_s3_bucket_name/$TF_VAR_bench_id .
- name: Send notification to Slack
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
with:
payload-file-path: "./tf-emqx-performance-test/slack-payload.json"
- name: terraform destroy
@ -330,13 +330,13 @@ jobs:
working-directory: ./tf-emqx-performance-test
run: |
terraform destroy -auto-approve
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: success()
with:
name: metrics
path: |
"./tf-emqx-performance-test/*.tar.gz"
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: terraform

2
.github/workflows/release.yaml vendored
View File

@ -31,7 +31,7 @@ jobs:
strategy:
fail-fast: false
steps:
- uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
- uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

4
.github/workflows/run_conf_tests.yaml vendored
View File

@ -25,7 +25,7 @@ jobs:
- emqx
- emqx-enterprise
steps:
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: ${{ matrix.profile }}
- name: extract artifact
@ -40,7 +40,7 @@ jobs:
if: failure()
run: |
cat _build/${{ matrix.profile }}/rel/emqx/logs/erlang.log.*
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: conftest-logs-${{ matrix.profile }}

12
.github/workflows/run_docker_tests.yaml vendored
View File

@ -37,14 +37,14 @@ jobs:
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: ${{ env.EMQX_NAME }}-docker
path: /tmp
- name: load docker image
run: |
EMQX_IMAGE_TAG=$(docker load < /tmp/${EMQX_NAME}-docker-${PKG_VSN}.tar.gz 2>/dev/null | sed 's/Loaded image: //g')
echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV
_EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/${EMQX_NAME}-docker-${PKG_VSN}.tar.gz 2>/dev/null | sed 's/Loaded image: //g')
echo "_EMQX_DOCKER_IMAGE_TAG=$_EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
- name: dashboard tests
working-directory: ./scripts/ui-tests
run: |
@ -52,7 +52,7 @@ jobs:
docker compose up --abort-on-container-exit --exit-code-from selenium
- name: test two nodes cluster with proto_dist=inet_tls in docker
run: |
./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
./scripts/test/start-two-nodes-in-docker.sh -P $_EMQX_DOCKER_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy)
./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
./scripts/test/start-two-nodes-in-docker.sh -c
@ -84,7 +84,7 @@ jobs:
- rlog
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: ${{ env.EMQX_NAME }}-docker
path: /tmp
@ -113,4 +113,4 @@ jobs:
- name: test node_dump
run: |
docker exec -t -u root node1.emqx.io bash -c 'apt-get -y update && apt-get -y install net-tools'
docker exec node1.emqx.io node_dump
docker exec -t -u root node1.emqx.io node_dump

2
.github/workflows/run_emqx_app_tests.yaml vendored
View File

@ -58,7 +58,7 @@ jobs:
./rebar3 eunit -v --name 'eunit@127.0.0.1'
./rebar3 as standalone_test ct --name 'test@127.0.0.1' -v --readable=true
./rebar3 proper -d test/props
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: logs-emqx-app-tests

2
.github/workflows/run_helm_tests.yaml vendored
View File

@ -45,7 +45,7 @@ jobs:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
path: source
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: "${{ env.EMQX_NAME }}-docker"
path: /tmp

14
.github/workflows/run_jmeter_tests.yaml vendored
View File

@ -16,7 +16,7 @@ jobs:
steps:
- name: Cache Jmeter
id: cache-jmeter
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
with:
path: /tmp/apache-jmeter.tgz
key: apache-jmeter-5.4.3.tgz
@ -35,7 +35,7 @@ jobs:
else
wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz $ARCHIVE_URL
fi
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: apache-jmeter.tgz
path: /tmp/apache-jmeter.tgz
@ -86,7 +86,7 @@ jobs:
echo "check logs failed"
exit 1
fi
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: always()
with:
name: jmeter_logs-advanced_feat-${{ matrix.scripts_type }}
@ -153,7 +153,7 @@ jobs:
if: failure()
run: |
docker compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: always()
with:
name: jmeter_logs-pgsql_authn_authz-${{ matrix.scripts_type }}_${{ matrix.pgsql_tag }}
@ -213,7 +213,7 @@ jobs:
echo "check logs failed"
exit 1
fi
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: always()
with:
name: jmeter_logs-mysql_authn_authz-${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}
@ -265,7 +265,7 @@ jobs:
echo "check logs failed"
exit 1
fi
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: always()
with:
name: jmeter_logs-JWT_authn-${{ matrix.scripts_type }}
@ -309,7 +309,7 @@ jobs:
echo "check logs failed"
exit 1
fi
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: always()
with:
name: jmeter_logs-built_in_database_authn_authz-${{ matrix.scripts_type }}

10
.github/workflows/run_relup_tests.yaml vendored
View File

@ -25,7 +25,7 @@ jobs:
run:
shell: bash
steps:
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: emqx-enterprise
- name: extract artifact
@ -45,7 +45,7 @@ jobs:
run: |
export PROFILE='emqx-enterprise'
make emqx-enterprise-tgz
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
name: Upload built emqx and test scenario
with:
name: relup_tests_emqx_built
@ -72,7 +72,7 @@ jobs:
run:
shell: bash
steps:
- uses: erlef/setup-beam@a34c98fd51e370b4d4981854aba1eb817ce4e483 # v1.17.0
- uses: erlef/setup-beam@8b9cac4c04dbcd7bf8fd673e16f988225d89b09b # v1.17.2
with:
otp-version: 26.2.1
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@ -88,7 +88,7 @@ jobs:
./configure
make
echo "$(pwd)/bin" >> $GITHUB_PATH
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
name: Download built emqx and test scenario
with:
name: relup_tests_emqx_built
@ -111,7 +111,7 @@ jobs:
docker logs node2.emqx.io | tee lux_logs/emqx2.log
exit 1
fi
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
name: Save debug data
if: failure()
with:

20
.github/workflows/run_test_cases.yaml vendored
View File

@ -41,7 +41,7 @@ jobs:
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps:
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: ${{ matrix.profile }}
- name: extract artifact
@ -64,7 +64,7 @@ jobs:
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
run: make proper
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: coverdata-${{ matrix.profile }}-${{ matrix.otp }}
path: _build/test/cover
@ -83,7 +83,7 @@ jobs:
shell: bash
steps:
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: ${{ matrix.profile }}
- name: extract artifact
@ -108,7 +108,7 @@ jobs:
ENABLE_COVER_COMPILE: 1
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }}
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: coverdata-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
path: _build/test/cover
@ -116,7 +116,7 @@ jobs:
- name: compress logs
if: failure()
run: tar -czf logs.tar.gz _build/test/logs
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
@ -138,7 +138,7 @@ jobs:
shell: bash
steps:
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: ${{ matrix.profile }}
- name: extract artifact
@ -155,7 +155,7 @@ jobs:
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
run: |
make "${{ matrix.app }}-ct"
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: coverdata-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
path: _build/test/cover
@ -164,7 +164,7 @@ jobs:
- name: compress logs
if: failure()
run: tar -czf logs.tar.gz _build/test/logs
- uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
@ -196,7 +196,7 @@ jobs:
profile:
- emqx-enterprise
steps:
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: ${{ matrix.profile }}
- name: extract artifact
@ -204,7 +204,7 @@ jobs:
unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
name: download coverdata
with:
pattern: coverdata-${{ matrix.profile }}-*

4
.github/workflows/scorecard.yaml vendored
View File

@ -39,7 +39,7 @@ jobs:
publish_results: true
- name: "Upload artifact"
uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: SARIF file
path: results.sarif
@ -47,6 +47,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard.
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@8e0b1c74b1d5a0077b04d064c76ee714d3da7637 # v2.22.1
uses: github/codeql-action/upload-sarif@7e187e1c529d80bac7b87a16e7a792427f65cf02 # v2.22.1
with:
sarif_file: results.sarif

2
.github/workflows/spellcheck.yaml vendored
View File

@ -19,7 +19,7 @@ jobs:
- emqx-enterprise
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
steps:
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
pattern: "${{ matrix.profile }}-schema-dump-*-x64"
merge-multiple: true

4
.github/workflows/static_checks.yaml vendored
View File

@ -30,14 +30,14 @@ jobs:
include: ${{ fromJson(inputs.ct-matrix) }}
container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
steps:
- uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
- uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
name: ${{ matrix.profile }}
- name: extract artifact
run: |
unzip -o -q ${{ matrix.profile }}.zip
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
- uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
with:
path: "emqx_dialyzer_${{ matrix.otp }}_plt"
key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*') }}

2
.github/workflows/upload-helm-charts.yaml vendored
View File

@ -18,7 +18,7 @@ jobs:
strategy:
fail-fast: false
steps:
- uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
- uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

1
.gitignore vendored
View File

@ -73,3 +73,4 @@ apps/emqx_conf/etc/emqx.conf.all.rendered*
rebar-git-cache.tar
# build docker image locally
.docker_image_tag
.git/

9
Makefile
View File

@ -21,7 +21,7 @@ endif
# Dashboard version
# from https://github.com/emqx/emqx-dashboard5
export EMQX_DASHBOARD_VERSION ?= v1.7.0
export EMQX_EE_DASHBOARD_VERSION ?= e1.5.0
export EMQX_EE_DASHBOARD_VERSION ?= e1.5.1-s3-beta.1
PROFILE ?= emqx
REL_PROFILES := emqx emqx-enterprise
@ -316,10 +316,9 @@ $(foreach tt,$(ALL_ELIXIR_TGZS),$(eval $(call gen-elixir-tgz-target,$(tt))))
.PHONY: fmt
fmt: $(REBAR)
@$(SCRIPTS)/erlfmt -w 'apps/*/{src,include,priv,test,integration_test}/**/*.{erl,hrl,app.src,eterm}'
@$(SCRIPTS)/erlfmt -w '**/*.escript' --exclude-files '_build/**'
@$(SCRIPTS)/erlfmt -w '**/rebar.config' --exclude-files '_build/**'
@$(SCRIPTS)/erlfmt -w 'rebar.config.erl'
@$(SCRIPTS)/erlfmt -w 'bin/nodetool'
@$(SCRIPTS)/erlfmt -w 'apps/*/rebar.config' 'apps/emqx/rebar.config.script' '.ci/fvt_tests/http_server/rebar.config'
@$(SCRIPTS)/erlfmt -w 'rebar.config' 'rebar.config.erl'
@$(SCRIPTS)/erlfmt -w 'scripts/*.escript' 'bin/*.escript' 'bin/nodetool'
@mix format
.PHONY: clean-test-cluster-config

1
README-CN.md
View File

@ -4,6 +4,7 @@
[![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml)
[![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
[![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/emqx/emqx/badge)](https://securityscorecards.dev/viewer/?uri=github.com/emqx/emqx)
[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES)
[![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)

1
README-RU.md
View File

@ -4,6 +4,7 @@
[![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml)
[![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
[![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/emqx/emqx/badge)](https://securityscorecards.dev/viewer/?uri=github.com/emqx/emqx)
[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES)
[![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)

1
README.md
View File

@ -4,6 +4,7 @@
[![Build Status](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml/badge.svg)](https://github.com/emqx/emqx/actions/workflows/_push-entrypoint.yaml)
[![Coverage Status](https://img.shields.io/coveralls/github/emqx/emqx/master?label=Coverage)](https://coveralls.io/github/emqx/emqx?branch=master)
[![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx?label=Docker%20Pulls)](https://hub.docker.com/r/emqx/emqx)
[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/emqx/emqx/badge)](https://securityscorecards.dev/viewer/?uri=github.com/emqx/emqx)
[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
[![Discord](https://img.shields.io/discord/931086341838622751?label=Discord&logo=discord)](https://discord.gg/xYGf3fQnES)
[![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)

131
Windows.md
View File

@ -1,131 +0,0 @@
# Build and run EMQX on Windows
NOTE: The instructions and examples are based on Windows 10.
## Build Environment
### Visual studio for C/C++ compile and link
EMQX includes Erlang NIF (Native Implemented Function) components, implemented
in C/C++. To compile and link C/C++ libraries, the easiest way is perhaps to
install Visual Studio.
Visual Studio 2019 is used in our tests.
If you are like me (@zmstone), do not know where to start,
please follow this OTP guide:
https://github.com/erlang/otp/blob/master/HOWTO/INSTALL-WIN32.md
NOTE: To avoid surprises, you may need to add below two paths to `Path` environment variable
and order them before other paths.
```
C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\bin\Hostx64\x64
C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build
```
Depending on your visual studio version and OS, the paths may differ.
The first path is for rebar3 port compiler to find `cl.exe` and `link.exe`
The second path is for CMD to setup environment variables.
### Erlang/OTP
Install Erlang/OTP 24 from https://www.erlang.org/downloads
You may need to edit the `Path` environment variable to allow running
Erlang commands such as `erl` from powershell.
To validate Erlang installation in CMD or powershell:
* Start (or restart) CMD or powershell
* Execute `erl` command to enter Erlang shell
* Evaluate Erlang expression `halt().` to exit Erlang shell.
e.g.
```
PS C:\Users\zmsto> erl
Eshell V12.2.1 (abort with ^G)
1> halt().
```
### bash
All EMQX build/run scripts are either in `bash` or `escript`.
`escript` is installed as a part of Erlang. To install a `bash`
environment in Windows, there are quite a few options.
Cygwin is what we tested with.
* Add `cygwin\bin` dir to `Path` environment variable
To do so, search for Edit environment variable in control panel and
add `C:\tools\cygwin\bin` (depending on the location where it was installed)
to `Path` list.
* Validate installation.
Start (restart) CMD or powershell console and execute `which bash`, it should
print out `/usr/bin/bash`
NOTE: Make sure cygwin's bin dir is added before `C:\Windows\system32` in `Path`,
otherwise the build scripts may end up using binaries from wsl instead of cygwin.
### Other tools
Some of the unix world tools are required to build EMQX. Including:
* git
* curl
* make
* cmake
* jq
* zip / unzip
We recommend using [scoop](https://scoop.sh/), or [Chocolatey](https://chocolatey.org/install) to install the tools.
When using scoop:
```
scoop install git curl make cmake jq zip unzip
```
## Build EMQX source code
* Clone the repo: `git clone https://github.com/emqx/emqx.git`
* Start CMD console
* Execute `vcvarsall.bat x86_amd64` to load environment variables
* Change to emqx directory and execute `make`
### Possible errors
* `'cl.exe' is not recognized as an internal or external command`
This error is likely due to Visual Studio executables are not set in `Path` environment variable.
To fix it, either add path like `C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29910\bin\Hostx64\x64`
to `Paht`. Or make sure `vcvarsall.bat x86_amd64` is executed prior to the `make` command
* `fatal error C1083: Cannot open include file: 'assert.h': No such file or directory`
If Visual Studio is installed correctly, this is likely `LIB` and `LIB_PATH` environment
variables are not set. Make sure `vcvarsall.bat x86_amd64` is executed prior to the `make` command
* `link: extra operand 'some.obj'`
This is likely due to the usage of GNU `lnik.exe` but not the one from Visual Studio.
Execute `link.exe --version` to inspect which one is in use. The one installed from
Visual Studio should print out `Microsoft (R) Incremental Linker`.
To fix it, Visual Studio's bin paths should be ordered prior to Cygwin's (or similar installation's)
bin paths in `Path` environment variable.
## Run EMQX
To start EMQX broker.
Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQX.
Then execute `_build\emqx\rel\emqx>.\bin\emqx_ctl status` to check status.
If everything works fine, it should print out
```
Node 'emqx@127.0.0.1' 4.3-beta.1 is started
Application emqx 4.3.0 is running
```

5
apps/emqx/include/emqx.hrl
View File

@ -88,10 +88,7 @@
%%--------------------------------------------------------------------
-record(banned, {
who ::
{clientid, binary()}
| {peerhost, inet:ip_address()}
| {username, binary()},
who :: emqx_types:banned_who(),
by :: binary(),
reason :: binary(),
at :: integer(),

11
apps/emqx/include/emqx_cm.hrl
View File

@ -23,11 +23,20 @@
-define(CHAN_INFO_TAB, emqx_channel_info).
-define(CHAN_LIVE_TAB, emqx_channel_live).
%% Mria/Mnesia Tables for channel management.
%% Mria table for session registration.
-define(CHAN_REG_TAB, emqx_channel_registry).
-define(T_KICK, 5_000).
-define(T_GET_INFO, 5_000).
-define(T_TAKEOVER, 15_000).
-define(CM_POOL, emqx_cm_pool).
%% Registered sessions.
-record(channel, {
chid :: emqx_types:clientid() | '_',
%% pid field is extended in 5.6.0 to support recording unregistration timestamp.
pid :: pid() | non_neg_integer() | '$1'
}).
-endif.

1
apps/emqx/include/http_api.hrl
View File

@ -17,6 +17,7 @@
%% HTTP API Auth
-define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD').
-define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET').
-define(API_KEY_NOT_ALLOW, 'API_KEY_NOT_ALLOW').
-define(API_KEY_NOT_ALLOW_MSG, <<"This API Key don't have permission to access this resource">>).
%% Bad Request

15
apps/emqx/include/logger.hrl
View File

@ -40,6 +40,21 @@
end
).
%% NOTE: do not forget to use atom for msg and add every used msg to
%% the default value of `log.thorttling.msgs` list.
-define(SLOG_THROTTLE(Level, Data),
?SLOG_THROTTLE(Level, Data, #{})
).
-define(SLOG_THROTTLE(Level, Data, Meta),
case emqx_log_throttler:allow(Level, maps:get(msg, Data)) of
true ->
?SLOG(Level, Data, Meta);
false ->
ok
end
).
-define(AUDIT_HANDLER, emqx_audit).
-define(TRACE_FILTER, emqx_trace_filter).
-define(OWN_KEYS, [level, filters, filter_default, handlers]).

2
apps/emqx/include/types.hrl
View File

@ -14,7 +14,7 @@
%% limitations under the License.
%%--------------------------------------------------------------------
-type maybe(T) :: undefined | T.
-type option(T) :: undefined | T.
-type startlink_ret() :: {ok, pid()} | ignore | {error, term()}.

199
apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%--------------------------------------------------------------------
-module(emqx_persistent_session_ds_SUITE).
@ -18,6 +18,9 @@
%% CT boilerplate
%%------------------------------------------------------------------------------
suite() ->
[{timetrap, {seconds, 60}}].
all() ->
emqx_common_test_helpers:all(?MODULE).
@ -51,12 +54,12 @@ init_per_testcase(TestCase, Config) when
init_per_testcase(t_session_gc = TestCase, Config) ->
Opts = #{
n => 3,
roles => [core, core, replicant],
roles => [core, core, core],
extra_emqx_conf =>
"\n session_persistence {"
"\n last_alive_update_interval = 500ms "
"\n session_gc_interval = 2s "
"\n session_gc_batch_size = 1 "
"\n session_gc_interval = 1s "
"\n session_gc_batch_size = 2 "
"\n }"
},
Cluster = cluster(Opts),
@ -88,7 +91,7 @@ end_per_testcase(_TestCase, _Config) ->
ok.
%%------------------------------------------------------------------------------
%% Helper fns
%% Helper functions
%%------------------------------------------------------------------------------
cluster(#{n := N} = Opts) ->
@ -144,9 +147,10 @@ start_client(Opts0 = #{}) ->
proto_ver => v5,
properties => #{'Session-Expiry-Interval' => 300}
},
Opts = maps:to_list(emqx_utils_maps:deep_merge(Defaults, Opts0)),
ct:pal("starting client with opts:\n ~p", [Opts]),
{ok, Client} = emqtt:start_link(Opts),
Opts = emqx_utils_maps:deep_merge(Defaults, Opts0),
?tp(notice, "starting client", Opts),
{ok, Client} = emqtt:start_link(maps:to_list(Opts)),
unlink(Client),
on_exit(fun() -> catch emqtt:stop(Client) end),
Client.
@ -161,58 +165,27 @@ is_persistent_connect_opts(#{properties := #{'Session-Expiry-Interval' := EI}})
EI > 0.
list_all_sessions(Node) ->
erpc:call(Node, emqx_persistent_session_ds, list_all_sessions, []).
erpc:call(Node, emqx_persistent_session_ds_state, list_sessions, []).
list_all_subscriptions(Node) ->
erpc:call(Node, emqx_persistent_session_ds, list_all_subscriptions, []).
Sessions = list_all_sessions(Node),
lists:flatmap(
fun(ClientId) ->
#{s := #{subscriptions := Subs}} = erpc:call(
Node, emqx_persistent_session_ds, print_session, [ClientId]
),
maps:to_list(Subs)
end,
Sessions
).
list_all_pubranges(Node) ->
erpc:call(Node, emqx_persistent_session_ds, list_all_pubranges, []).
prop_only_cores_run_gc(CoreNodes) ->
{"only core nodes run gc", fun(Trace) -> ?MODULE:prop_only_cores_run_gc(Trace, CoreNodes) end}.
prop_only_cores_run_gc(Trace, CoreNodes) ->
GCNodes = lists:usort([
N
|| #{
?snk_kind := K,
?snk_meta := #{node := N}
} <- Trace,
lists:member(K, [ds_session_gc, ds_session_gc_lock_taken]),
N =/= node()
]),
?assertEqual(lists:usort(CoreNodes), GCNodes).
%%------------------------------------------------------------------------------
%% Testcases
%%------------------------------------------------------------------------------
t_non_persistent_session_subscription(_Config) ->
ClientId = atom_to_binary(?FUNCTION_NAME),
SubTopicFilter = <<"t/#">>,
?check_trace(
begin
?tp(notice, "starting", #{}),
Client = start_client(#{
clientid => ClientId,
properties => #{'Session-Expiry-Interval' => 0}
}),
{ok, _} = emqtt:connect(Client),
?tp(notice, "subscribing", #{}),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client, SubTopicFilter, qos2),
ok = emqtt:stop(Client),
ok
end,
fun(Trace) ->
ct:pal("trace:\n ~p", [Trace]),
?assertEqual([], ?of_kind(ds_session_subscription_added, Trace)),
ok
end
),
ok.
t_session_subscription_idempotency(Config) ->
[Node1Spec | _] = ?config(node_specs, Config),
[Node1] = ?config(nodes, Config),
@ -220,6 +193,7 @@ t_session_subscription_idempotency(Config) ->
SubTopicFilter = <<"t/+">>,
ClientId = <<"myclientid">>,
?check_trace(
#{timetrap => 30_000},
begin
?force_ordering(
#{?snk_kind := persistent_session_ds_subscription_added},
@ -281,11 +255,11 @@ t_session_unsubscription_idempotency(Config) ->
SubTopicFilter = <<"t/+">>,
ClientId = <<"myclientid">>,
?check_trace(
#{timetrap => 30_000},
begin
?force_ordering(
#{
?snk_kind := persistent_session_ds_subscription_delete,
?snk_span := {complete, _}
?snk_kind := persistent_session_ds_subscription_delete
},
_NEvents0 = 1,
#{?snk_kind := will_restart_node},
@ -385,6 +359,7 @@ do_t_session_discard(Params) ->
ReconnectOpts = ReconnectOpts0#{clientid => ClientId},
SubTopicFilter = <<"t/+">>,
?check_trace(
#{timetrap => 30_000},
begin
?tp(notice, "starting", #{}),
Client0 = start_client(#{
@ -402,27 +377,26 @@ do_t_session_discard(Params) ->
?retry(
_Sleep0 = 100,
_Attempts0 = 50,
true = map_size(emqx_persistent_session_ds:list_all_streams()) > 0
#{} = emqx_persistent_session_ds_state:print_session(ClientId)
),
ok = emqtt:stop(Client0),
?tp(notice, "disconnected", #{}),
?tp(notice, "reconnecting", #{}),
%% we still have streams
?assert(map_size(emqx_persistent_session_ds:list_all_streams()) > 0),
%% we still have the session:
?assertMatch(#{}, emqx_persistent_session_ds_state:print_session(ClientId)),
Client1 = start_client(ReconnectOpts),
{ok, _} = emqtt:connect(Client1),
?assertEqual([], emqtt:subscriptions(Client1)),
case is_persistent_connect_opts(ReconnectOpts) of
true ->
?assertMatch(#{ClientId := _}, emqx_persistent_session_ds:list_all_sessions());
?assertMatch(#{}, emqx_persistent_session_ds_state:print_session(ClientId));
false ->
?assertEqual(#{}, emqx_persistent_session_ds:list_all_sessions())
?assertEqual(
undefined, emqx_persistent_session_ds_state:print_session(ClientId)
)
end,
?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()),
?assertEqual([], emqx_persistent_session_ds_router:topics()),
?assertEqual(#{}, emqx_persistent_session_ds:list_all_streams()),
?assertEqual(#{}, emqx_persistent_session_ds:list_all_pubranges()),
ok = emqtt:stop(Client1),
?tp(notice, "disconnected", #{}),
@ -436,6 +410,8 @@ do_t_session_discard(Params) ->
ok.
t_session_expiration1(Config) ->
%% This testcase verifies that the properties passed in the
%% CONNECT packet are respected by the GC process:
ClientId = atom_to_binary(?FUNCTION_NAME),
Opts = #{
clientid => ClientId,
@ -448,6 +424,9 @@ t_session_expiration1(Config) ->
do_t_session_expiration(Config, Opts).
t_session_expiration2(Config) ->
%% This testcase updates the expiry interval for the session in
%% the _DISCONNECT_ packet. This setting should be respected by GC
%% process:
ClientId = atom_to_binary(?FUNCTION_NAME),
Opts = #{
clientid => ClientId,
@ -462,6 +441,8 @@ t_session_expiration2(Config) ->
do_t_session_expiration(Config, Opts).
do_t_session_expiration(_Config, Opts) ->
%% Sequence is a list of pairs of properties passed through the
%% CONNECT and for the DISCONNECT for each session:
#{
clientid := ClientId,
sequence := [
@ -472,13 +453,14 @@ do_t_session_expiration(_Config, Opts) ->
} = Opts,
CommonParams = #{proto_ver => v5, clientid => ClientId},
?check_trace(
#{timetrap => 30_000},
begin
Topic = <<"some/topic">>,
Params0 = maps:merge(CommonParams, FirstConn),
Client0 = start_client(Params0),
{ok, _} = emqtt:connect(Client0),
{ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(Client0, Topic, ?QOS_2),
Subs0 = emqx_persistent_session_ds:list_all_subscriptions(),
#{s := #{subscriptions := Subs0}} = emqx_persistent_session_ds:print_session(ClientId),
?assertEqual(1, map_size(Subs0), #{subs => Subs0}),
Info0 = maps:from_list(emqtt:info(Client0)),
?assertEqual(0, maps:get(session_present, Info0), #{info => Info0}),
@ -493,7 +475,7 @@ do_t_session_expiration(_Config, Opts) ->
?assertEqual([], Subs1),
emqtt:disconnect(Client1, ?RC_NORMAL_DISCONNECTION, SecondDisconn),
ct:sleep(1_500),
ct:sleep(2_500),
Params2 = maps:merge(CommonParams, ThirdConn),
Client2 = start_client(Params2),
@ -505,9 +487,9 @@ do_t_session_expiration(_Config, Opts) ->
emqtt:publish(Client2, Topic, <<"payload">>),
?assertNotReceive({publish, #{topic := Topic}}),
%% ensure subscriptions are absent from table.
?assertEqual(#{}, emqx_persistent_session_ds:list_all_subscriptions()),
#{s := #{subscriptions := Subs3}} = emqx_persistent_session_ds:print_session(ClientId),
?assertEqual([], maps:to_list(Subs3)),
emqtt:disconnect(Client2, ?RC_NORMAL_DISCONNECTION, ThirdDisconn),
ok
end,
[]
@ -515,14 +497,13 @@ do_t_session_expiration(_Config, Opts) ->
ok.
t_session_gc(Config) ->
GCInterval = ?config(gc_interval, Config),
[Node1, Node2, _Node3] = Nodes = ?config(nodes, Config),
CoreNodes = [Node1, Node2],
[
Port1,
Port2,
Port3
] = lists:map(fun(N) -> get_mqtt_port(N, tcp) end, Nodes),
ct:pal("Ports: ~p", [[Port1, Port2, Port3]]),
CommonParams = #{
clean_start => false,
proto_ver => v5
@ -539,15 +520,16 @@ t_session_gc(Config) ->
end,
?check_trace(
#{timetrap => 30_000},
begin
ClientId0 = <<"session_gc0">>,
Client0 = StartClient(ClientId0, Port1, 30),
ClientId1 = <<"session_gc1">>,
Client1 = StartClient(ClientId1, Port2, 1),
Client1 = StartClient(ClientId1, Port1, 30),
ClientId2 = <<"session_gc2">>,
Client2 = StartClient(ClientId2, Port3, 1),
Client2 = StartClient(ClientId2, Port2, 1),
ClientId3 = <<"session_gc3">>,
Client3 = StartClient(ClientId3, Port3, 1),
lists:foreach(
fun(Client) ->
@ -557,55 +539,48 @@ t_session_gc(Config) ->
{ok, _} = emqtt:publish(Client, Topic, Payload, ?QOS_1),
ok
end,
[Client0, Client1, Client2]
[Client1, Client2, Client3]
),
%% Clients are still alive; no session is garbage collected.
Res0 = ?block_until(
#{
?snk_kind := ds_session_gc,
?snk_span := {complete, _},
?snk_meta := #{node := N}
} when
N =/= node(),
3 * GCInterval + 1_000
),
?assertMatch({ok, _}, Res0),
{ok, #{?snk_meta := #{time := T0}}} = Res0,
Sessions0 = list_all_sessions(Node1),
Subs0 = list_all_subscriptions(Node1),
?assertEqual(3, map_size(Sessions0), #{sessions => Sessions0}),
?assertEqual(3, map_size(Subs0), #{subs => Subs0}),
%% Now we disconnect 2 of them; only those should be GC'ed.
?assertMatch(
{ok, {ok, _}},
?wait_async_action(
emqtt:stop(Client1),
#{?snk_kind := terminate},
1_000
{ok, _},
?block_until(
#{
?snk_kind := ds_session_gc,
?snk_span := {complete, _},
?snk_meta := #{node := N}
} when N =/= node()
)
),
ct:pal("disconnected client1"),
?assertMatch([_, _, _], list_all_sessions(Node1), sessions),
?assertMatch([_, _, _], list_all_subscriptions(Node1), subscriptions),
%% Now we disconnect 2 of them; only those should be GC'ed.
?assertMatch(
{ok, {ok, _}},
?wait_async_action(
emqtt:stop(Client2),
#{?snk_kind := terminate},
1_000
#{?snk_kind := terminate}
)
),
ct:pal("disconnected client2"),
?tp(notice, "disconnected client1", #{}),
?assertMatch(
{ok, {ok, _}},
?wait_async_action(
emqtt:stop(Client3),
#{?snk_kind := terminate}
)
),
?tp(notice, "disconnected client2", #{}),
?assertMatch(
{ok, _},
?block_until(
#{
?snk_kind := ds_session_gc_cleaned,
?snk_meta := #{node := N, time := T},
session_ids := [ClientId1]
} when
N =/= node() andalso T > T0,
4 * GCInterval + 1_000
session_id := ClientId2
}
)
),
?assertMatch(
@ -613,22 +588,14 @@ t_session_gc(Config) ->
?block_until(
#{
?snk_kind := ds_session_gc_cleaned,
?snk_meta := #{node := N, time := T},
session_ids := [ClientId2]
} when
N =/= node() andalso T > T0,
4 * GCInterval + 1_000
session_id := ClientId3
}
)
),
Sessions1 = list_all_sessions(Node1),
Subs1 = list_all_subscriptions(Node1),
?assertEqual(1, map_size(Sessions1), #{sessions => Sessions1}),
?assertEqual(1, map_size(Subs1), #{subs => Subs1}),
?retry(50, 3, [ClientId1] = list_all_sessions(Node1)),
?assertMatch([_], list_all_subscriptions(Node1), subscriptions),
ok
end,
[
prop_only_cores_run_gc(CoreNodes)
]
[]
),
ok.

2
apps/emqx/priv/bpapi.versions
View File

@ -22,6 +22,8 @@
{emqx_delayed,3}.
{emqx_ds,1}.
{emqx_ds,2}.
{emqx_ds,3}.
{emqx_ds,4}.
{emqx_eviction_agent,1}.
{emqx_eviction_agent,2}.
{emqx_exhook,1}.

6
apps/emqx/rebar.config
View File

@ -28,9 +28,9 @@
{gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
{cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.1"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.18.3"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.18.4"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.40.4"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.41.0"}}},
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
{recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
@ -71,7 +71,7 @@
{statistics, true}
]}.
{project_plugins, [erlfmt]}.
{project_plugins, [{erlfmt, "1.3.0"}]}.
{erlfmt, [
{files, [

9
apps/emqx/src/emqx_access_control.erl
View File

@ -183,8 +183,13 @@ log_result(#{username := Username}, Topic, Action, From, Result) ->
}
end,
case Result of
allow -> ?SLOG(info, (LogMeta())#{msg => "authorization_permission_allowed"});
deny -> ?SLOG(warning, (LogMeta())#{msg => "authorization_permission_denied"})
allow ->
?SLOG(info, (LogMeta())#{msg => "authorization_permission_allowed"});
deny ->
?SLOG_THROTTLE(
warning,
(LogMeta())#{msg => authorization_permission_denied}
)
end.
%% @private Format authorization rules source.

12
apps/emqx/src/emqx_alarm.erl
View File

@ -21,12 +21,9 @@
-include("emqx.hrl").
-include("logger.hrl").
%% Mnesia bootstrap
-export([mnesia/1]).
-boot_mnesia({mnesia, [boot]}).
-export([create_tables/0]).
-export([start_link/0]).
%% API
-export([
activate/1,
@ -86,7 +83,7 @@
%% Mnesia bootstrap
%%--------------------------------------------------------------------
mnesia(boot) ->
create_tables() ->
ok = mria:create_table(
?ACTIVATED_ALARM,
[
@ -106,7 +103,8 @@ mnesia(boot) ->
{record_name, deactivated_alarm},
{attributes, record_info(fields, deactivated_alarm)}
]
).
),
[?ACTIVATED_ALARM, ?DEACTIVATED_ALARM].
%%--------------------------------------------------------------------
%% API

195
apps/emqx/src/emqx_banned.erl
View File

@ -25,9 +25,7 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% Mnesia bootstrap
-export([mnesia/1]).
-boot_mnesia({mnesia, [boot]}).
-export([create_tables/0]).
-export([start_link/0, stop/0]).
@ -39,7 +37,9 @@
info/1,
format/1,
parse/1,
clear/0
clear/0,
who/2,
tables/0
]).
%% gen_server callbacks
@ -61,7 +61,8 @@
-elvis([{elvis_style, state_record_and_type, disable}]).
-define(BANNED_TAB, ?MODULE).
-define(BANNED_INDIVIDUAL_TAB, ?MODULE).
-define(BANNED_RULE_TAB, emqx_banned_rules).
%% The default expiration time should be infinite
%% but for compatibility, a large number (1 years) is used here to represent the 'infinite'
@ -76,20 +77,26 @@
%% Mnesia bootstrap
%%--------------------------------------------------------------------
mnesia(boot) ->
ok = mria:create_table(?BANNED_TAB, [
create_tables() ->
Options = [
{type, set},
{rlog_shard, ?COMMON_SHARD},
{storage, disc_copies},
{record_name, banned},
{attributes, record_info(fields, banned)},
{storage_properties, [{ets, [{read_concurrency, true}]}]}
]).
],
ok = mria:create_table(?BANNED_INDIVIDUAL_TAB, Options),
ok = mria:create_table(?BANNED_RULE_TAB, Options),
[?BANNED_INDIVIDUAL_TAB, ?BANNED_RULE_TAB].
%%--------------------------------------------------------------------
%% Data backup
%%--------------------------------------------------------------------
backup_tables() -> [?BANNED_TAB].
backup_tables() -> tables().
-spec tables() -> [atom()].
tables() -> [?BANNED_RULE_TAB, ?BANNED_INDIVIDUAL_TAB].
%% @doc Start the banned server.
-spec start_link() -> startlink_ret().
@ -104,16 +111,10 @@ stop() -> gen_server:stop(?MODULE).
check(ClientInfo) ->
do_check({clientid, maps:get(clientid, ClientInfo, undefined)}) orelse
do_check({username, maps:get(username, ClientInfo, undefined)}) orelse
do_check({peerhost, maps:get(peerhost, ClientInfo, undefined)}).
do_check({_, undefined}) ->
false;
do_check(Who) when is_tuple(Who) ->
case mnesia:dirty_read(?BANNED_TAB, Who) of
[] -> false;
[#banned{until = Until}] -> Until > erlang:system_time(second)
end.
do_check({peerhost, maps:get(peerhost, ClientInfo, undefined)}) orelse
do_check_rules(ClientInfo).
-spec format(emqx_types:banned()) -> map().
format(#banned{
who = Who0,
by = By,
@ -121,7 +122,7 @@ format(#banned{
at = At,
until = Until
}) ->
{As, Who} = maybe_format_host(Who0),
{As, Who} = format_who(Who0),
#{
as => As,
who => Who,
@ -131,6 +132,7 @@ format(#banned{
until => to_rfc3339(Until)
}.
-spec parse(map()) -> emqx_types:banned() | {error, term()}.
parse(Params) ->
case parse_who(Params) of
{error, Reason} ->
@ -155,24 +157,6 @@ parse(Params) ->
{error, ErrorReason}
end
end.
parse_who(#{as := As, who := Who}) ->
parse_who(#{<<"as">> => As, <<"who">> => Who});
parse_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) ->
case inet:parse_address(binary_to_list(Peerhost0)) of
{ok, Peerhost} -> {peerhost, Peerhost};
{error, einval} -> {error, "bad peerhost"}
end;
parse_who(#{<<"as">> := As, <<"who">> := Who}) ->
{As, Who}.
maybe_format_host({peerhost, Host}) ->
AddrBinary = list_to_binary(inet:ntoa(Host)),
{peerhost, AddrBinary};
maybe_format_host({As, Who}) ->
{As, Who}.
to_rfc3339(Timestamp) ->
emqx_utils_calendar:epoch_to_rfc3339(Timestamp, second).
-spec create(emqx_types:banned() | map()) ->
{ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}.
@ -194,7 +178,7 @@ create(#{
create(Banned = #banned{who = Who}) ->
case look_up(Who) of
[] ->
insert_banned(Banned),
insert_banned(table(Who), Banned),
{ok, Banned};
[OldBanned = #banned{until = Until}] ->
%% Don't support shorten or extend the until time by overwrite.
@ -204,33 +188,52 @@ create(Banned = #banned{who = Who}) ->
{error, {already_exist, OldBanned}};
%% overwrite expired one is ok.
false ->
insert_banned(Banned),
insert_banned(table(Who), Banned),
{ok, Banned}
end
end.
-spec look_up(emqx_types:banned_who() | map()) -> [emqx_types:banned()].
look_up(Who) when is_map(Who) ->
look_up(parse_who(Who));
look_up(Who) ->
mnesia:dirty_read(?BANNED_TAB, Who).
mnesia:dirty_read(table(Who), Who).
-spec delete(
{clientid, emqx_types:clientid()}
| {username, emqx_types:username()}
| {peerhost, emqx_types:peerhost()}
) -> ok.
-spec delete(map() | emqx_types:banned_who()) -> ok.
delete(Who) when is_map(Who) ->
delete(parse_who(Who));
delete(Who) ->
mria:dirty_delete(?BANNED_TAB, Who).
mria:dirty_delete(table(Who), Who).
info(InfoKey) ->
mnesia:table_info(?BANNED_TAB, InfoKey).
-spec info(size) -> non_neg_integer().
info(size) ->
mnesia:table_info(?BANNED_INDIVIDUAL_TAB, size) + mnesia:table_info(?BANNED_RULE_TAB, size).
-spec clear() -> ok.
clear() ->
_ = mria:clear_table(?BANNED_TAB),
_ = mria:clear_table(?BANNED_INDIVIDUAL_TAB),
_ = mria:clear_table(?BANNED_RULE_TAB),
ok.
%% Creating banned with `#banned{}` records is exposed as a public API
%% so we need helpers to create the `who` field of `#banned{}` records
-spec who(atom(), binary() | inet:ip_address() | esockd_cidr:cidr()) -> emqx_types:banned_who().
who(clientid, ClientId) when is_binary(ClientId) -> {clientid, ClientId};
who(username, Username) when is_binary(Username) -> {username, Username};
who(peerhost, Peerhost) when is_tuple(Peerhost) -> {peerhost, Peerhost};
who(peerhost, Peerhost) when is_binary(Peerhost) ->
{ok, Addr} = inet:parse_address(binary_to_list(Peerhost)),
{peerhost, Addr};
who(clientid_re, RE) when is_binary(RE) ->
{ok, RECompiled} = re:compile(RE),
{clientid_re, {RECompiled, RE}};
who(username_re, RE) when is_binary(RE) ->
{ok, RECompiled} = re:compile(RE),
{username_re, {RECompiled, RE}};
who(peerhost_net, CIDR) when is_tuple(CIDR) -> {peerhost_net, CIDR};
who(peerhost_net, CIDR) when is_binary(CIDR) ->
{peerhost_net, esockd_cidr:parse(binary_to_list(CIDR), true)}.
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
@ -265,6 +268,81 @@ code_change(_OldVsn, State, _Extra) ->
%% Internal functions
%%--------------------------------------------------------------------
do_check({_, undefined}) ->
false;
do_check(Who) when is_tuple(Who) ->
case mnesia:dirty_read(table(Who), Who) of
[] -> false;
[#banned{until = Until}] -> Until > erlang:system_time(second)
end.
do_check_rules(ClientInfo) ->
Rules = all_rules(),
Now = erlang:system_time(second),
lists:any(
fun(Rule) -> is_rule_actual(Rule, Now) andalso do_check_rule(Rule, ClientInfo) end, Rules
).
is_rule_actual(#banned{until = Until}, Now) ->
Until > Now.
do_check_rule(#banned{who = {clientid_re, {RE, _}}}, #{clientid := ClientId}) ->
is_binary(ClientId) andalso re:run(ClientId, RE) =/= nomatch;
do_check_rule(#banned{who = {clientid_re, _}}, #{}) ->
false;
do_check_rule(#banned{who = {username_re, {RE, _}}}, #{username := Username}) ->
is_binary(Username) andalso re:run(Username, RE) =/= nomatch;
do_check_rule(#banned{who = {username_re, _}}, #{}) ->
false;
do_check_rule(#banned{who = {peerhost_net, CIDR}}, #{peerhost := Peerhost}) ->
esockd_cidr:match(Peerhost, CIDR);
do_check_rule(#banned{who = {peerhost_net, _}}, #{}) ->
false.
parse_who(#{as := As, who := Who}) ->
parse_who(#{<<"as">> => As, <<"who">> => Who});
parse_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) ->
case inet:parse_address(binary_to_list(Peerhost0)) of
{ok, Peerhost} -> {peerhost, Peerhost};
{error, einval} -> {error, "bad peerhost"}
end;
parse_who(#{<<"as">> := peerhost_net, <<"who">> := CIDRString}) ->
try esockd_cidr:parse(binary_to_list(CIDRString), true) of
CIDR -> {peerhost_net, CIDR}
catch
error:Error -> {error, Error}
end;
parse_who(#{<<"as">> := AsRE, <<"who">> := Who}) when
AsRE =:= clientid_re orelse AsRE =:= username_re
->
case re:compile(Who) of
{ok, RE} -> {AsRE, {RE, Who}};
{error, _} = Error -> Error
end;
parse_who(#{<<"as">> := As, <<"who">> := Who}) when As =:= clientid orelse As =:= username ->
{As, Who}.
format_who({peerhost, Host}) ->
AddrBinary = list_to_binary(inet:ntoa(Host)),
{peerhost, AddrBinary};
format_who({peerhost_net, CIDR}) ->
CIDRBinary = list_to_binary(esockd_cidr:to_string(CIDR)),
{peerhost_net, CIDRBinary};
format_who({AsRE, {_RE, REOriginal}}) when AsRE =:= clientid_re orelse AsRE =:= username_re ->
{AsRE, REOriginal};
format_who({As, Who}) when As =:= clientid orelse As =:= username ->
{As, Who}.
to_rfc3339(Timestamp) ->
emqx_utils_calendar:epoch_to_rfc3339(Timestamp, second).
table({username, _Username}) -> ?BANNED_INDIVIDUAL_TAB;
table({clientid, _ClientId}) -> ?BANNED_INDIVIDUAL_TAB;
table({peerhost, _Peerhost}) -> ?BANNED_INDIVIDUAL_TAB;
table({username_re, _UsernameRE}) -> ?BANNED_RULE_TAB;
table({clientid_re, _ClientIdRE}) -> ?BANNED_RULE_TAB;
table({peerhost_net, _PeerhostNet}) -> ?BANNED_RULE_TAB.
-ifdef(TEST).
ensure_expiry_timer(State) ->
State#{expiry_timer := emqx_utils:start_timer(10, expire)}.
@ -274,19 +352,27 @@ ensure_expiry_timer(State) ->
-endif.
expire_banned_items(Now) ->
lists:foreach(
fun(Tab) ->
expire_banned_items(Now, Tab)
end,
[?BANNED_INDIVIDUAL_TAB, ?BANNED_RULE_TAB]
).
expire_banned_items(Now, Tab) ->
mnesia:foldl(
fun
(B = #banned{until = Until}, _Acc) when Until < Now ->
mnesia:delete_object(?BANNED_TAB, B, sticky_write);
mnesia:delete_object(Tab, B, sticky_write);
(_, _Acc) ->
ok
end,
ok,
?BANNED_TAB
Tab
).
insert_banned(Banned) ->
mria:dirty_write(?BANNED_TAB, Banned),
insert_banned(Tab, Banned) ->
mria:dirty_write(Tab, Banned),
on_banned(Banned).
on_banned(#banned{who = {clientid, ClientId}}) ->
@ -302,3 +388,6 @@ on_banned(#banned{who = {clientid, ClientId}}) ->
ok;
on_banned(_) ->
ok.
all_rules() ->
ets:tab2list(?BANNED_RULE_TAB).

24
apps/emqx/src/emqx_broker.erl
View File

@ -85,13 +85,13 @@
%% Guards
-define(IS_SUBID(Id), (is_binary(Id) orelse is_atom(Id))).
-define(cast_or_eval(Pid, Msg, Expr),
case Pid =:= self() of
true ->
-define(cast_or_eval(PICK, Msg, Expr),
case PICK of
__X_Pid when __X_Pid =:= self() ->
_ = Expr,
ok;
false ->
cast(Pid, Msg)
__X_Pid ->
cast(__X_Pid, Msg)
end
).
@ -243,7 +243,7 @@ publish(Msg) when is_record(Msg, message) ->
[];
Msg1 = #message{topic = Topic} ->
PersistRes = persist_publish(Msg1),
PersistRes ++ route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1))
route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1), PersistRes)
end.
persist_publish(Msg) ->
@ -283,18 +283,20 @@ delivery(Msg) -> #delivery{sender = self(), message = Msg}.
%% Route
%%--------------------------------------------------------------------
-spec route([emqx_types:route_entry()], emqx_types:delivery()) ->
-spec route([emqx_types:route_entry()], emqx_types:delivery(), nil() | [persisted]) ->
emqx_types:publish_result().
route([], #delivery{message = Msg}) ->
route([], #delivery{message = Msg}, _PersistRes = []) ->
ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, no_subscribers]),
ok = inc_dropped_cnt(Msg),
[];
route(Routes, Delivery) ->
route([], _Delivery, PersistRes = [_ | _]) ->
PersistRes;
route(Routes, Delivery, PersistRes) ->
lists:foldl(
fun(Route, Acc) ->
[do_route(Route, Delivery) | Acc]
end,
[],
PersistRes,
Routes
).
@ -438,7 +440,7 @@ subscribed(SubId, Topic) when ?IS_SUBID(SubId) ->
SubPid = emqx_broker_helper:lookup_subpid(SubId),
ets:member(?SUBOPTION, {Topic, SubPid}).
-spec get_subopts(pid(), emqx_types:topic() | emqx_types:share()) -> maybe(emqx_types:subopts()).
-spec get_subopts(pid(), emqx_types:topic() | emqx_types:share()) -> option(emqx_types:subopts()).
get_subopts(SubPid, Topic) when is_pid(SubPid), ?IS_TOPIC(Topic) ->
lookup_value(?SUBOPTION, {Topic, SubPid});
get_subopts(SubId, Topic) when ?IS_SUBID(SubId) ->

4
apps/emqx/src/emqx_broker_helper.erl
View File

@ -71,11 +71,11 @@ register_sub(SubPid, SubId) when is_pid(SubPid) ->
error(subid_conflict)
end.
-spec lookup_subid(pid()) -> maybe(emqx_types:subid()).
-spec lookup_subid(pid()) -> option(emqx_types:subid()).
lookup_subid(SubPid) when is_pid(SubPid) ->
emqx_utils_ets:lookup_value(?SUBMON, SubPid).
-spec lookup_subpid(emqx_types:subid()) -> maybe(pid()).
-spec lookup_subpid(emqx_types:subid()) -> option(pid()).
lookup_subpid(SubId) ->
emqx_utils_ets:lookup_value(?SUBID, SubId).

4
apps/emqx/src/emqx_broker_sup.erl
View File

@ -23,6 +23,10 @@
-export([init/1]).
start_link() ->
ok = mria:wait_for_tables(
emqx_shared_sub:create_tables() ++
emqx_exclusive_subscription:create_tables()
),
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
%%--------------------------------------------------------------------

59
apps/emqx/src/emqx_channel.erl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2019-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -84,21 +84,21 @@
%% MQTT ClientInfo
clientinfo :: emqx_types:clientinfo(),
%% MQTT Session
session :: maybe(emqx_session:t()),
session :: option(emqx_session:t()),
%% Keepalive
keepalive :: maybe(emqx_keepalive:keepalive()),
keepalive :: option(emqx_keepalive:keepalive()),
%% MQTT Will Msg
will_msg :: maybe(emqx_types:message()),
will_msg :: option(emqx_types:message()),
%% MQTT Topic Aliases
topic_aliases :: emqx_types:topic_aliases(),
%% MQTT Topic Alias Maximum
alias_maximum :: maybe(map()),
alias_maximum :: option(map()),
%% Authentication Data Cache
auth_cache :: maybe(map()),
auth_cache :: option(map()),
%% Quota checkers
quota :: emqx_limiter_container:container(),
%% Timers
timers :: #{atom() => disabled | maybe(reference())},
timers :: #{atom() => disabled | option(reference())},
%% Conn State
conn_state :: conn_state(),
%% Takeover
@ -191,7 +191,11 @@ info(topic_aliases, #channel{topic_aliases = Aliases}) ->
info(alias_maximum, #channel{alias_maximum = Limits}) ->
Limits;
info(timers, #channel{timers = Timers}) ->
Timers.
Timers;
info(session_state, #channel{session = Session}) ->
Session;
info(impl, #channel{session = Session}) ->
emqx_session:info(impl, Session).
set_conn_state(ConnState, Channel) ->
Channel#channel{conn_state = ConnState}.
@ -536,13 +540,17 @@ handle_in(?AUTH_PACKET(), Channel) ->
handle_out(disconnect, ?RC_IMPLEMENTATION_SPECIFIC_ERROR, Channel);
handle_in({frame_error, Reason}, Channel = #channel{conn_state = idle}) ->
shutdown(shutdown_count(frame_error, Reason), Channel);
handle_in({frame_error, frame_too_large}, Channel = #channel{conn_state = connecting}) ->
handle_in(
{frame_error, #{cause := frame_too_large} = R}, Channel = #channel{conn_state = connecting}
) ->
shutdown(
shutdown_count(frame_error, frame_too_large), ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE), Channel
shutdown_count(frame_error, R), ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE), Channel
);
handle_in({frame_error, Reason}, Channel = #channel{conn_state = connecting}) ->
shutdown(shutdown_count(frame_error, Reason), ?CONNACK_PACKET(?RC_MALFORMED_PACKET), Channel);
handle_in({frame_error, frame_too_large}, Channel = #channel{conn_state = ConnState}) when
handle_in(
{frame_error, #{cause := frame_too_large}}, Channel = #channel{conn_state = ConnState}
) when
ConnState =:= connected orelse ConnState =:= reauthenticating
->
handle_out(disconnect, {?RC_PACKET_TOO_LARGE, frame_too_large}, Channel);
@ -608,10 +616,10 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
Msg = packet_to_message(NPacket, NChannel),
do_publish(PacketId, Msg, NChannel);
{error, Rc = ?RC_NOT_AUTHORIZED, NChannel} ->
?SLOG(
?SLOG_THROTTLE(
warning,
#{
msg => "cannot_publish_to_topic",
msg => cannot_publish_to_topic_due_to_not_authorized,
reason => emqx_reason_codes:name(Rc)
},
#{topic => Topic}
@ -627,10 +635,10 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
handle_out(disconnect, Rc, NChannel)
end;
{error, Rc = ?RC_QUOTA_EXCEEDED, NChannel} ->
?SLOG(
?SLOG_THROTTLE(
warning,
#{
msg => "cannot_publish_to_topic",
msg => cannot_publish_to_topic_due_to_quota_exceeded,
reason => emqx_reason_codes:name(Rc)
},
#{topic => Topic}
@ -928,7 +936,8 @@ handle_deliver(
Delivers1 = maybe_nack(Delivers),
Messages = emqx_session:enrich_delivers(ClientInfo, Delivers1, Session),
NSession = emqx_session_mem:enqueue(ClientInfo, Messages, Session),
{ok, Channel#channel{session = NSession}};
%% we need to update stats here, as the stats_timer is canceled after disconnected
{ok, {event, updated}, Channel#channel{session = NSession}};
handle_deliver(Delivers, Channel) ->
Delivers1 = emqx_external_trace:start_trace_send(Delivers, trace_info(Channel)),
do_handle_deliver(Delivers1, Channel).
@ -1546,6 +1555,8 @@ set_username(
set_username(_ConnPkt, ClientInfo) ->
{ok, ClientInfo}.
%% The `is_bridge` bit flag in CONNECT packet (parsed as `bridge_mode`)
%% is invented by mosquitto, named 'try_private': https://mosquitto.org/man/mosquitto-conf-5.html
set_bridge_mode(#mqtt_packet_connect{is_bridge = true}, ClientInfo) ->
{ok, ClientInfo#{is_bridge => true}};
set_bridge_mode(_ConnPkt, _ClientInfo) ->
@ -2002,14 +2013,15 @@ merge_default_subopts(SubOpts) ->
%%--------------------------------------------------------------------
%% Enrich ConnAck Caps
enrich_connack_caps(
AckProps,
?IS_MQTT_V5 = #channel{
enrich_connack_caps(AckProps, ?IS_MQTT_V5 = Channel) ->
#channel{
clientinfo = #{
zone := Zone
},
conninfo = #{
receive_maximum := ReceiveMaximum
}
}
) ->
} = Channel,
#{
max_packet_size := MaxPktSize,
max_qos_allowed := MaxQoS,
@ -2024,7 +2036,8 @@ enrich_connack_caps(
'Topic-Alias-Maximum' => MaxAlias,
'Wildcard-Subscription-Available' => flag(Wildcard),
'Subscription-Identifier-Available' => 1,
'Shared-Subscription-Available' => flag(Shared)
'Shared-Subscription-Available' => flag(Shared),
'Receive-Maximum' => ReceiveMaximum
},
%% MQTT 5.0 - 3.2.2.3.4:
%% It is a Protocol Error to include Maximum QoS more than once,
@ -2318,6 +2331,8 @@ shutdown(Reason, Reply, Packet, Channel) ->
%% process exits with {shutdown, #{shutdown_count := Kind}} will trigger
%% the connection supervisor (esockd) to keep a shutdown-counter grouped by Kind
shutdown_count(_Kind, #{cause := Cause} = Reason) when is_atom(Cause) ->
Reason#{shutdown_count => Cause};
shutdown_count(Kind, Reason) when is_map(Reason) ->
Reason#{shutdown_count => Kind};
shutdown_count(Kind, Reason) ->

25
apps/emqx/src/emqx_cm.erl
View File

@ -1,5 +1,5 @@
%%-------------------------------------------------------------------
%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -124,7 +124,8 @@
{?CHAN_TAB, 'channels.count', 'channels.max'},
{?CHAN_TAB, 'sessions.count', 'sessions.max'},
{?CHAN_CONN_TAB, 'connections.count', 'connections.max'},
{?CHAN_LIVE_TAB, 'live_connections.count', 'live_connections.max'}
{?CHAN_LIVE_TAB, 'live_connections.count', 'live_connections.max'},
{?CHAN_REG_TAB, 'cluster_sessions.count', 'cluster_sessions.max'}
]).
%% Batch drain
@ -200,12 +201,12 @@ do_unregister_channel({_ClientId, ChanPid} = Chan) ->
true.
%% @doc Get info of a channel.
-spec get_chan_info(emqx_types:clientid()) -> maybe(emqx_types:infos()).
-spec get_chan_info(emqx_types:clientid()) -> option(emqx_types:infos()).
get_chan_info(ClientId) ->
with_channel(ClientId, fun(ChanPid) -> get_chan_info(ClientId, ChanPid) end).
-spec do_get_chan_info(emqx_types:clientid(), chan_pid()) ->
maybe(emqx_types:infos()).
option(emqx_types:infos()).
do_get_chan_info(ClientId, ChanPid) ->
Chan = {ClientId, ChanPid},
try
@ -215,7 +216,7 @@ do_get_chan_info(ClientId, ChanPid) ->
end.
-spec get_chan_info(emqx_types:clientid(), chan_pid()) ->
maybe(emqx_types:infos()).
option(emqx_types:infos()).
get_chan_info(ClientId, ChanPid) ->
wrap_rpc(emqx_cm_proto_v2:get_chan_info(ClientId, ChanPid)).
@ -230,12 +231,12 @@ set_chan_info(ClientId, Info) when ?IS_CLIENTID(ClientId) ->
end.
%% @doc Get channel's stats.
-spec get_chan_stats(emqx_types:clientid()) -> maybe(emqx_types:stats()).
-spec get_chan_stats(emqx_types:clientid()) -> option(emqx_types:stats()).
get_chan_stats(ClientId) ->
with_channel(ClientId, fun(ChanPid) -> get_chan_stats(ClientId, ChanPid) end).
-spec do_get_chan_stats(emqx_types:clientid(), chan_pid()) ->
maybe(emqx_types:stats()).
option(emqx_types:stats()).
do_get_chan_stats(ClientId, ChanPid) ->
Chan = {ClientId, ChanPid},
try
@ -245,7 +246,7 @@ do_get_chan_stats(ClientId, ChanPid) ->
end.
-spec get_chan_stats(emqx_types:clientid(), chan_pid()) ->
maybe(emqx_types:stats()).
option(emqx_types:stats()).
get_chan_stats(ClientId, ChanPid) ->
wrap_rpc(emqx_cm_proto_v2:get_chan_stats(ClientId, ChanPid)).
@ -325,7 +326,7 @@ takeover_session_end({ConnMod, ChanPid}) ->
end.
-spec pick_channel(emqx_types:clientid()) ->
maybe(pid()).
option(pid()).
pick_channel(ClientId) ->
case lookup_channels(ClientId) of
[] ->
@ -670,7 +671,11 @@ handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}
ChanPids = [Pid | emqx_utils:drain_down(BatchSize)],
{Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon),
lists:foreach(fun mark_channel_disconnected/1, ChanPids),
ok = emqx_pool:async_submit(fun lists:foreach/2, [fun ?MODULE:clean_down/1, Items]),
ok = emqx_pool:async_submit_to_pool(
?CM_POOL,
fun lists:foreach/2,
[fun ?MODULE:clean_down/1, Items]
),
{noreply, State#{chan_pmon := PMon1}};
handle_info(Info, State) ->
?SLOG(error, #{msg => "unexpected_info", info => Info}),

2
apps/emqx/src/emqx_cm_locker.erl
View File

@ -32,7 +32,7 @@ start_link() ->
ekka_locker:start_link(?MODULE).
-spec trans(
maybe(emqx_types:clientid()),
option(emqx_types:clientid()),
fun(([node()]) -> any())
) -> any().
trans(undefined, Fun) ->

172
apps/emqx/src/emqx_cm_registry.erl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2019-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2019-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -19,18 +19,15 @@
-behaviour(gen_server).
-include("emqx.hrl").
-include("emqx_cm.hrl").
-include("logger.hrl").
-include("types.hrl").
-export([start_link/0]).
-export([is_enabled/0]).
-export([is_enabled/0, is_hist_enabled/0]).
-export([
register_channel/1,
unregister_channel/1
register_channel2/1,
unregister_channel/1,
unregister_channel2/1
]).
-export([lookup_channels/1]).
@ -50,10 +47,13 @@
do_cleanup_channels/1
]).
-define(REGISTRY, ?MODULE).
-define(LOCK, {?MODULE, cleanup_down}).
-include("emqx.hrl").
-include("emqx_cm.hrl").
-include("logger.hrl").
-include("types.hrl").
-record(channel, {chid, pid}).
-define(REGISTRY, ?MODULE).
-define(NODE_DOWN_CLEANUP_LOCK, {?MODULE, cleanup_down}).
%% @doc Start the global channel registry.
-spec start_link() -> startlink_ret().
@ -69,6 +69,11 @@ start_link() ->
is_enabled() ->
emqx:get_config([broker, enable_session_registry]).
%% @doc Is the global session registration history enabled?
-spec is_hist_enabled() -> boolean().
is_hist_enabled() ->
retain_duration() > 0.
%% @doc Register a global channel.
-spec register_channel(
emqx_types:clientid()
@ -77,11 +82,21 @@ is_enabled() ->
register_channel(ClientId) when is_binary(ClientId) ->
register_channel({ClientId, self()});
register_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) ->
IsHistEnabled = is_hist_enabled(),
case is_enabled() of
true -> mria:dirty_write(?CHAN_REG_TAB, record(ClientId, ChanPid));
false -> ok
true when IsHistEnabled ->
mria:async_dirty(?CM_SHARD, fun ?MODULE:register_channel2/1, [record(ClientId, ChanPid)]);
true ->
mria:dirty_write(?CHAN_REG_TAB, record(ClientId, ChanPid));
false ->
ok
end.
%% @private
register_channel2(#channel{chid = ClientId} = Record) ->
_ = delete_hist_d(ClientId),
mria:dirty_write(?CHAN_REG_TAB, Record).
%% @doc Unregister a global channel.
-spec unregister_channel(
emqx_types:clientid()
@ -90,19 +105,54 @@ register_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid)
unregister_channel(ClientId) when is_binary(ClientId) ->
unregister_channel({ClientId, self()});
unregister_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) ->
IsHistEnabled = is_hist_enabled(),
case is_enabled() of
true -> mria:dirty_delete_object(?CHAN_REG_TAB, record(ClientId, ChanPid));
false -> ok
true when IsHistEnabled ->
mria:async_dirty(?CM_SHARD, fun ?MODULE:unregister_channel2/1, [
record(ClientId, ChanPid)
]);
true ->
mria:dirty_delete_object(?CHAN_REG_TAB, record(ClientId, ChanPid));
false ->
ok
end.
%% @private
unregister_channel2(#channel{chid = ClientId} = Record) ->
mria:dirty_delete_object(?CHAN_REG_TAB, Record),
ok = insert_hist_d(ClientId).
%% @doc Lookup the global channels.
-spec lookup_channels(emqx_types:clientid()) -> list(pid()).
lookup_channels(ClientId) ->
[ChanPid || #channel{pid = ChanPid} <- mnesia:dirty_read(?CHAN_REG_TAB, ClientId)].
lists:filtermap(
fun
(#channel{pid = ChanPid}) when is_pid(ChanPid) ->
case is_pid_down(ChanPid) of
true ->
false;
_ ->
{true, ChanPid}
end;
(_) ->
false
end,
mnesia:dirty_read(?CHAN_REG_TAB, ClientId)
).
%% Return 'true' or 'false' if it's a local pid.
%% Otherwise return 'unknown'.
is_pid_down(Pid) when node(Pid) =:= node() ->
not erlang:is_process_alive(Pid);
is_pid_down(_) ->
unknown.
record(ClientId, ChanPid) ->
#channel{chid = ClientId, pid = ChanPid}.
hist(ClientId) ->
#channel{chid = ClientId, pid = now_ts()}.
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
@ -158,15 +208,95 @@ code_change(_OldVsn, State, _Extra) ->
cleanup_channels(Node) ->
global:trans(
{?LOCK, self()},
{?NODE_DOWN_CLEANUP_LOCK, self()},
fun() ->
mria:transaction(?CM_SHARD, fun ?MODULE:do_cleanup_channels/1, [Node])
end
).
do_cleanup_channels(Node) ->
Pat = [{#channel{pid = '$1', _ = '_'}, [{'==', {node, '$1'}, Node}], ['$_']}],
lists:foreach(fun delete_channel/1, mnesia:select(?CHAN_REG_TAB, Pat, write)).
Pat = [
{
#channel{pid = '$1', _ = '_'},
_Match = [{'andalso', {is_pid, '$1'}, {'==', {node, '$1'}, Node}}],
_Return = ['$_']
}
],
IsHistEnabled = is_hist_enabled(),
lists:foreach(
fun(Chan) -> delete_channel(IsHistEnabled, Chan) end,
mnesia:select(?CHAN_REG_TAB, Pat, write)
).
delete_channel(Chan) ->
mnesia:delete_object(?CHAN_REG_TAB, Chan, write).
delete_channel(IsHistEnabled, Chan) ->
mnesia:delete_object(?CHAN_REG_TAB, Chan, write),
case IsHistEnabled of
true ->
insert_hist_t(Chan#channel.chid);
false ->
ok
end.
%%--------------------------------------------------------------------
%% History entry operations
%% Insert unregistration history in a transaction when unregistering the last channel for a clientid.
insert_hist_t(ClientId) ->
case delete_hist_t(ClientId) of
true ->
ok;
false ->
mnesia:write(?CHAN_REG_TAB, hist(ClientId), write)
end.
%% Dirty insert unregistration history.
%% Since dirty opts are used, async pool workers may race deletes and inserts,
%% so there could be more than one history records for a clientid,
%% but it should be eventually consistent after the client re-registers or the periodic cleanup.
insert_hist_d(ClientId) ->
%% delete old hist records first
case delete_hist_d(ClientId) of
true ->
ok;
false ->
mria:dirty_write(?CHAN_REG_TAB, hist(ClientId))
end.
%% Current timestamp in seconds.
now_ts() ->
erlang:system_time(seconds).
%% Delete all history records for a clientid, return true if there is a Pid found.
delete_hist_t(ClientId) ->
fold_hist(
fun(Hist) -> mnesia:delete_object(?CHAN_REG_TAB, Hist, write) end,
mnesia:read(?CHAN_REG_TAB, ClientId, write)
).
%% Delete all history records for a clientid, return true if there is a Pid found.
delete_hist_d(ClientId) ->
fold_hist(
fun(Hist) -> mria:dirty_delete_object(?CHAN_REG_TAB, Hist) end,
mnesia:dirty_read(?CHAN_REG_TAB, ClientId)
).
%% Fold over the history records, return true if there is a Pid found.
fold_hist(F, List) ->
lists:foldl(
fun(#channel{pid = Ts} = Record, HasPid) ->
case is_integer(Ts) of
true ->
ok = F(Record),
HasPid;
false ->
true
end
end,
false,
List
).
%% Return the session registration history retain duration.
-spec retain_duration() -> non_neg_integer().
retain_duration() ->
emqx:get_config([broker, session_history_retain]).

194
apps/emqx/src/emqx_cm_registry_keeper.erl Normal file
View File

@ -0,0 +1,194 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc This module implements the global session registry history cleaner.
-module(emqx_cm_registry_keeper).
-behaviour(gen_server).
-export([
start_link/0,
count/1
]).
%% gen_server callbacks
-export([
init/1,
handle_call/3,
handle_cast/2,
handle_info/2,
terminate/2,
code_change/3
]).
-include_lib("stdlib/include/ms_transform.hrl").
-include("emqx_cm.hrl").
-define(CACHE_COUNT_THRESHOLD, 1000).
-define(MIN_COUNT_INTERVAL_SECONDS, 5).
-define(CLEANUP_CHUNK_SIZE, 10000).
-define(IS_HIST_ENABLED(RETAIN), (RETAIN > 0)).
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
init(_) ->
case mria_config:whoami() =:= replicant of
true ->
ignore;
false ->
ok = send_delay_start(),
{ok, #{next_clientid => undefined}}
end.
%% @doc Count the number of sessions.
%% Include sessions which are expired since the given timestamp if `since' is greater than 0.
-spec count(non_neg_integer()) -> non_neg_integer().
count(Since) ->
Retain = retain_duration(),
Now = now_ts(),
%% Get table size if hist is not enabled or
%% Since is before the earliest possible retention time.
IsCountAll = (not ?IS_HIST_ENABLED(Retain) orelse (Now - Retain >= Since)),
case IsCountAll of
true ->
mnesia:table_info(?CHAN_REG_TAB, size);
false ->
%% make a gen call to avoid many callers doing the same concurrently
gen_server:call(?MODULE, {count, Since}, infinity)
end.
handle_call({count, Since}, _From, State) ->
{LastCountTime, LastCount} =
case State of
#{last_count_time := T, last_count := C} ->
{T, C};
_ ->
{0, 0}
end,
Now = now_ts(),
Total = mnesia:table_info(?CHAN_REG_TAB, size),
%% Always count if the table is small enough
%% or when the last count is too old
IsTableSmall = (Total < ?CACHE_COUNT_THRESHOLD),
IsLastCountOld = (Now - LastCountTime > ?MIN_COUNT_INTERVAL_SECONDS),
case IsTableSmall orelse IsLastCountOld of
true ->
Count = do_count(Since),
CountFinishedAt = now_ts(),
{reply, Count, State#{last_count_time => CountFinishedAt, last_count => Count}};
false ->
{reply, LastCount, State}
end;
handle_call(_Request, _From, State) ->
{reply, ok, State}.
handle_cast(_Msg, State) ->
{noreply, State}.
handle_info(start, #{next_clientid := NextClientId} = State) ->
case is_hist_enabled() of
true ->
NewNext =
case cleanup_one_chunk(NextClientId) of
'$end_of_table' ->
ok = send_delay_start(),
undefined;
Id ->
_ = erlang:garbage_collect(),
Id
end,
{noreply, State#{next_clientid := NewNext}};
false ->
%% if not enabled, delay and check again
%% because it might be enabled from online config change while waiting
ok = send_delay_start(),
{noreply, State}
end;
handle_info(_Info, State) ->
{noreply, State}.
terminate(_Reason, _State) ->
ok.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
cleanup_one_chunk(NextClientId) ->
Retain = retain_duration(),
Now = now_ts(),
IsExpired = fun(#channel{pid = Ts}) ->
is_integer(Ts) andalso (Ts < Now - Retain)
end,
cleanup_loop(NextClientId, ?CLEANUP_CHUNK_SIZE, IsExpired).
cleanup_loop(ClientId, 0, _IsExpired) ->
ClientId;
cleanup_loop('$end_of_table', _Count, _IsExpired) ->
'$end_of_table';
cleanup_loop(undefined, Count, IsExpired) ->
cleanup_loop(mnesia:dirty_first(?CHAN_REG_TAB), Count, IsExpired);
cleanup_loop(ClientId, Count, IsExpired) ->
Records = mnesia:dirty_read(?CHAN_REG_TAB, ClientId),
Next = mnesia:dirty_next(?CHAN_REG_TAB, ClientId),
lists:foreach(
fun(R) ->
case IsExpired(R) of
true ->
mria:dirty_delete_object(?CHAN_REG_TAB, R);
false ->
ok
end
end,
Records
),
cleanup_loop(Next, Count - 1, IsExpired).
is_hist_enabled() ->
retain_duration() > 0.
%% Return the session registration history retain duration in seconds.
-spec retain_duration() -> non_neg_integer().
retain_duration() ->
emqx:get_config([broker, session_history_retain]).
cleanup_delay() ->
Default = timer:minutes(2),
case retain_duration() of
0 ->
%% prepare for online config change
Default;
RetainSeconds ->
Min = max(timer:seconds(1), timer:seconds(RetainSeconds) div 4),
min(Min, Default)
end.
send_delay_start() ->
Delay = cleanup_delay(),
ok = send_delay_start(Delay).
send_delay_start(Delay) ->
_ = erlang:send_after(Delay, self(), start),
ok.
now_ts() ->
erlang:system_time(seconds).
do_count(Since) ->
Ms = ets:fun2ms(fun(#channel{pid = V}) ->
is_pid(V) orelse (is_integer(V) andalso (V >= Since))
end),
ets:select_count(?CHAN_REG_TAB, Ms).

7
apps/emqx/src/emqx_cm_sup.erl
View File

@ -25,11 +25,14 @@
%% for test
-export([restart_flapping/0]).
-include("emqx_cm.hrl").
%%--------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------
start_link() ->
ok = mria:wait_for_tables(emqx_banned:create_tables()),
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
%%--------------------------------------------------------------------
@ -45,7 +48,9 @@ init([]) ->
Banned = child_spec(emqx_banned, 1000, worker),
Flapping = child_spec(emqx_flapping, 1000, worker),
Locker = child_spec(emqx_cm_locker, 5000, worker),
CmPool = emqx_pool_sup:spec(emqx_cm_pool_sup, [?CM_POOL, random, {emqx_pool, start_link, []}]),
Registry = child_spec(emqx_cm_registry, 5000, worker),
RegistryKeeper = child_spec(emqx_cm_registry_keeper, 5000, worker),
Manager = child_spec(emqx_cm, 5000, worker),
DSSessionGCSup = child_spec(emqx_persistent_session_ds_sup, infinity, supervisor),
Children =
@ -53,7 +58,9 @@ init([]) ->
Banned,
Flapping,
Locker,
CmPool,
Registry,
RegistryKeeper,
Manager,
DSSessionGCSup
],

14
apps/emqx/src/emqx_config_handler.erl
View File

@ -675,9 +675,19 @@ merge_to_override_config(RawConf, Opts) ->
maps:merge(UpgradedOldConf, RawConf).
upgrade_conf(Conf) ->
ConfigLoader = emqx_app:get_config_loader(),
%% ensure module loaded
_ = ConfigLoader:module_info(),
case erlang:function_exported(ConfigLoader, schema_module, 0) of
true ->
try_upgrade_conf(apply(ConfigLoader, schema_module, []), Conf);
false ->
%% this happens during emqx app standalone test
Conf
end.
try_upgrade_conf(SchemaModule, Conf) ->
try
ConfLoader = emqx_app:get_config_loader(),
SchemaModule = apply(ConfLoader, schema_module, []),
apply(SchemaModule, upgrade_raw_conf, [Conf])
catch
ErrorType:Reason:Stack ->

8
apps/emqx/src/emqx_connection.erl
View File

@ -99,13 +99,13 @@
%% Channel State
channel :: emqx_channel:channel(),
%% GC State
gc_state :: maybe(emqx_gc:gc_state()),
gc_state :: option(emqx_gc:gc_state()),
%% Stats Timer
stats_timer :: disabled | maybe(reference()),
stats_timer :: disabled | option(reference()),
%% Idle Timeout
idle_timeout :: integer() | infinity,
%% Idle Timer
idle_timer :: maybe(reference()),
idle_timer :: option(reference()),
%% Zone name
zone :: atom(),
%% Listener Type and Name
@ -121,7 +121,7 @@
limiter_timer :: undefined | reference(),
%% QUIC conn owner pid if in use.
quic_conn_pid :: maybe(pid())
quic_conn_pid :: option(pid())
}).
-record(retry, {

11
apps/emqx/src/emqx_exclusive_subscription.erl
View File

@ -22,14 +22,11 @@
-logger_header("[exclusive]").
%% Mnesia bootstrap
-export([mnesia/1]).
-export([create_tables/0]).
%% For upgrade
-export([on_add_module/0, on_delete_module/0]).
-boot_mnesia({mnesia, [boot]}).
-copy_mnesia({mnesia, [copy]}).
-export([
check_subscribe/2,
unsubscribe/2,
@ -53,7 +50,7 @@
%% Mnesia bootstrap
%%--------------------------------------------------------------------
mnesia(boot) ->
create_tables() ->
StoreProps = [
{ets, [
{read_concurrency, true},
@ -68,14 +65,14 @@ mnesia(boot) ->
{attributes, record_info(fields, exclusive_subscription)},
{storage_properties, StoreProps}
]),
ok = mria_rlog:wait_for_shards([?EXCLUSIVE_SHARD], infinity).
[?TAB].
%%--------------------------------------------------------------------
%% Upgrade
%%--------------------------------------------------------------------
on_add_module() ->
mnesia(boot).
mria:wait_for_tables(create_tables()).
on_delete_module() ->
clear().

2
apps/emqx/src/emqx_flapping.erl
View File

@ -150,7 +150,7 @@ handle_cast(
),
Now = erlang:system_time(second),
Banned = #banned{
who = {clientid, ClientId},
who = emqx_banned:who(clientid, ClientId),
by = <<"flapping detector">>,
reason = <<"flapping is detected">>,
at = Now,

82
apps/emqx/src/emqx_frame.erl
View File

@ -168,7 +168,7 @@ parse_remaining_len(Rest, Header, Options) ->
parse_remaining_len(_Bin, _Header, _Multiplier, Length, #{max_size := MaxSize}) when
Length > MaxSize
->
?PARSE_ERR(frame_too_large);
?PARSE_ERR(#{cause => frame_too_large, limit => MaxSize, received => Length});
parse_remaining_len(<<>>, Header, Multiplier, Length, Options) ->
{more, {{len, #{hdr => Header, len => {Multiplier, Length}}}, Options}};
%% Match DISCONNECT without payload
@ -189,12 +189,12 @@ parse_remaining_len(
parse_remaining_len(
<<0:8, _Rest/binary>>, _Header = #mqtt_packet_header{type = ?PINGRESP}, 1, 0, _Options
) ->
?PARSE_ERR(#{hint => unexpected_packet, header_type => 'PINGRESP'});
?PARSE_ERR(#{cause => unexpected_packet, header_type => 'PINGRESP'});
%% All other types of messages should not have a zero remaining length.
parse_remaining_len(
<<0:8, _Rest/binary>>, Header, 1, 0, _Options
) ->
?PARSE_ERR(#{hint => zero_remaining_len, header_type => Header#mqtt_packet_header.type});
?PARSE_ERR(#{cause => zero_remaining_len, header_type => Header#mqtt_packet_header.type});
%% Match PUBACK, PUBREC, PUBREL, PUBCOMP, UNSUBACK...
parse_remaining_len(<<0:1, 2:7, Rest/binary>>, Header, 1, 0, Options) ->
parse_frame(Rest, Header, 2, Options);
@ -213,7 +213,7 @@ parse_remaining_len(
) ->
FrameLen = Value + Len * Multiplier,
case FrameLen > MaxSize of
true -> ?PARSE_ERR(frame_too_large);
true -> ?PARSE_ERR(#{cause => frame_too_large, limit => MaxSize, received => FrameLen});
false -> parse_frame(Rest, Header, FrameLen, Options)
end.
@ -267,7 +267,7 @@ packet(Header, Variable, Payload) ->
#mqtt_packet{header = Header, variable = Variable, payload = Payload}.
parse_connect(FrameBin, StrictMode) ->
{ProtoName, Rest} = parse_utf8_string_with_hint(FrameBin, StrictMode, invalid_proto_name),
{ProtoName, Rest} = parse_utf8_string_with_cause(FrameBin, StrictMode, invalid_proto_name),
case ProtoName of
<<"MQTT">> ->
ok;
@ -277,7 +277,7 @@ parse_connect(FrameBin, StrictMode) ->
%% from spec: the server MAY send disconnect with reason code 0x84
%% we chose to close socket because the client is likely not talking MQTT anyway
?PARSE_ERR(#{
hint => invalid_proto_name,
cause => invalid_proto_name,
expected => <<"'MQTT' or 'MQIsdp'">>,
received => ProtoName
})
@ -296,11 +296,12 @@ parse_connect2(
1 -> ?PARSE_ERR(reserved_connect_flag)
end,
{Properties, Rest3} = parse_properties(Rest2, ProtoVer, StrictMode),
{ClientId, Rest4} = parse_utf8_string_with_hint(Rest3, StrictMode, invalid_clientid),
{ClientId, Rest4} = parse_utf8_string_with_cause(Rest3, StrictMode, invalid_clientid),
ConnPacket = #mqtt_packet_connect{
proto_name = ProtoName,
proto_ver = ProtoVer,
%% For bridge mode, non-standard implementation
%% Invented by mosquitto, named 'try_private': https://mosquitto.org/man/mosquitto-conf-5.html
is_bridge = (BridgeTag =:= 8),
clean_start = bool(CleanStart),
will_flag = bool(WillFlag),
@ -314,14 +315,14 @@ parse_connect2(
{Username, Rest6} = parse_optional(
Rest5,
fun(Bin) ->
parse_utf8_string_with_hint(Bin, StrictMode, invalid_username)
parse_utf8_string_with_cause(Bin, StrictMode, invalid_username)
end,
bool(UsernameFlag)
),
{Password, Rest7} = parse_optional(
Rest6,
fun(Bin) ->
parse_utf8_string_with_hint(Bin, StrictMode, invalid_password)
parse_utf8_string_with_cause(Bin, StrictMode, invalid_password)
end,
bool(PasswordFlag)
),
@ -329,10 +330,14 @@ parse_connect2(
<<>> ->
ConnPacket1#mqtt_packet_connect{username = Username, password = Password};
_ ->
?PARSE_ERR(malformed_connect_data)
?PARSE_ERR(#{
cause => malformed_connect,
unexpected_trailing_bytes => size(Rest7)
})
end;
parse_connect2(_ProtoName, _, _) ->
?PARSE_ERR(malformed_connect_header).
parse_connect2(_ProtoName, Bin, _StrictMode) ->
%% sent less than 32 bytes
?PARSE_ERR(#{cause => malformed_connect, header_bytes => Bin}).
parse_packet(
#mqtt_packet_header{type = ?CONNECT},
@ -361,7 +366,7 @@ parse_packet(
Bin,
#{strict_mode := StrictMode, version := Ver}
) ->
{TopicName, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_topic),
{TopicName, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_topic),
{PacketId, Rest1} =
case QoS of
?QOS_0 -> {undefined, Rest};
@ -473,7 +478,7 @@ parse_packet(
{Properties, <<>>} = parse_properties(Rest, ?MQTT_PROTO_V5, StrictMode),
#mqtt_packet_auth{reason_code = ReasonCode, properties = Properties};
parse_packet(Header, _FrameBin, _Options) ->
?PARSE_ERR(#{hint => malformed_packet, header_type => Header#mqtt_packet_header.type}).
?PARSE_ERR(#{cause => malformed_packet, header_type => Header#mqtt_packet_header.type}).
parse_will_message(
Packet = #mqtt_packet_connect{
@ -484,8 +489,8 @@ parse_will_message(
StrictMode
) ->
{Props, Rest} = parse_properties(Bin, Ver, StrictMode),
{Topic, Rest1} = parse_utf8_string_with_hint(Rest, StrictMode, invalid_topic),
{Payload, Rest2} = parse_binary_data(Rest1),
{Topic, Rest1} = parse_utf8_string_with_cause(Rest, StrictMode, invalid_topic),
{Payload, Rest2} = parse_will_payload(Rest1),
{
Packet#mqtt_packet_connect{
will_props = Props,
@ -517,7 +522,7 @@ parse_properties(Bin, ?MQTT_PROTO_V5, StrictMode) ->
{parse_property(PropsBin, #{}, StrictMode), Rest1};
_ ->
?PARSE_ERR(#{
hint => user_property_not_enough_bytes,
cause => user_property_not_enough_bytes,
parsed_key_length => Len,
remaining_bytes_length => byte_size(Rest)
})
@ -530,10 +535,10 @@ parse_property(<<16#01, Val, Bin/binary>>, Props, StrictMode) ->
parse_property(<<16#02, Val:32/big, Bin/binary>>, Props, StrictMode) ->
parse_property(Bin, Props#{'Message-Expiry-Interval' => Val}, StrictMode);
parse_property(<<16#03, Bin/binary>>, Props, StrictMode) ->
{Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_content_type),
{Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_content_type),
parse_property(Rest, Props#{'Content-Type' => Val}, StrictMode);
parse_property(<<16#08, Bin/binary>>, Props, StrictMode) ->
{Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_response_topic),
{Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_response_topic),
parse_property(Rest, Props#{'Response-Topic' => Val}, StrictMode);
parse_property(<<16#09, Len:16/big, Val:Len/binary, Bin/binary>>, Props, StrictMode) ->
parse_property(Bin, Props#{'Correlation-Data' => Val}, StrictMode);
@ -543,12 +548,12 @@ parse_property(<<16#0B, Bin/binary>>, Props, StrictMode) ->
parse_property(<<16#11, Val:32/big, Bin/binary>>, Props, StrictMode) ->
parse_property(Bin, Props#{'Session-Expiry-Interval' => Val}, StrictMode);
parse_property(<<16#12, Bin/binary>>, Props, StrictMode) ->
{Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_assigned_client_id),
{Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_assigned_client_id),
parse_property(Rest, Props#{'Assigned-Client-Identifier' => Val}, StrictMode);
parse_property(<<16#13, Val:16, Bin/binary>>, Props, StrictMode) ->
parse_property(Bin, Props#{'Server-Keep-Alive' => Val}, StrictMode);
parse_property(<<16#15, Bin/binary>>, Props, StrictMode) ->
{Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_authn_method),
{Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_authn_method),
parse_property(Rest, Props#{'Authentication-Method' => Val}, StrictMode);
parse_property(<<16#16, Len:16/big, Val:Len/binary, Bin/binary>>, Props, StrictMode) ->
parse_property(Bin, Props#{'Authentication-Data' => Val}, StrictMode);
@ -559,13 +564,13 @@ parse_property(<<16#18, Val:32, Bin/binary>>, Props, StrictMode) ->
parse_property(<<16#19, Val, Bin/binary>>, Props, StrictMode) ->
parse_property(Bin, Props#{'Request-Response-Information' => Val}, StrictMode);
parse_property(<<16#1A, Bin/binary>>, Props, StrictMode) ->
{Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_response_info),
{Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_response_info),
parse_property(Rest, Props#{'Response-Information' => Val}, StrictMode);
parse_property(<<16#1C, Bin/binary>>, Props, StrictMode) ->
{Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_server_reference),
{Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_server_reference),
parse_property(Rest, Props#{'Server-Reference' => Val}, StrictMode);
parse_property(<<16#1F, Bin/binary>>, Props, StrictMode) ->
{Val, Rest} = parse_utf8_string_with_hint(Bin, StrictMode, invalid_reason_string),
{Val, Rest} = parse_utf8_string_with_cause(Bin, StrictMode, invalid_reason_string),
parse_property(Rest, Props#{'Reason-String' => Val}, StrictMode);
parse_property(<<16#21, Val:16/big, Bin/binary>>, Props, StrictMode) ->
parse_property(Bin, Props#{'Receive-Maximum' => Val}, StrictMode);
@ -634,7 +639,7 @@ parse_utf8_pair(<<LenK:16/big, Rest/binary>>, _StrictMode) when
LenK > byte_size(Rest)
->
?PARSE_ERR(#{
hint => user_property_not_enough_bytes,
cause => user_property_not_enough_bytes,
parsed_key_length => LenK,
remaining_bytes_length => byte_size(Rest)
});
@ -643,7 +648,7 @@ parse_utf8_pair(<<LenK:16/big, _Key:LenK/binary, LenV:16/big, Rest/binary>>, _St
LenV > byte_size(Rest)
->
?PARSE_ERR(#{
hint => malformed_user_property_value,
cause => malformed_user_property_value,
parsed_key_length => LenK,
parsed_value_length => LenV,
remaining_bytes_length => byte_size(Rest)
@ -652,16 +657,16 @@ parse_utf8_pair(Bin, _StrictMode) when
4 > byte_size(Bin)
->
?PARSE_ERR(#{
hint => user_property_not_enough_bytes,
cause => user_property_not_enough_bytes,
total_bytes => byte_size(Bin)
}).
parse_utf8_string_with_hint(Bin, StrictMode, Hint) ->
parse_utf8_string_with_cause(Bin, StrictMode, Cause) ->
try
parse_utf8_string(Bin, StrictMode)
catch
throw:{?FRAME_PARSE_ERROR, Reason} when is_map(Reason) ->
?PARSE_ERR(Reason#{hint => Hint})
?PARSE_ERR(Reason#{cause => Cause})
end.
parse_optional(Bin, F, true) ->
@ -677,7 +682,7 @@ parse_utf8_string(<<Len:16/big, Rest/binary>>, _) when
Len > byte_size(Rest)
->
?PARSE_ERR(#{
hint => malformed_utf8_string,
cause => malformed_utf8_string,
parsed_length => Len,
remaining_bytes_length => byte_size(Rest)
});
@ -686,20 +691,24 @@ parse_utf8_string(Bin, _) when
->
?PARSE_ERR(#{reason => malformed_utf8_string_length}).
parse_binary_data(<<Len:16/big, Data:Len/binary, Rest/binary>>) ->
parse_will_payload(<<Len:16/big, Data:Len/binary, Rest/binary>>) ->
{Data, Rest};
parse_binary_data(<<Len:16/big, Rest/binary>>) when
parse_will_payload(<<Len:16/big, Rest/binary>>) when
Len > byte_size(Rest)
->
?PARSE_ERR(#{
hint => malformed_binary_data,
cause => malformed_will_payload,
parsed_length => Len,
remaining_bytes_length => byte_size(Rest)
remaining_bytes => byte_size(Rest)
});
parse_binary_data(Bin) when
parse_will_payload(Bin) when
2 > byte_size(Bin)
->
?PARSE_ERR(malformed_binary_data_length).
?PARSE_ERR(#{
cause => malformed_will_payload,
length_bytes => size(Bin),
expected_bytes => 2
}).
%%--------------------------------------------------------------------
%% Serialize MQTT Packet
@ -772,6 +781,7 @@ serialize_variable(
proto_name = ProtoName,
proto_ver = ProtoVer,
%% For bridge mode, non-standard implementation
%% Invented by mosquitto, named 'try_private': https://mosquitto.org/man/mosquitto-conf-5.html
is_bridge = IsBridge,
clean_start = CleanStart,
will_flag = WillFlag,

4
apps/emqx/src/emqx_gc.erl
View File

@ -86,11 +86,11 @@ do_run([{K, N} | T], St) ->
end.
%% @doc Info of GC state.
-spec info(maybe(gc_state())) -> maybe(map()).
-spec info(option(gc_state())) -> option(map()).
info(?GCS(St)) -> St.
%% @doc Reset counters to zero.
-spec reset(maybe(gc_state())) -> gc_state().
-spec reset(option(gc_state())) -> gc_state().
reset(?GCS(St)) ->
?GCS(do_reset(St)).

2
apps/emqx/src/emqx_hooks.erl
View File

@ -76,7 +76,7 @@
-record(callback, {
action :: action(),
filter :: maybe(filter()),
filter :: option(filter()),
priority :: integer()
}).

3
apps/emqx/src/emqx_kernel_sup.erl
View File

@ -40,7 +40,8 @@ init([]) ->
child_spec(emqx_authn_authz_metrics_sup, supervisor),
child_spec(emqx_ocsp_cache, worker),
child_spec(emqx_crl_cache, worker),
child_spec(emqx_tls_lib_sup, supervisor)
child_spec(emqx_tls_lib_sup, supervisor),
child_spec(emqx_log_throttler, worker)
]
}}.

151
apps/emqx/src/emqx_log_throttler.erl Normal file
View File

@ -0,0 +1,151 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_log_throttler).
-behaviour(gen_server).
-include("logger.hrl").
-include("types.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-export([start_link/0]).
%% throttler API
-export([allow/2]).
%% gen_server callbacks
-export([
init/1,
handle_call/3,
handle_cast/2,
handle_info/2,
terminate/2,
code_change/3
]).
-define(SEQ_ID(Msg), {?MODULE, Msg}).
-define(NEW_SEQ, atomics:new(1, [{signed, false}])).
-define(GET_SEQ(Msg), persistent_term:get(?SEQ_ID(Msg), undefined)).
-define(RESET_SEQ(SeqRef), atomics:put(SeqRef, 1, 0)).
-define(INC_SEQ(SeqRef), atomics:add(SeqRef, 1, 1)).
-define(GET_DROPPED(SeqRef), atomics:get(SeqRef, 1) - 1).
-define(IS_ALLOWED(SeqRef), atomics:add_get(SeqRef, 1, 1) =:= 1).
-define(NEW_THROTTLE(Msg, SeqRef), persistent_term:put(?SEQ_ID(Msg), SeqRef)).
-define(MSGS_LIST, emqx:get_config([log, throttling, msgs], [])).
-define(TIME_WINDOW_MS, timer:seconds(emqx:get_config([log, throttling, time_window], 60))).
-spec allow(logger:level(), atom()) -> boolean().
allow(debug, _Msg) ->
true;
allow(_Level, Msg) when is_atom(Msg) ->
Seq = persistent_term:get(?SEQ_ID(Msg), undefined),
case Seq of
undefined ->
%% This is either a race condition (emqx_log_throttler is not started yet)
%% or a developer mistake (msg used in ?SLOG_THROTTLE/2,3 macro is
%% not added to the default value of `log.throttling.msgs`.
?SLOG(info, #{
msg => "missing_log_throttle_sequence",
throttled_msg => Msg
}),
true;
SeqRef ->
?IS_ALLOWED(SeqRef)
end.
-spec start_link() -> startlink_ret().
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
%%--------------------------------------------------------------------
%% gen_server callbacks
%%--------------------------------------------------------------------
init([]) ->
ok = lists:foreach(fun(Msg) -> ?NEW_THROTTLE(Msg, ?NEW_SEQ) end, ?MSGS_LIST),
CurrentPeriodMs = ?TIME_WINDOW_MS,
TimerRef = schedule_refresh(CurrentPeriodMs),
{ok, #{timer_ref => TimerRef, current_period_ms => CurrentPeriodMs}}.
handle_call(Req, _From, State) ->
?SLOG(error, #{msg => "unexpected_call", call => Req}),
{reply, ignored, State}.
handle_cast(Msg, State) ->
?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
{noreply, State}.
handle_info(refresh, #{current_period_ms := PeriodMs} = State) ->
Msgs = ?MSGS_LIST,
DroppedStats = lists:foldl(
fun(Msg, Acc) ->
case ?GET_SEQ(Msg) of
%% Should not happen, unless the static ids list is updated at run-time.
undefined ->
?NEW_THROTTLE(Msg, ?NEW_SEQ),
?tp(log_throttler_new_msg, #{throttled_msg => Msg}),
Acc;
SeqRef ->
Dropped = ?GET_DROPPED(SeqRef),
ok = ?RESET_SEQ(SeqRef),
?tp(log_throttler_dropped, #{dropped_count => Dropped, throttled_msg => Msg}),
maybe_add_dropped(Msg, Dropped, Acc)
end
end,
#{},
Msgs
),
maybe_log_dropped(DroppedStats, PeriodMs),
NewPeriodMs = ?TIME_WINDOW_MS,
State1 = State#{
timer_ref => schedule_refresh(NewPeriodMs),
current_period_ms => NewPeriodMs
},
{noreply, State1};
handle_info(Info, State) ->
?SLOG(error, #{msg => "unxpected_info", info => Info}),
{noreply, State}.
terminate(_Reason, _State) ->
ok.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
%%--------------------------------------------------------------------
%% internal functions
%%--------------------------------------------------------------------
maybe_add_dropped(Msg, Dropped, DroppedAcc) when Dropped > 0 ->
DroppedAcc#{Msg => Dropped};
maybe_add_dropped(_Msg, _Dropped, DroppedAcc) ->
DroppedAcc.
maybe_log_dropped(DroppedStats, PeriodMs) when map_size(DroppedStats) > 0 ->
?SLOG(warning, #{
msg => "log_events_throttled_during_last_period",
dropped => DroppedStats,
period => emqx_utils_calendar:human_readable_duration_string(PeriodMs)
});
maybe_log_dropped(_DroppedStats, _PeriodMs) ->
ok.
schedule_refresh(PeriodMs) ->
?tp(log_throttler_sched_refresh, #{new_period_ms => PeriodMs}),
erlang:send_after(PeriodMs, ?MODULE, refresh).

12
apps/emqx/src/emqx_maybe.erl
View File

@ -23,30 +23,30 @@
-export([define/2]).
-export([apply/2]).
-type t(T) :: maybe(T).
-type t(T) :: option(T).
-export_type([t/1]).
-spec to_list(maybe(A)) -> [A].
-spec to_list(option(A)) -> [A].
to_list(undefined) ->
[];
to_list(Term) ->
[Term].
-spec from_list([A]) -> maybe(A).
-spec from_list([A]) -> option(A).
from_list([]) ->
undefined;
from_list([Term]) ->
Term.
-spec define(maybe(A), B) -> A | B.
-spec define(option(A), B) -> A | B.
define(undefined, Term) ->
Term;
define(Term, _) ->
Term.
%% @doc Apply a function to a maybe argument.
-spec apply(fun((A) -> B), maybe(A)) ->
maybe(B).
-spec apply(fun((A) -> B), option(A)) ->
option(B).
apply(_Fun, undefined) ->
undefined;
apply(Fun, Term) when is_function(Fun) ->

28
apps/emqx/src/emqx_message.erl
View File

@ -65,7 +65,7 @@
]).
-export([
is_expired/1,
is_expired/2,
update_expiry/1,
timestamp_now/0
]).
@ -186,7 +186,7 @@ estimate_size(#message{topic = Topic, payload = Payload}) ->
TopicLengthSize = 2,
FixedHeaderSize + VarLenSize + TopicLengthSize + TopicSize + PacketIdSize + PayloadSize.
-spec id(emqx_types:message()) -> maybe(binary()).
-spec id(emqx_types:message()) -> option(binary()).
id(#message{id = Id}) -> Id.
-spec qos(emqx_types:message()) -> emqx_types:qos().
@ -229,7 +229,7 @@ get_flag(Flag, Msg) ->
get_flag(Flag, #message{flags = Flags}, Default) ->
maps:get(Flag, Flags, Default).
-spec get_flags(emqx_types:message()) -> maybe(map()).
-spec get_flags(emqx_types:message()) -> option(map()).
get_flags(#message{flags = Flags}) -> Flags.
-spec set_flag(emqx_types:flag(), emqx_types:message()) -> emqx_types:message().
@ -252,7 +252,7 @@ unset_flag(Flag, Msg = #message{flags = Flags}) ->
set_headers(New, Msg = #message{headers = Old}) when is_map(New) ->
Msg#message{headers = maps:merge(Old, New)}.
-spec get_headers(emqx_types:message()) -> maybe(map()).
-spec get_headers(emqx_types:message()) -> option(map()).
get_headers(Msg) -> Msg#message.headers.
-spec get_header(term(), emqx_types:message()) -> term().
@ -273,14 +273,20 @@ remove_header(Hdr, Msg = #message{headers = Headers}) ->
false -> Msg
end.
-spec is_expired(emqx_types:message()) -> boolean().
is_expired(#message{
headers = #{properties := #{'Message-Expiry-Interval' := Interval}},
timestamp = CreatedAt
}) ->
-spec is_expired(emqx_types:message(), atom()) -> boolean().
is_expired(
#message{
headers = #{properties := #{'Message-Expiry-Interval' := Interval}},
timestamp = CreatedAt
},
_
) ->
elapsed(CreatedAt) > timer:seconds(Interval);
is_expired(_Msg) ->
false.
is_expired(#message{timestamp = CreatedAt}, Zone) ->
case emqx_config:get_zone_conf(Zone, [mqtt, message_expiry_interval], infinity) of
infinity -> false;
Interval -> elapsed(CreatedAt) > Interval
end.
-spec update_expiry(emqx_types:message()) -> emqx_types:message().
update_expiry(

8
apps/emqx/src/emqx_mountpoint.erl
View File

@ -32,7 +32,7 @@
-type mountpoint() :: binary().
-spec mount(maybe(mountpoint()), Any) -> Any when
-spec mount(option(mountpoint()), Any) -> Any when
Any ::
emqx_types:topic()
| emqx_types:share()
@ -47,7 +47,7 @@ mount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) ->
mount(MountPoint, TopicFilters) when is_list(TopicFilters) ->
[{prefix_maybe_share(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters].
-spec prefix_maybe_share(maybe(mountpoint()), Any) -> Any when
-spec prefix_maybe_share(option(mountpoint()), Any) -> Any when
Any ::
emqx_types:topic()
| emqx_types:share().
@ -60,7 +60,7 @@ prefix_maybe_share(MountPoint, #share{group = Group, topic = Topic}) when
->
#share{group = Group, topic = prefix_maybe_share(MountPoint, Topic)}.
-spec unmount(maybe(mountpoint()), Any) -> Any when
-spec unmount(option(mountpoint()), Any) -> Any when
Any ::
emqx_types:topic()
| emqx_types:share()
@ -84,7 +84,7 @@ unmount_maybe_share(MountPoint, TopicFilter = #share{topic = Topic}) when
->
TopicFilter#share{topic = unmount_maybe_share(MountPoint, Topic)}.
-spec replvar(maybe(mountpoint()), map()) -> maybe(mountpoint()).
-spec replvar(option(mountpoint()), map()) -> option(mountpoint()).
replvar(undefined, _Vars) ->
undefined;
replvar(MountPoint, Vars) ->

2
apps/emqx/src/emqx_mqueue.erl
View File

@ -189,7 +189,7 @@ stats(#mqueue{max_len = MaxLen, dropped = Dropped} = MQ) ->
[{len, len(MQ)}, {max_len, MaxLen}, {dropped, Dropped}].
%% @doc Enqueue a message.
-spec in(message(), mqueue()) -> {maybe(message()), mqueue()}.
-spec in(message(), mqueue()) -> {option(message()), mqueue()}.
in(Msg = #message{qos = ?QOS_0}, MQ = #mqueue{store_qos0 = false}) ->
{_Dropped = Msg, MQ};
in(

4
apps/emqx/src/emqx_packet.erl
View File

@ -493,8 +493,8 @@ format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, Pa
"" -> [HeaderIO, ")"];
VarIO -> [HeaderIO, ", ", VarIO, ")"]
end;
%% receive a frame error packet, such as {frame_error,frame_too_large} or
%% {frame_error,#{expected => <<"'MQTT' or 'MQIsdp'">>,hint => invalid_proto_name,received => <<"bad_name">>}}
%% receive a frame error packet, such as {frame_error,#{cause := frame_too_large}} or
%% {frame_error,#{expected => <<"'MQTT' or 'MQIsdp'">>,cause => invalid_proto_name,received => <<"bad_name">>}}
format(FrameError, _PayloadEncode) ->
lists:flatten(io_lib:format("~tp", [FrameError])).

2
apps/emqx/src/emqx_pd.erl
View File

@ -48,7 +48,7 @@ get_counter(Key) ->
Cnt -> Cnt
end.
-spec inc_counter(key(), number()) -> maybe(number()).
-spec inc_counter(key(), number()) -> option(number()).
inc_counter(Key, Inc) ->
put(Key, get_counter(Key) + Inc).

10
apps/emqx/src/emqx_persistent_message.erl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2021-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -61,7 +61,11 @@ force_ds() ->
emqx_config:get([session_persistence, force_persistence]).
storage_backend(#{
builtin := #{enable := true, n_shards := NShards, replication_factor := ReplicationFactor}
builtin := #{
enable := true,
n_shards := NShards,
replication_factor := ReplicationFactor
}
}) ->
#{
backend => builtin,
@ -93,7 +97,7 @@ needs_persistence(Msg) ->
-spec store_message(emqx_types:message()) -> emqx_ds:store_batch_result().
store_message(Msg) ->
emqx_ds:store_batch(?PERSISTENT_MESSAGE_DB, [Msg]).
emqx_ds:store_batch(?PERSISTENT_MESSAGE_DB, [Msg], #{sync => false}).
has_subscribers(#message{topic = Topic}) ->
emqx_persistent_session_ds_router:has_any_route(Topic).

157
apps/emqx/src/emqx_persistent_message_ds_gc_worker.erl Normal file
View File

@ -0,0 +1,157 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_persistent_message_ds_gc_worker).
-behaviour(gen_server).
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include_lib("stdlib/include/qlc.hrl").
-include_lib("stdlib/include/ms_transform.hrl").
-include("emqx_persistent_session_ds.hrl").
%% API
-export([
start_link/0,
gc/0
]).
%% `gen_server' API
-export([
init/1,
handle_call/3,
handle_cast/2,
handle_info/2
]).
%% call/cast/info records
-record(gc, {}).
%%--------------------------------------------------------------------------------
%% API
%%--------------------------------------------------------------------------------
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
%% For testing or manual ops
gc() ->
gen_server:call(?MODULE, #gc{}, infinity).
%%--------------------------------------------------------------------------------
%% `gen_server' API
%%--------------------------------------------------------------------------------
init(_Opts) ->
ensure_gc_timer(),
State = #{},
{ok, State}.
handle_call(#gc{}, _From, State) ->
maybe_gc(),
{reply, ok, State};
handle_call(_Call, _From, State) ->
{reply, error, State}.
handle_cast(_Cast, State) ->
{noreply, State}.
handle_info(#gc{}, State) ->
try_gc(),
ensure_gc_timer(),
{noreply, State};
handle_info(_Info, State) ->
{noreply, State}.
%%--------------------------------------------------------------------------------
%% Internal fns
%%--------------------------------------------------------------------------------
ensure_gc_timer() ->
Timeout = emqx_config:get([session_persistence, message_retention_period]),
_ = erlang:send_after(Timeout, self(), #gc{}),
ok.
try_gc() ->
%% Only cores should run GC.
CoreNodes = mria_membership:running_core_nodelist(),
Res = global:trans(
{?MODULE, self()},
fun maybe_gc/0,
CoreNodes,
%% Note: we set retries to 1 here because, in rare occasions, GC might start at the
%% same time in more than one node, and each one will abort the other. By allowing
%% one retry, at least one node will (hopefully) get to enter the transaction and
%% the other will abort. If GC runs too fast, both nodes might run in sequence.
%% But, in that case, GC is clearly not too costly, and that shouldn't be a problem,
%% resource-wise.
_Retries = 1
),
case Res of
aborted ->
?tp(ds_message_gc_lock_taken, #{}),
ok;
ok ->
ok
end.
now_ms() ->
erlang:system_time(millisecond).
maybe_gc() ->
AllGens = emqx_ds:list_generations_with_lifetimes(?PERSISTENT_MESSAGE_DB),
NowMS = now_ms(),
RetentionPeriod = emqx_config:get([session_persistence, message_retention_period]),
TimeThreshold = NowMS - RetentionPeriod,
maybe_create_new_generation(AllGens, TimeThreshold),
?tp_span(
ps_message_gc,
#{},
begin
ExpiredGens =
maps:filter(
fun(_GenId, #{until := Until}) ->
is_number(Until) andalso Until =< TimeThreshold
end,
AllGens
),
ExpiredGenIds = maps:keys(ExpiredGens),
lists:foreach(
fun(GenId) ->
ok = emqx_ds:drop_generation(?PERSISTENT_MESSAGE_DB, GenId),
?tp(message_gc_generation_dropped, #{gen_id => GenId})
end,
ExpiredGenIds
)
end
).
maybe_create_new_generation(AllGens, TimeThreshold) ->
NeedNewGen =
lists:all(
fun({_GenId, #{created_at := CreatedAt}}) ->
CreatedAt =< TimeThreshold
end,
maps:to_list(AllGens)
),
case NeedNewGen of
false ->
?tp(ps_message_gc_too_early, #{}),
ok;
true ->
ok = emqx_ds:add_generation(?PERSISTENT_MESSAGE_DB),
?tp(ps_message_gc_added_gen, #{})
end.

795
apps/emqx/src/emqx_persistent_message_ds_replayer.erl
View File

@ -1,795 +0,0 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc This module implements the routines for replaying streams of
%% messages.
-module(emqx_persistent_message_ds_replayer).
%% API:
-export([new/0, open/1, next_packet_id/1, n_inflight/1]).
-export([poll/4, replay/2, commit_offset/4]).
-export([seqno_to_packet_id/1, packet_id_to_seqno/2]).
-export([committed_until/2]).
%% internal exports:
-export([]).
-export_type([inflight/0, seqno/0]).
-include_lib("emqx/include/logger.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("emqx_utils/include/emqx_message.hrl").
-include("emqx_persistent_session_ds.hrl").
-ifdef(TEST).
-include_lib("proper/include/proper.hrl").
-include_lib("eunit/include/eunit.hrl").
-endif.
-define(EPOCH_SIZE, 16#10000).
-define(ACK, 0).
-define(COMP, 1).
-define(TRACK_FLAG(WHICH), (1 bsl WHICH)).
-define(TRACK_FLAGS_ALL, ?TRACK_FLAG(?ACK) bor ?TRACK_FLAG(?COMP)).
-define(TRACK_FLAGS_NONE, 0).
%%================================================================================
%% Type declarations
%%================================================================================
%% Note: sequence numbers are monotonic; they don't wrap around:
-type seqno() :: non_neg_integer().
-type track() :: ack | comp.
-type commit_type() :: rec.
-record(inflight, {
next_seqno = 1 :: seqno(),
commits = #{ack => 1, comp => 1, rec => 1} :: #{track() | commit_type() => seqno()},
%% Ranges are sorted in ascending order of their sequence numbers.
offset_ranges = [] :: [ds_pubrange()]
}).
-opaque inflight() :: #inflight{}.
-type message() :: emqx_types:message().
-type replies() :: [emqx_session:reply()].
-type preproc_fun() :: fun((message()) -> message() | [message()]).
%%================================================================================
%% API funcions
%%================================================================================
-spec new() -> inflight().
new() ->
#inflight{}.
-spec open(emqx_persistent_session_ds:id()) -> inflight().
open(SessionId) ->
{Ranges, RecUntil} = ro_transaction(
fun() -> {get_ranges(SessionId), get_committed_offset(SessionId, rec)} end
),
{Commits, NextSeqno} = compute_inflight_range(Ranges),
#inflight{
commits = Commits#{rec => RecUntil},
next_seqno = NextSeqno,
offset_ranges = Ranges
}.
-spec next_packet_id(inflight()) -> {emqx_types:packet_id(), inflight()}.
next_packet_id(Inflight0 = #inflight{next_seqno = LastSeqno}) ->
Inflight = Inflight0#inflight{next_seqno = next_seqno(LastSeqno)},
{seqno_to_packet_id(LastSeqno), Inflight}.
-spec n_inflight(inflight()) -> non_neg_integer().
n_inflight(#inflight{offset_ranges = Ranges}) ->
%% TODO
%% This is not very efficient. Instead, we can take the maximum of
%% `range_size(AckedUntil, NextSeqno)` and `range_size(CompUntil, NextSeqno)`.
%% This won't be exact number but a pessimistic estimate, but this way we
%% will penalize clients that PUBACK QoS 1 messages but don't PUBCOMP QoS 2
%% messages for some reason. For that to work, we need to additionally track
%% actual `AckedUntil` / `CompUntil` during `commit_offset/4`.
lists:foldl(
fun
(#ds_pubrange{type = ?T_CHECKPOINT}, N) ->
N;
(#ds_pubrange{type = ?T_INFLIGHT} = Range, N) ->
N + range_size(Range)
end,
0,
Ranges
).
-spec replay(preproc_fun(), inflight()) -> {emqx_session:replies(), inflight()}.
replay(PreprocFunFun, Inflight0 = #inflight{offset_ranges = Ranges0, commits = Commits}) ->
{Ranges, Replies} = lists:mapfoldr(
fun(Range, Acc) ->
replay_range(PreprocFunFun, Commits, Range, Acc)
end,
[],
Ranges0
),
Inflight = Inflight0#inflight{offset_ranges = Ranges},
{Replies, Inflight}.
-spec commit_offset(emqx_persistent_session_ds:id(), Offset, emqx_types:packet_id(), inflight()) ->
{_IsValidOffset :: boolean(), inflight()}
when
Offset :: track() | commit_type().
commit_offset(
SessionId,
Track,
PacketId,
Inflight0 = #inflight{commits = Commits}
) when Track == ack orelse Track == comp ->
case validate_commit(Track, PacketId, Inflight0) of
CommitUntil when is_integer(CommitUntil) ->
%% TODO
%% We do not preserve `CommitUntil` in the database. Instead, we discard
%% fully acked ranges from the database. In effect, this means that the
%% most recent `CommitUntil` the client has sent may be lost in case of a
%% crash or client loss.
Inflight1 = Inflight0#inflight{commits = Commits#{Track := CommitUntil}},
Inflight = discard_committed(SessionId, Inflight1),
{true, Inflight};
false ->
{false, Inflight0}
end;
commit_offset(
SessionId,
CommitType = rec,
PacketId,
Inflight0 = #inflight{commits = Commits}
) ->
case validate_commit(CommitType, PacketId, Inflight0) of
CommitUntil when is_integer(CommitUntil) ->
update_committed_offset(SessionId, CommitType, CommitUntil),
Inflight = Inflight0#inflight{commits = Commits#{CommitType := CommitUntil}},
{true, Inflight};
false ->
{false, Inflight0}
end.
-spec poll(preproc_fun(), emqx_persistent_session_ds:id(), inflight(), pos_integer()) ->
{emqx_session:replies(), inflight()}.
poll(PreprocFun, SessionId, Inflight0, WindowSize) when WindowSize > 0, WindowSize < ?EPOCH_SIZE ->
MinBatchSize = emqx_config:get([session_persistence, min_batch_size]),
FetchThreshold = min(MinBatchSize, ceil(WindowSize / 2)),
FreeSpace = WindowSize - n_inflight(Inflight0),
case FreeSpace >= FetchThreshold of
false ->
%% TODO: this branch is meant to avoid fetching data from
%% the DB in chunks that are too small. However, this
%% logic is not exactly good for the latency. Can the
%% client get stuck even?
{[], Inflight0};
true ->
%% TODO: Wrap this in `mria:async_dirty/2`?
Checkpoints = find_checkpoints(Inflight0#inflight.offset_ranges),
StreamGroups = group_streams(get_streams(SessionId)),
{Publihes, Inflight} =
fetch(PreprocFun, SessionId, Inflight0, Checkpoints, StreamGroups, FreeSpace, []),
%% Discard now irrelevant QoS0-only ranges, if any.
{Publihes, discard_committed(SessionId, Inflight)}
end.
%% Which seqno this track is committed until.
%% "Until" means this is first seqno that is _not yet committed_ for this track.
-spec committed_until(track() | commit_type(), inflight()) -> seqno().
committed_until(Track, #inflight{commits = Commits}) ->
maps:get(Track, Commits).
-spec seqno_to_packet_id(seqno()) -> emqx_types:packet_id() | 0.
seqno_to_packet_id(Seqno) ->
Seqno rem ?EPOCH_SIZE.
%% Reconstruct session counter by adding most significant bits from
%% the current counter to the packet id.
-spec packet_id_to_seqno(emqx_types:packet_id(), inflight()) -> seqno().
packet_id_to_seqno(PacketId, #inflight{next_seqno = NextSeqno}) ->
packet_id_to_seqno_(NextSeqno, PacketId).
%%================================================================================
%% Internal exports
%%================================================================================
%%================================================================================
%% Internal functions
%%================================================================================
compute_inflight_range([]) ->
{#{ack => 1, comp => 1}, 1};
compute_inflight_range(Ranges) ->
_RangeLast = #ds_pubrange{until = LastSeqno} = lists:last(Ranges),
AckedUntil = find_committed_until(ack, Ranges),
CompUntil = find_committed_until(comp, Ranges),
Commits = #{
ack => emqx_maybe:define(AckedUntil, LastSeqno),
comp => emqx_maybe:define(CompUntil, LastSeqno)
},
{Commits, LastSeqno}.
find_committed_until(Track, Ranges) ->
RangesUncommitted = lists:dropwhile(
fun(Range) ->
case Range of
#ds_pubrange{type = ?T_CHECKPOINT} ->
true;
#ds_pubrange{type = ?T_INFLIGHT, tracks = Tracks} ->
not has_track(Track, Tracks)
end
end,
Ranges
),
case RangesUncommitted of
[#ds_pubrange{id = {_, CommittedUntil, _StreamRef}} | _] ->
CommittedUntil;
[] ->
undefined
end.
-spec get_ranges(emqx_persistent_session_ds:id()) -> [ds_pubrange()].
get_ranges(SessionId) ->
Pat = erlang:make_tuple(
record_info(size, ds_pubrange),
'_',
[{1, ds_pubrange}, {#ds_pubrange.id, {SessionId, '_', '_'}}]
),
mnesia:match_object(?SESSION_PUBRANGE_TAB, Pat, read).
fetch(PreprocFun, SessionId, Inflight0, CPs, Groups, N, Acc) when N > 0, Groups =/= [] ->
#inflight{next_seqno = FirstSeqno, offset_ranges = Ranges} = Inflight0,
{Stream, Groups2} = get_the_first_stream(Groups),
case get_next_n_messages_from_stream(Stream, CPs, N) of
[] ->
fetch(PreprocFun, SessionId, Inflight0, CPs, Groups2, N, Acc);
{ItBegin, ItEnd, Messages} ->
%% We need to preserve the iterator pointing to the beginning of the
%% range, so that we can replay it if needed.
{Publishes, UntilSeqno} = publish_fetch(PreprocFun, FirstSeqno, Messages),
Size = range_size(FirstSeqno, UntilSeqno),
Range0 = #ds_pubrange{
id = {SessionId, FirstSeqno, Stream#ds_stream.ref},
type = ?T_INFLIGHT,
tracks = compute_pub_tracks(Publishes),
until = UntilSeqno,
iterator = ItBegin
},
ok = preserve_range(Range0),
%% ...Yet we need to keep the iterator pointing past the end of the
%% range, so that we can pick up where we left off: it will become
%% `ItBegin` of the next range for this stream.
Range = keep_next_iterator(ItEnd, Range0),
Inflight = Inflight0#inflight{
next_seqno = UntilSeqno,
offset_ranges = Ranges ++ [Range]
},
fetch(PreprocFun, SessionId, Inflight, CPs, Groups2, N - Size, [Publishes | Acc])
end;
fetch(_ReplyFun, _SessionId, Inflight, _CPs, _Groups, _N, Acc) ->
Publishes = lists:append(lists:reverse(Acc)),
{Publishes, Inflight}.
discard_committed(
SessionId,
Inflight0 = #inflight{commits = Commits, offset_ranges = Ranges0}
) ->
%% TODO: This could be kept and incrementally updated in the inflight state.
Checkpoints = find_checkpoints(Ranges0),
%% TODO: Wrap this in `mria:async_dirty/2`?
Ranges = discard_committed_ranges(SessionId, Commits, Checkpoints, Ranges0),
Inflight0#inflight{offset_ranges = Ranges}.
find_checkpoints(Ranges) ->
lists:foldl(
fun(#ds_pubrange{id = {_SessionId, _, StreamRef}} = Range, Acc) ->
%% For each stream, remember the last range over this stream.
Acc#{StreamRef => Range}
end,
#{},
Ranges
).
discard_committed_ranges(
SessionId,
Commits,
Checkpoints,
Ranges = [Range = #ds_pubrange{id = {_SessionId, _, StreamRef}} | Rest]
) ->
case discard_committed_range(Commits, Range) of
discard ->
%% This range has been fully committed.
%% Either discard it completely, or preserve the iterator for the next range
%% over this stream (i.e. a checkpoint).
RangeKept =
case maps:get(StreamRef, Checkpoints) of
Range ->
[checkpoint_range(Range)];
_Previous ->
discard_range(Range),
[]
end,
%% Since we're (intentionally) not using transactions here, it's important to
%% issue database writes in the same order in which ranges are stored: from
%% the oldest to the newest. This is also why we need to compute which ranges
%% should become checkpoints before we start writing anything.
RangeKept ++ discard_committed_ranges(SessionId, Commits, Checkpoints, Rest);
keep ->
%% This range has not been fully committed.
[Range | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)];
keep_all ->
%% The rest of ranges (if any) still have uncommitted messages.
Ranges;
TracksLeft ->
%% Only some track has been committed.
%% Preserve the uncommitted tracks in the database.
RangeKept = Range#ds_pubrange{tracks = TracksLeft},
preserve_range(restore_first_iterator(RangeKept)),
[RangeKept | discard_committed_ranges(SessionId, Commits, Checkpoints, Rest)]
end;
discard_committed_ranges(_SessionId, _Commits, _Checkpoints, []) ->
[].
discard_committed_range(_Commits, #ds_pubrange{type = ?T_CHECKPOINT}) ->
discard;
discard_committed_range(
#{ack := AckedUntil, comp := CompUntil},
#ds_pubrange{until = Until}
) when Until > AckedUntil andalso Until > CompUntil ->
keep_all;
discard_committed_range(Commits, #ds_pubrange{until = Until, tracks = Tracks}) ->
case discard_tracks(Commits, Until, Tracks) of
0 ->
discard;
Tracks ->
keep;
TracksLeft ->
TracksLeft
end.
discard_tracks(#{ack := AckedUntil, comp := CompUntil}, Until, Tracks) ->
TAck =
case Until > AckedUntil of
true -> ?TRACK_FLAG(?ACK) band Tracks;
false -> 0
end,
TComp =
case Until > CompUntil of
true -> ?TRACK_FLAG(?COMP) band Tracks;
false -> 0
end,
TAck bor TComp.
replay_range(
PreprocFun,
Commits,
Range0 = #ds_pubrange{
type = ?T_INFLIGHT, id = {_, First, _StreamRef}, until = Until, iterator = It
},
Acc
) ->
Size = range_size(First, Until),
{ok, ItNext, MessagesUnacked} = emqx_ds:next(?PERSISTENT_MESSAGE_DB, It, Size),
%% Asserting that range is consistent with the message storage state.
{Replies, Until} = publish_replay(PreprocFun, Commits, First, MessagesUnacked),
%% Again, we need to keep the iterator pointing past the end of the
%% range, so that we can pick up where we left off.
Range = keep_next_iterator(ItNext, Range0),
{Range, Replies ++ Acc};
replay_range(_PreprocFun, _Commits, Range0 = #ds_pubrange{type = ?T_CHECKPOINT}, Acc) ->
{Range0, Acc}.
validate_commit(
Track,
PacketId,
Inflight = #inflight{commits = Commits, next_seqno = NextSeqno}
) ->
Seqno = packet_id_to_seqno_(NextSeqno, PacketId),
CommittedUntil = maps:get(Track, Commits),
CommitNext = get_commit_next(Track, Inflight),
case Seqno >= CommittedUntil andalso Seqno < CommitNext of
true ->
next_seqno(Seqno);
false ->
?SLOG(warning, #{
msg => "out-of-order_commit",
track => Track,
packet_id => PacketId,
commit_seqno => Seqno,
committed_until => CommittedUntil,
commit_next => CommitNext
}),
false
end.
get_commit_next(ack, #inflight{next_seqno = NextSeqno}) ->
NextSeqno;
get_commit_next(rec, #inflight{next_seqno = NextSeqno}) ->
NextSeqno;
get_commit_next(comp, #inflight{commits = Commits}) ->
maps:get(rec, Commits).
publish_fetch(PreprocFun, FirstSeqno, Messages) ->
flatmapfoldl(
fun({_DSKey, MessageIn}, Acc) ->
Message = PreprocFun(MessageIn),
publish_fetch(Message, Acc)
end,
FirstSeqno,
Messages
).
publish_fetch(#message{qos = ?QOS_0} = Message, Seqno) ->
{{undefined, Message}, Seqno};
publish_fetch(#message{} = Message, Seqno) ->
PacketId = seqno_to_packet_id(Seqno),
{{PacketId, Message}, next_seqno(Seqno)};
publish_fetch(Messages, Seqno) ->
flatmapfoldl(fun publish_fetch/2, Seqno, Messages).
publish_replay(PreprocFun, Commits, FirstSeqno, Messages) ->
#{ack := AckedUntil, comp := CompUntil, rec := RecUntil} = Commits,
flatmapfoldl(
fun({_DSKey, MessageIn}, Acc) ->
Message = PreprocFun(MessageIn),
publish_replay(Message, AckedUntil, CompUntil, RecUntil, Acc)
end,
FirstSeqno,
Messages
).
publish_replay(#message{qos = ?QOS_0}, _, _, _, Seqno) ->
%% QoS 0 (at most once) messages should not be replayed.
{[], Seqno};
publish_replay(#message{qos = Qos} = Message, AckedUntil, CompUntil, RecUntil, Seqno) ->
case Qos of
?QOS_1 when Seqno < AckedUntil ->
%% This message has already been acked, so we can skip it.
%% We still need to advance seqno, because previously we assigned this message
%% a unique Packet Id.
{[], next_seqno(Seqno)};
?QOS_2 when Seqno < CompUntil ->
%% This message's flow has already been fully completed, so we can skip it.
%% We still need to advance seqno, because previously we assigned this message
%% a unique Packet Id.
{[], next_seqno(Seqno)};
?QOS_2 when Seqno < RecUntil ->
%% This message's flow has been partially completed, we need to resend a PUBREL.
PacketId = seqno_to_packet_id(Seqno),
Pub = {pubrel, PacketId},
{Pub, next_seqno(Seqno)};
_ ->
%% This message flow hasn't been acked and/or received, we need to resend it.
PacketId = seqno_to_packet_id(Seqno),
Pub = {PacketId, emqx_message:set_flag(dup, true, Message)},
{Pub, next_seqno(Seqno)}
end;
publish_replay([], _, _, _, Seqno) ->
{[], Seqno};
publish_replay(Messages, AckedUntil, CompUntil, RecUntil, Seqno) ->
flatmapfoldl(
fun(Message, Acc) ->
publish_replay(Message, AckedUntil, CompUntil, RecUntil, Acc)
end,
Seqno,
Messages
).
-spec compute_pub_tracks(replies()) -> non_neg_integer().
compute_pub_tracks(Pubs) ->
compute_pub_tracks(Pubs, ?TRACK_FLAGS_NONE).
compute_pub_tracks(_Pubs, Tracks = ?TRACK_FLAGS_ALL) ->
Tracks;
compute_pub_tracks([Pub | Rest], Tracks) ->
Track =
case Pub of
{_PacketId, #message{qos = ?QOS_1}} -> ?TRACK_FLAG(?ACK);
{_PacketId, #message{qos = ?QOS_2}} -> ?TRACK_FLAG(?COMP);
{pubrel, _PacketId} -> ?TRACK_FLAG(?COMP);
_ -> ?TRACK_FLAGS_NONE
end,
compute_pub_tracks(Rest, Track bor Tracks);
compute_pub_tracks([], Tracks) ->
Tracks.
keep_next_iterator(ItNext, Range = #ds_pubrange{iterator = ItFirst, misc = Misc}) ->
Range#ds_pubrange{
iterator = ItNext,
%% We need to keep the first iterator around, in case we need to preserve
%% this range again, updating still uncommitted tracks it's part of.
misc = Misc#{iterator_first => ItFirst}
}.
restore_first_iterator(Range = #ds_pubrange{misc = Misc = #{iterator_first := ItFirst}}) ->
Range#ds_pubrange{
iterator = ItFirst,
misc = maps:remove(iterator_first, Misc)
}.
-spec preserve_range(ds_pubrange()) -> ok.
preserve_range(Range = #ds_pubrange{type = ?T_INFLIGHT}) ->
mria:dirty_write(?SESSION_PUBRANGE_TAB, Range).
has_track(ack, Tracks) ->
(?TRACK_FLAG(?ACK) band Tracks) > 0;
has_track(comp, Tracks) ->
(?TRACK_FLAG(?COMP) band Tracks) > 0.
-spec discard_range(ds_pubrange()) -> ok.
discard_range(#ds_pubrange{id = RangeId}) ->
mria:dirty_delete(?SESSION_PUBRANGE_TAB, RangeId).
-spec checkpoint_range(ds_pubrange()) -> ds_pubrange().
checkpoint_range(Range0 = #ds_pubrange{type = ?T_INFLIGHT}) ->
Range = Range0#ds_pubrange{type = ?T_CHECKPOINT, misc = #{}},
ok = mria:dirty_write(?SESSION_PUBRANGE_TAB, Range),
Range;
checkpoint_range(Range = #ds_pubrange{type = ?T_CHECKPOINT}) ->
%% This range should have been checkpointed already.
Range.
get_last_iterator(Stream = #ds_stream{ref = StreamRef}, Checkpoints) ->
case maps:get(StreamRef, Checkpoints, none) of
none ->
Stream#ds_stream.beginning;
#ds_pubrange{iterator = ItNext} ->
ItNext
end.
-spec get_streams(emqx_persistent_session_ds:id()) -> [ds_stream()].
get_streams(SessionId) ->
mnesia:dirty_read(?SESSION_STREAM_TAB, SessionId).
-spec get_committed_offset(emqx_persistent_session_ds:id(), _Name) -> seqno().
get_committed_offset(SessionId, Name) ->
case mnesia:read(?SESSION_COMMITTED_OFFSET_TAB, {SessionId, Name}) of
[] ->
1;
[#ds_committed_offset{until = Seqno}] ->
Seqno
end.
-spec update_committed_offset(emqx_persistent_session_ds:id(), _Name, seqno()) -> ok.
update_committed_offset(SessionId, Name, Until) ->
mria:dirty_write(?SESSION_COMMITTED_OFFSET_TAB, #ds_committed_offset{
id = {SessionId, Name}, until = Until
}).
next_seqno(Seqno) ->
NextSeqno = Seqno + 1,
case seqno_to_packet_id(NextSeqno) of
0 ->
%% We skip sequence numbers that lead to PacketId = 0 to
%% simplify math. Note: it leads to occasional gaps in the
%% sequence numbers.
NextSeqno + 1;
_ ->
NextSeqno
end.
packet_id_to_seqno_(NextSeqno, PacketId) ->
Epoch = NextSeqno bsr 16,
case (Epoch bsl 16) + PacketId of
N when N =< NextSeqno ->
N;
N ->
N - ?EPOCH_SIZE
end.
range_size(#ds_pubrange{id = {_, First, _StreamRef}, until = Until}) ->
range_size(First, Until).
range_size(FirstSeqno, UntilSeqno) ->
%% This function assumes that gaps in the sequence ID occur _only_ when the
%% packet ID wraps.
Size = UntilSeqno - FirstSeqno,
Size + (FirstSeqno bsr 16) - (UntilSeqno bsr 16).
%%================================================================================
%% stream scheduler
%% group streams by the first position in the rank
-spec group_streams(list(ds_stream())) -> list(list(ds_stream())).
group_streams(Streams) ->
Groups = maps:groups_from_list(
fun(#ds_stream{rank = {RankX, _}}) -> RankX end,
Streams
),
shuffle(maps:values(Groups)).
-spec shuffle([A]) -> [A].
shuffle(L0) ->
L1 = lists:map(
fun(A) ->
%% maybe topic/stream prioritization could be introduced here?
{rand:uniform(), A}
end,
L0
),
L2 = lists:sort(L1),
{_, L} = lists:unzip(L2),
L.
get_the_first_stream([Group | Groups]) ->
case get_next_stream_from_group(Group) of
{Stream, {sorted, []}} ->
{Stream, Groups};
{Stream, Group2} ->
{Stream, [Group2 | Groups]};
undefined ->
get_the_first_stream(Groups)
end;
get_the_first_stream([]) ->
%% how this possible ?
throw(#{reason => no_valid_stream}).
%% the scheduler is simple, try to get messages from the same shard, but it's okay to take turns
get_next_stream_from_group({sorted, [H | T]}) ->
{H, {sorted, T}};
get_next_stream_from_group({sorted, []}) ->
undefined;
get_next_stream_from_group(Streams) ->
[Stream | T] = lists:sort(
fun(#ds_stream{rank = {_, RankA}}, #ds_stream{rank = {_, RankB}}) ->
RankA < RankB
end,
Streams
),
{Stream, {sorted, T}}.
get_next_n_messages_from_stream(Stream, CPs, N) ->
ItBegin = get_last_iterator(Stream, CPs),
case emqx_ds:next(?PERSISTENT_MESSAGE_DB, ItBegin, N) of
{ok, _ItEnd, []} ->
[];
{ok, ItEnd, Messages} ->
{ItBegin, ItEnd, Messages};
{ok, end_of_stream} ->
%% TODO: how to skip this closed stream or it should be taken over by lower level layer
[]
end.
%%================================================================================
-spec flatmapfoldl(fun((X, Acc) -> {Y | [Y], Acc}), Acc, [X]) -> {[Y], Acc}.
flatmapfoldl(_Fun, Acc, []) ->
{[], Acc};
flatmapfoldl(Fun, Acc, [X | Xs]) ->
{Ys, NAcc} = Fun(X, Acc),
{Zs, FAcc} = flatmapfoldl(Fun, NAcc, Xs),
case is_list(Ys) of
true ->
{Ys ++ Zs, FAcc};
_ ->
{[Ys | Zs], FAcc}
end.
ro_transaction(Fun) ->
{atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun),
Res.
-ifdef(TEST).
%% This test only tests boundary conditions (to make sure property-based test didn't skip them):
packet_id_to_seqno_test() ->
%% Packet ID = 1; first epoch:
?assertEqual(1, packet_id_to_seqno_(1, 1)),
?assertEqual(1, packet_id_to_seqno_(10, 1)),
?assertEqual(1, packet_id_to_seqno_(1 bsl 16 - 1, 1)),
?assertEqual(1, packet_id_to_seqno_(1 bsl 16, 1)),
%% Packet ID = 1; second and 3rd epochs:
?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(1 bsl 16 + 1, 1)),
?assertEqual(1 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16, 1)),
?assertEqual(2 bsl 16 + 1, packet_id_to_seqno_(2 bsl 16 + 1, 1)),
%% Packet ID = 16#ffff:
PID = 1 bsl 16 - 1,
?assertEqual(PID, packet_id_to_seqno_(PID, PID)),
?assertEqual(PID, packet_id_to_seqno_(1 bsl 16, PID)),
?assertEqual(1 bsl 16 + PID, packet_id_to_seqno_(2 bsl 16, PID)),
ok.
packet_id_to_seqno_test_() ->
Opts = [{numtests, 1000}, {to_file, user}],
{timeout, 30, fun() -> ?assert(proper:quickcheck(packet_id_to_seqno_prop(), Opts)) end}.
packet_id_to_seqno_prop() ->
?FORALL(
NextSeqNo,
next_seqno_gen(),
?FORALL(
SeqNo,
seqno_gen(NextSeqNo),
begin
PacketId = seqno_to_packet_id(SeqNo),
?assertEqual(SeqNo, packet_id_to_seqno_(NextSeqNo, PacketId)),
true
end
)
).
next_seqno_gen() ->
?LET(
{Epoch, Offset},
{non_neg_integer(), non_neg_integer()},
Epoch bsl 16 + Offset
).
seqno_gen(NextSeqNo) ->
WindowSize = 1 bsl 16 - 1,
Min = max(0, NextSeqNo - WindowSize),
Max = max(0, NextSeqNo - 1),
range(Min, Max).
range_size_test_() ->
[
?_assertEqual(0, range_size(42, 42)),
?_assertEqual(1, range_size(42, 43)),
?_assertEqual(1, range_size(16#ffff, 16#10001)),
?_assertEqual(16#ffff - 456 + 123, range_size(16#1f0000 + 456, 16#200000 + 123))
].
compute_inflight_range_test_() ->
[
?_assertEqual(
{#{ack => 1, comp => 1}, 1},
compute_inflight_range([])
),
?_assertEqual(
{#{ack => 12, comp => 13}, 42},
compute_inflight_range([
#ds_pubrange{id = {<<>>, 1, 0}, until = 2, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 4, 0}, until = 8, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 11, 0}, until = 12, type = ?T_CHECKPOINT},
#ds_pubrange{
id = {<<>>, 12, 0},
until = 13,
type = ?T_INFLIGHT,
tracks = ?TRACK_FLAG(?ACK)
},
#ds_pubrange{
id = {<<>>, 13, 0},
until = 20,
type = ?T_INFLIGHT,
tracks = ?TRACK_FLAG(?COMP)
},
#ds_pubrange{
id = {<<>>, 20, 0},
until = 42,
type = ?T_INFLIGHT,
tracks = ?TRACK_FLAG(?ACK) bor ?TRACK_FLAG(?COMP)
}
])
),
?_assertEqual(
{#{ack => 13, comp => 13}, 13},
compute_inflight_range([
#ds_pubrange{id = {<<>>, 1, 0}, until = 2, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 4, 0}, until = 8, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 11, 0}, until = 12, type = ?T_CHECKPOINT},
#ds_pubrange{id = {<<>>, 12, 0}, until = 13, type = ?T_CHECKPOINT}
])
)
].
-endif.

1544
apps/emqx/src/emqx_persistent_session_ds.erl
View File

File diff suppressed because it is too large Load Diff

111
apps/emqx/src/emqx_persistent_session_ds.hrl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -25,75 +25,54 @@
-define(SESSION_COMMITTED_OFFSET_TAB, emqx_ds_committed_offset_tab).
-define(DS_MRIA_SHARD, emqx_ds_session_shard).
-define(T_INFLIGHT, 1).
-define(T_CHECKPOINT, 2).
%%%%% Session sequence numbers:
-record(ds_sub, {
id :: emqx_persistent_session_ds:subscription_id(),
start_time :: emqx_ds:time(),
props = #{} :: map(),
extra = #{} :: map()
}).
-type ds_sub() :: #ds_sub{}.
%%
%% -----|----------|-----|-----|------> seqno
%% | | | |
%% committed dup rec next
%% (Qos2)
-record(ds_stream, {
session :: emqx_persistent_session_ds:id(),
ref :: _StreamRef,
stream :: emqx_ds:stream(),
rank :: emqx_ds:stream_rank(),
beginning :: emqx_ds:iterator()
}).
-type ds_stream() :: #ds_stream{}.
%% Seqno becomes committed after receiving PUBACK for QoS1 or PUBCOMP
%% for QoS2.
-define(committed(QOS), QOS).
%% Seqno becomes dup after broker sends QoS1 or QoS2 message to the
%% client. Upon session reconnect, messages with seqno in the
%% committed..dup range are retransmitted with DUP flag.
%%
-define(dup(QOS), (10 + QOS)).
%% Rec flag is specific for the QoS2. It contains seqno of the last
%% PUBREC received from the client. When the session reconnects,
%% PUBREL packages for the dup..rec range are retransmitted.
-define(rec, 22).
%% Last seqno assigned to a message (it may not be sent yet).
-define(next(QOS), (30 + QOS)).
-record(ds_pubrange, {
id :: {
%% What session this range belongs to.
_Session :: emqx_persistent_session_ds:id(),
%% Where this range starts.
_First :: emqx_persistent_message_ds_replayer:seqno(),
%% Which stream this range is over.
_StreamRef
},
%% Where this range ends: the first seqno that is not included in the range.
until :: emqx_persistent_message_ds_replayer:seqno(),
%% Type of a range:
%% * Inflight range is a range of yet unacked messages from this stream.
%% * Checkpoint range was already acked, its purpose is to keep track of the
%% very last iterator for this stream.
type :: ?T_INFLIGHT | ?T_CHECKPOINT,
%% What commit tracks this range is part of.
tracks = 0 :: non_neg_integer(),
%% Meaning of this depends on the type of the range:
%% * For inflight range, this is the iterator pointing to the first message in
%% the range.
%% * For checkpoint range, this is the iterator pointing right past the last
%% message in the range.
iterator :: emqx_ds:iterator(),
%% Reserved for future use.
misc = #{} :: map()
}).
-type ds_pubrange() :: #ds_pubrange{}.
-record(ds_committed_offset, {
id :: {
%% What session this marker belongs to.
_Session :: emqx_persistent_session_ds:id(),
%% Marker name.
_CommitType
},
%% Where this marker is pointing to: the first seqno that is not marked.
until :: emqx_persistent_message_ds_replayer:seqno()
%%%%% Stream Replay State:
-record(srs, {
rank_x :: emqx_ds:rank_x(),
rank_y :: emqx_ds:rank_y(),
%% Iterators at the beginning and the end of the last batch:
it_begin :: emqx_ds:iterator() | undefined,
it_end :: emqx_ds:iterator() | end_of_stream,
%% Size of the last batch:
batch_size = 0 :: non_neg_integer(),
%% Session sequence numbers at the time when the batch was fetched:
first_seqno_qos1 = 0 :: emqx_persistent_session_ds:seqno(),
first_seqno_qos2 = 0 :: emqx_persistent_session_ds:seqno(),
%% Sequence numbers that have to be committed for the batch:
last_seqno_qos1 = 0 :: emqx_persistent_session_ds:seqno(),
last_seqno_qos2 = 0 :: emqx_persistent_session_ds:seqno(),
%% This stream belongs to an unsubscribed topic-filter, and is
%% marked for deletion:
unsubscribed = false :: boolean()
}).
-record(session, {
%% same as clientid
id :: emqx_persistent_session_ds:id(),
%% creation time
created_at :: _Millisecond :: non_neg_integer(),
last_alive_at :: _Millisecond :: non_neg_integer(),
conninfo :: emqx_types:conninfo(),
%% for future usage
props = #{} :: map()
}).
%% Session metadata keys:
-define(created_at, created_at).
-define(last_alive_at, last_alive_at).
-define(expiry_interval, expiry_interval).
%% Unique integer used to create unique identities
-define(last_id, last_id).
-endif.

77
apps/emqx/src/emqx_persistent_session_ds_gc_worker.erl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -69,7 +69,7 @@ handle_info(_Info, State) ->
{noreply, State}.
%%--------------------------------------------------------------------------------
%% Internal fns
%% Internal functions
%%--------------------------------------------------------------------------------
ensure_gc_timer() ->
@ -104,58 +104,33 @@ now_ms() ->
erlang:system_time(millisecond).
start_gc() ->
do_gc(more).
zombie_session_ms() ->
NowMS = now_ms(),
GCInterval = emqx_config:get([session_persistence, session_gc_interval]),
BumpInterval = emqx_config:get([session_persistence, last_alive_update_interval]),
TimeThreshold = max(GCInterval, BumpInterval) * 3,
ets:fun2ms(
fun(
#session{
id = DSSessionId,
last_alive_at = LastAliveAt,
conninfo = #{expiry_interval := EI}
}
) when
LastAliveAt + EI + TimeThreshold =< NowMS
->
DSSessionId
end
).
MinLastAlive = now_ms() - TimeThreshold,
gc_loop(MinLastAlive, emqx_persistent_session_ds_state:make_session_iterator()).
do_gc(more) ->
gc_loop(MinLastAlive, It0) ->
GCBatchSize = emqx_config:get([session_persistence, session_gc_batch_size]),
MS = zombie_session_ms(),
{atomic, Next} = mria:transaction(?DS_MRIA_SHARD, fun() ->
Res = mnesia:select(?SESSION_TAB, MS, GCBatchSize, write),
case Res of
'$end_of_table' ->
done;
{[], Cont} ->
%% since `GCBatchsize' is just a "recommendation" for `select', we try only
%% _once_ the continuation and then stop if it yields nothing, to avoid a
%% dead loop.
case mnesia:select(Cont) of
'$end_of_table' ->
done;
{[], _Cont} ->
done;
{DSSessionIds0, _Cont} ->
do_gc_(DSSessionIds0),
more
end;
{DSSessionIds0, _Cont} ->
do_gc_(DSSessionIds0),
more
end
end),
do_gc(Next);
do_gc(done) ->
ok.
case emqx_persistent_session_ds_state:session_iterator_next(It0, GCBatchSize) of
{[], _It} ->
ok;
{Sessions, It} ->
[do_gc(SessionId, MinLastAlive, Metadata) || {SessionId, Metadata} <- Sessions],
gc_loop(MinLastAlive, It)
end.
do_gc_(DSSessionIds) ->
lists:foreach(fun emqx_persistent_session_ds:destroy_session/1, DSSessionIds),
?tp(ds_session_gc_cleaned, #{session_ids => DSSessionIds}),
ok.
do_gc(SessionId, MinLastAlive, Metadata) ->
#{?last_alive_at := LastAliveAt, ?expiry_interval := EI} = Metadata,
case LastAliveAt + EI < MinLastAlive of
true ->
emqx_persistent_session_ds:destroy_session(SessionId),
?tp(debug, ds_session_gc_cleaned, #{
session_id => SessionId,
last_alive_at => LastAliveAt,
expiry_interval => EI,
min_last_alive => MinLastAlive
});
false ->
ok
end.

347
apps/emqx/src/emqx_persistent_session_ds_inflight.erl Normal file
View File

@ -0,0 +1,347 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_persistent_session_ds_inflight).
%% API:
-export([
new/1,
push/2,
pop/1,
n_buffered/2,
n_inflight/1,
puback/2,
pubrec/2,
pubcomp/2,
receive_maximum/1
]).
%% internal exports:
-export([]).
-export_type([t/0]).
-include("emqx.hrl").
-include("emqx_mqtt.hrl").
-ifdef(TEST).
-include_lib("proper/include/proper.hrl").
-include_lib("eunit/include/eunit.hrl").
-endif.
%%================================================================================
%% Type declarations
%%================================================================================
-type payload() ::
{emqx_persistent_session_ds:seqno() | undefined, emqx_types:message()}
| {pubrel, emqx_persistent_session_ds:seqno()}.
-record(inflight, {
receive_maximum :: pos_integer(),
%% Main queue:
queue :: queue:queue(payload()),
%% Queues that are used to track sequence numbers of ack tracks:
puback_queue :: iqueue(),
pubrec_queue :: iqueue(),
pubcomp_queue :: iqueue(),
%% Counters:
n_inflight = 0 :: non_neg_integer(),
n_qos0 = 0 :: non_neg_integer(),
n_qos1 = 0 :: non_neg_integer(),
n_qos2 = 0 :: non_neg_integer()
}).
-type t() :: #inflight{}.
%%================================================================================
%% API funcions
%%================================================================================
-spec new(non_neg_integer()) -> t().
new(ReceiveMaximum) when ReceiveMaximum > 0 ->
#inflight{
receive_maximum = ReceiveMaximum,
queue = queue:new(),
puback_queue = iqueue_new(),
pubrec_queue = iqueue_new(),
pubcomp_queue = iqueue_new()
}.
-spec receive_maximum(t()) -> pos_integer().
receive_maximum(#inflight{receive_maximum = ReceiveMaximum}) ->
ReceiveMaximum.
-spec push(payload(), t()) -> t().
push(Payload = {pubrel, _SeqNo}, Rec = #inflight{queue = Q}) ->
Rec#inflight{queue = queue:in(Payload, Q)};
push(Payload = {_, Msg}, Rec) ->
#inflight{queue = Q0, n_qos0 = NQos0, n_qos1 = NQos1, n_qos2 = NQos2} = Rec,
Q = queue:in(Payload, Q0),
case Msg#message.qos of
?QOS_0 ->
Rec#inflight{queue = Q, n_qos0 = NQos0 + 1};
?QOS_1 ->
Rec#inflight{queue = Q, n_qos1 = NQos1 + 1};
?QOS_2 ->
Rec#inflight{queue = Q, n_qos2 = NQos2 + 1}
end.
-spec pop(t()) -> {payload(), t()} | undefined.
pop(Rec0) ->
#inflight{
receive_maximum = ReceiveMaximum,
n_inflight = NInflight,
queue = Q0,
puback_queue = QAck,
pubrec_queue = QRec,
pubcomp_queue = QComp,
n_qos0 = NQos0,
n_qos1 = NQos1,
n_qos2 = NQos2
} = Rec0,
case NInflight < ReceiveMaximum andalso queue:out(Q0) of
{{value, Payload}, Q} ->
Rec =
case Payload of
{pubrel, _} ->
Rec0#inflight{queue = Q};
{SeqNo, #message{qos = Qos}} ->
case Qos of
?QOS_0 ->
Rec0#inflight{queue = Q, n_qos0 = NQos0 - 1};
?QOS_1 ->
Rec0#inflight{
queue = Q,
n_qos1 = NQos1 - 1,
n_inflight = NInflight + 1,
puback_queue = ipush(SeqNo, QAck)
};
?QOS_2 ->
Rec0#inflight{
queue = Q,
n_qos2 = NQos2 - 1,
n_inflight = NInflight + 1,
pubrec_queue = ipush(SeqNo, QRec),
pubcomp_queue = ipush(SeqNo, QComp)
}
end
end,
{Payload, Rec};
_ ->
undefined
end.
-spec n_buffered(?QOS_0..?QOS_2 | all, t()) -> non_neg_integer().
n_buffered(?QOS_0, #inflight{n_qos0 = NQos0}) ->
NQos0;
n_buffered(?QOS_1, #inflight{n_qos1 = NQos1}) ->
NQos1;
n_buffered(?QOS_2, #inflight{n_qos2 = NQos2}) ->
NQos2;
n_buffered(all, #inflight{n_qos0 = NQos0, n_qos1 = NQos1, n_qos2 = NQos2}) ->
NQos0 + NQos1 + NQos2.
-spec n_inflight(t()) -> non_neg_integer().
n_inflight(#inflight{n_inflight = NInflight}) ->
NInflight.
-spec puback(emqx_persistent_session_ds:seqno(), t()) -> {ok, t()} | {error, Expected} when
Expected :: emqx_persistent_session_ds:seqno() | undefined.
puback(SeqNo, Rec = #inflight{puback_queue = Q0, n_inflight = N}) ->
case ipop(Q0) of
{{value, SeqNo}, Q} ->
{ok, Rec#inflight{
puback_queue = Q,
n_inflight = max(0, N - 1)
}};
{{value, Expected}, _} ->
{error, Expected};
_ ->
{error, undefined}
end.
-spec pubcomp(emqx_persistent_session_ds:seqno(), t()) -> {ok, t()} | {error, Expected} when
Expected :: emqx_persistent_session_ds:seqno() | undefined.
pubcomp(SeqNo, Rec = #inflight{pubcomp_queue = Q0, n_inflight = N}) ->
case ipop(Q0) of
{{value, SeqNo}, Q} ->
{ok, Rec#inflight{
pubcomp_queue = Q,
n_inflight = max(0, N - 1)
}};
{{value, Expected}, _} ->
{error, Expected};
_ ->
{error, undefined}
end.
%% PUBREC doesn't affect inflight window:
%% https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Flow_Control
-spec pubrec(emqx_persistent_session_ds:seqno(), t()) -> {ok, t()} | {error, Expected} when
Expected :: emqx_persistent_session_ds:seqno() | undefined.
pubrec(SeqNo, Rec = #inflight{pubrec_queue = Q0}) ->
case ipop(Q0) of
{{value, SeqNo}, Q} ->
{ok, Rec#inflight{
pubrec_queue = Q
}};
{{value, Expected}, _} ->
{error, Expected};
_ ->
{error, undefined}
end.
%%================================================================================
%% Internal functions
%%================================================================================
%%%% Interval queue:
%% "Interval queue": a data structure that represents a queue of
%% monotonically increasing non-negative integers in a compact manner.
%% It is functionally equivalent to a `queue:queue(integer())'.
-record(iqueue, {
%% Head interval:
head = 0 :: integer(),
head_end = 0 :: integer(),
%% Intermediate ranges:
queue :: queue:queue({integer(), integer()}),
%% End interval:
tail = 0 :: integer(),
tail_end = 0 :: integer()
}).
-type iqueue() :: #iqueue{}.
iqueue_new() ->
#iqueue{
queue = queue:new()
}.
%% @doc Push a value into the interval queue:
-spec ipush(integer(), iqueue()) -> iqueue().
ipush(Val, Q = #iqueue{tail_end = Val, head_end = Val}) ->
%% Optimization: head and tail intervals overlap, and the newly
%% inserted value extends both. Attach it to both intervals, to
%% avoid `queue:out' in `ipop':
Q#iqueue{
tail_end = Val + 1,
head_end = Val + 1
};
ipush(Val, Q = #iqueue{tail_end = Val}) ->
%% Extend tail interval:
Q#iqueue{
tail_end = Val + 1
};
ipush(Val, Q = #iqueue{tail = Tl, tail_end = End, queue = IQ0}) when is_number(Val), Val > End ->
IQ = queue:in({Tl, End}, IQ0),
%% Begin a new interval:
Q#iqueue{
queue = IQ,
tail = Val,
tail_end = Val + 1
}.
-spec ipop(iqueue()) -> {{value, integer()}, iqueue()} | {empty, iqueue()}.
ipop(Q = #iqueue{head = Hd, head_end = HdEnd}) when Hd < HdEnd ->
%% Head interval is not empty. Consume a value from it:
{{value, Hd}, Q#iqueue{head = Hd + 1}};
ipop(Q = #iqueue{head_end = End, tail_end = End}) ->
%% Head interval is fully consumed, and it's overlaps with the
%% tail interval. It means the queue is empty:
{empty, Q};
ipop(Q = #iqueue{head = Hd0, tail = Tl, tail_end = TlEnd, queue = IQ0}) ->
%% Head interval is fully consumed, and it doesn't overlap with
%% the tail interval. Replace the head interval with the next
%% interval from the queue or with the tail interval:
case queue:out(IQ0) of
{{value, {Hd, HdEnd}}, IQ} ->
ipop(Q#iqueue{head = max(Hd0, Hd), head_end = HdEnd, queue = IQ});
{empty, _} ->
ipop(Q#iqueue{head = max(Hd0, Tl), head_end = TlEnd})
end.
-ifdef(TEST).
%% Test that behavior of iqueue is identical to that of a regular queue of integers:
iqueue_compat_test_() ->
Props = [iqueue_compat()],
Opts = [{numtests, 1000}, {to_file, user}, {max_size, 100}],
{timeout, 30, [?_assert(proper:quickcheck(Prop, Opts)) || Prop <- Props]}.
%% Generate a sequence of pops and pushes with monotonically
%% increasing arguments, and verify replaying produces equivalent
%% results for the optimized and the reference implementation:
iqueue_compat() ->
?FORALL(
Cmds,
iqueue_commands(),
begin
lists:foldl(
fun
({push, N}, {IQ, Q, Acc}) ->
{ipush(N, IQ), queue:in(N, Q), [N | Acc]};
(pop, {IQ0, Q0, Acc}) ->
{Ret, IQ} = ipop(IQ0),
{Expected, Q} = queue:out(Q0),
?assertEqual(
Expected,
Ret,
#{
sequence => lists:reverse(Acc),
q => queue:to_list(Q0),
iq0 => iqueue_print(IQ0),
iq => iqueue_print(IQ)
}
),
{IQ, Q, [pop | Acc]}
end,
{iqueue_new(), queue:new(), []},
Cmds
),
true
end
).
iqueue_cmd() ->
oneof([
pop,
{push, range(1, 3)}
]).
iqueue_commands() ->
?LET(
Cmds,
list(iqueue_cmd()),
process_test_cmds(Cmds, 0)
).
process_test_cmds([], _) ->
[];
process_test_cmds([pop | Tl], Cnt) ->
[pop | process_test_cmds(Tl, Cnt)];
process_test_cmds([{push, N} | Tl], Cnt0) ->
Cnt = Cnt0 + N,
[{push, Cnt} | process_test_cmds(Tl, Cnt)].
iqueue_print(I = #iqueue{head = Hd, head_end = HdEnd, queue = Q, tail = Tl, tail_end = TlEnd}) ->
#{
hd => {Hd, HdEnd},
tl => {Tl, TlEnd},
q => queue:to_list(Q)
}.
-endif.

586
apps/emqx/src/emqx_persistent_session_ds_state.erl Normal file
View File

@ -0,0 +1,586 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc CRUD interface for the persistent session
%%
%% This module encapsulates the data related to the state of the
%% inflight messages for the persistent session based on DS.
%%
%% It is responsible for saving, caching, and restoring session state.
%% It is completely devoid of business logic. Not even the default
%% values should be set in this module.
-module(emqx_persistent_session_ds_state).
-export([create_tables/0]).
-export([open/1, create_new/1, delete/1, commit/1, format/1, print_session/1, list_sessions/0]).
-export([get_created_at/1, set_created_at/2]).
-export([get_last_alive_at/1, set_last_alive_at/2]).
-export([get_expiry_interval/1, set_expiry_interval/2]).
-export([new_id/1]).
-export([get_stream/2, put_stream/3, del_stream/2, fold_streams/3]).
-export([get_seqno/2, put_seqno/3]).
-export([get_rank/2, put_rank/3, del_rank/2, fold_ranks/3]).
-export([get_subscriptions/1, put_subscription/4, del_subscription/3]).
-export([make_session_iterator/0, session_iterator_next/2]).
-export_type([
t/0, metadata/0, subscriptions/0, seqno_type/0, stream_key/0, rank_key/0, session_iterator/0
]).
-include("emqx_mqtt.hrl").
-include("emqx_persistent_session_ds.hrl").
-include_lib("snabbkaffe/include/trace.hrl").
-include_lib("stdlib/include/qlc.hrl").
%%================================================================================
%% Type declarations
%%================================================================================
-type subscriptions() :: emqx_topic_gbt:t(_SubId, emqx_persistent_session_ds:subscription()).
-opaque session_iterator() :: emqx_persistent_session_ds:id() | '$end_of_table'.
%% Generic key-value wrapper that is used for exporting arbitrary
%% terms to mnesia:
-record(kv, {k, v}).
%% Persistent map.
%%
%% Pmap accumulates the updates in a term stored in the heap of a
%% process, so they can be committed all at once in a single
%% transaction.
%%
%% It should be possible to make frequent changes to the pmap without
%% stressing Mria.
%%
%% It's implemented as two maps: `cache', and `dirty'. `cache' stores
%% the data, and `dirty' contains information about dirty and deleted
%% keys. When `commit/1' is called, dirty keys are dumped to the
%% tables, and deleted keys are removed from the tables.
-record(pmap, {table, cache, dirty}).
-type pmap(K, V) ::
#pmap{
table :: atom(),
cache :: #{K => V},
dirty :: #{K => dirty | del}
}.
-type metadata() ::
#{
?created_at => emqx_persistent_session_ds:timestamp(),
?last_alive_at => emqx_persistent_session_ds:timestamp(),
?expiry_interval => non_neg_integer(),
?last_id => integer()
}.
-type seqno_type() ::
?next(?QOS_1)
| ?dup(?QOS_1)
| ?committed(?QOS_1)
| ?next(?QOS_2)
| ?dup(?QOS_2)
| ?rec
| ?committed(?QOS_2).
-opaque t() :: #{
id := emqx_persistent_session_ds:id(),
dirty := boolean(),
metadata := metadata(),
subscriptions := subscriptions(),
seqnos := pmap(seqno_type(), emqx_persistent_session_ds:seqno()),
streams := pmap(emqx_ds:stream(), emqx_persistent_session_ds:stream_state()),
ranks := pmap(term(), integer())
}.
-define(session_tab, emqx_ds_session_tab).
-define(subscription_tab, emqx_ds_session_subscriptions).
-define(stream_tab, emqx_ds_session_streams).
-define(seqno_tab, emqx_ds_session_seqnos).
-define(rank_tab, emqx_ds_session_ranks).
-define(pmap_tables, [?stream_tab, ?seqno_tab, ?rank_tab, ?subscription_tab]).
%% Enable this flag if you suspect some code breaks the sequence:
-ifndef(CHECK_SEQNO).
-define(set_dirty, dirty => true).
-define(unset_dirty, dirty => false).
-else.
-define(set_dirty, dirty => true, '_' => do_seqno()).
-define(unset_dirty, dirty => false, '_' => do_seqno()).
-endif.
%%================================================================================
%% API funcions
%%================================================================================
-spec create_tables() -> ok.
create_tables() ->
ok = mria:create_table(
?session_tab,
[
{rlog_shard, ?DS_MRIA_SHARD},
{type, ordered_set},
{storage, rocksdb_copies},
{record_name, kv},
{attributes, record_info(fields, kv)}
]
),
[create_kv_pmap_table(Table) || Table <- ?pmap_tables],
mria:wait_for_tables([?session_tab | ?pmap_tables]).
-spec open(emqx_persistent_session_ds:id()) -> {ok, t()} | undefined.
open(SessionId) ->
ro_transaction(fun() ->
case kv_restore(?session_tab, SessionId) of
[Metadata] ->
Rec = #{
id => SessionId,
metadata => Metadata,
subscriptions => read_subscriptions(SessionId),
streams => pmap_open(?stream_tab, SessionId),
seqnos => pmap_open(?seqno_tab, SessionId),
ranks => pmap_open(?rank_tab, SessionId),
?unset_dirty
},
{ok, Rec};
[] ->
undefined
end
end).
-spec print_session(emqx_persistent_session_ds:id()) -> map() | undefined.
print_session(SessionId) ->
case open(SessionId) of
undefined ->
undefined;
{ok, Session} ->
format(Session)
end.
-spec format(t()) -> map().
format(#{
metadata := Metadata,
subscriptions := SubsGBT,
streams := Streams,
seqnos := Seqnos,
ranks := Ranks
}) ->
Subs = emqx_topic_gbt:fold(
fun(Key, Sub, Acc) ->
maps:put(emqx_topic_gbt:get_topic(Key), Sub, Acc)
end,
#{},
SubsGBT
),
#{
metadata => Metadata,
subscriptions => Subs,
streams => pmap_format(Streams),
seqnos => pmap_format(Seqnos),
ranks => pmap_format(Ranks)
}.
-spec list_sessions() -> [emqx_persistent_session_ds:id()].
list_sessions() ->
mnesia:dirty_all_keys(?session_tab).
-spec delete(emqx_persistent_session_ds:id()) -> ok.
delete(Id) ->
transaction(
fun() ->
[kv_pmap_delete(Table, Id) || Table <- ?pmap_tables],
mnesia:delete(?session_tab, Id, write)
end
).
-spec commit(t()) -> t().
commit(Rec = #{dirty := false}) ->
Rec;
commit(
Rec = #{
id := SessionId,
metadata := Metadata,
streams := Streams,
seqnos := SeqNos,
ranks := Ranks
}
) ->
check_sequence(Rec),
transaction(fun() ->
kv_persist(?session_tab, SessionId, Metadata),
Rec#{
streams => pmap_commit(SessionId, Streams),
seqnos => pmap_commit(SessionId, SeqNos),
ranks => pmap_commit(SessionId, Ranks),
?unset_dirty
}
end).
-spec create_new(emqx_persistent_session_ds:id()) -> t().
create_new(SessionId) ->
transaction(fun() ->
delete(SessionId),
#{
id => SessionId,
metadata => #{},
subscriptions => emqx_topic_gbt:new(),
streams => pmap_open(?stream_tab, SessionId),
seqnos => pmap_open(?seqno_tab, SessionId),
ranks => pmap_open(?rank_tab, SessionId),
?set_dirty
}
end).
%%
-spec get_created_at(t()) -> emqx_persistent_session_ds:timestamp() | undefined.
get_created_at(Rec) ->
get_meta(?created_at, Rec).
-spec set_created_at(emqx_persistent_session_ds:timestamp(), t()) -> t().
set_created_at(Val, Rec) ->
set_meta(?created_at, Val, Rec).
-spec get_last_alive_at(t()) -> emqx_persistent_session_ds:timestamp() | undefined.
get_last_alive_at(Rec) ->
get_meta(?last_alive_at, Rec).
-spec set_last_alive_at(emqx_persistent_session_ds:timestamp(), t()) -> t().
set_last_alive_at(Val, Rec) ->
set_meta(?last_alive_at, Val, Rec).
-spec get_expiry_interval(t()) -> non_neg_integer() | undefined.
get_expiry_interval(Rec) ->
get_meta(?expiry_interval, Rec).
-spec set_expiry_interval(non_neg_integer(), t()) -> t().
set_expiry_interval(Val, Rec) ->
set_meta(?expiry_interval, Val, Rec).
-spec new_id(t()) -> {emqx_persistent_session_ds:subscription_id(), t()}.
new_id(Rec) ->
LastId =
case get_meta(?last_id, Rec) of
undefined -> 0;
N when is_integer(N) -> N
end,
{LastId, set_meta(?last_id, LastId + 1, Rec)}.
%%
-spec get_subscriptions(t()) -> subscriptions().
get_subscriptions(#{subscriptions := Subs}) ->
Subs.
-spec put_subscription(
emqx_persistent_session_ds:topic_filter(),
_SubId,
emqx_persistent_session_ds:subscription(),
t()
) -> t().
put_subscription(TopicFilter, SubId, Subscription, Rec = #{id := Id, subscriptions := Subs0}) ->
%% Note: currently changes to the subscriptions are persisted immediately.
Key = {TopicFilter, SubId},
transaction(fun() -> kv_pmap_persist(?subscription_tab, Id, Key, Subscription) end),
Subs = emqx_topic_gbt:insert(TopicFilter, SubId, Subscription, Subs0),
Rec#{subscriptions => Subs}.
-spec del_subscription(emqx_persistent_session_ds:topic_filter(), _SubId, t()) -> t().
del_subscription(TopicFilter, SubId, Rec = #{id := Id, subscriptions := Subs0}) ->
%% Note: currently the subscriptions are persisted immediately.
Key = {TopicFilter, SubId},
transaction(fun() -> kv_pmap_delete(?subscription_tab, Id, Key) end),
Subs = emqx_topic_gbt:delete(TopicFilter, SubId, Subs0),
Rec#{subscriptions => Subs}.
%%
-type stream_key() :: {emqx_persistent_session_ds:subscription_id(), _StreamId}.
-spec get_stream(stream_key(), t()) ->
emqx_persistent_session_ds:stream_state() | undefined.
get_stream(Key, Rec) ->
gen_get(streams, Key, Rec).
-spec put_stream(stream_key(), emqx_persistent_session_ds:stream_state(), t()) -> t().
put_stream(Key, Val, Rec) ->
gen_put(streams, Key, Val, Rec).
-spec del_stream(stream_key(), t()) -> t().
del_stream(Key, Rec) ->
gen_del(streams, Key, Rec).
-spec fold_streams(fun(), Acc, t()) -> Acc.
fold_streams(Fun, Acc, Rec) ->
gen_fold(streams, Fun, Acc, Rec).
%%
-spec get_seqno(seqno_type(), t()) -> emqx_persistent_session_ds:seqno() | undefined.
get_seqno(Key, Rec) ->
gen_get(seqnos, Key, Rec).
-spec put_seqno(seqno_type(), emqx_persistent_session_ds:seqno(), t()) -> t().
put_seqno(Key, Val, Rec) ->
gen_put(seqnos, Key, Val, Rec).
%%
-type rank_key() :: {emqx_persistent_session_ds:subscription_id(), emqx_ds:rank_x()}.
-spec get_rank(rank_key(), t()) -> integer() | undefined.
get_rank(Key, Rec) ->
gen_get(ranks, Key, Rec).
-spec put_rank(rank_key(), integer(), t()) -> t().
put_rank(Key, Val, Rec) ->
gen_put(ranks, Key, Val, Rec).
-spec del_rank(rank_key(), t()) -> t().
del_rank(Key, Rec) ->
gen_del(ranks, Key, Rec).
-spec fold_ranks(fun(), Acc, t()) -> Acc.
fold_ranks(Fun, Acc, Rec) ->
gen_fold(ranks, Fun, Acc, Rec).
-spec make_session_iterator() -> session_iterator().
make_session_iterator() ->
case mnesia:dirty_first(?session_tab) of
'$end_of_table' ->
'$end_of_table';
Key ->
Key
end.
-spec session_iterator_next(session_iterator(), pos_integer()) ->
{[{emqx_persistent_session_ds:id(), metadata()}], session_iterator()}.
session_iterator_next(Cursor, 0) ->
{[], Cursor};
session_iterator_next('$end_of_table', _N) ->
{[], '$end_of_table'};
session_iterator_next(Cursor0, N) ->
ThisVal = [
{Cursor0, Metadata}
|| #kv{v = Metadata} <- mnesia:dirty_read(?session_tab, Cursor0)
],
{NextVals, Cursor} = session_iterator_next(mnesia:dirty_next(?session_tab, Cursor0), N - 1),
{ThisVal ++ NextVals, Cursor}.
%%================================================================================
%% Internal functions
%%================================================================================
%% All mnesia reads and writes are passed through this function.
%% Backward compatiblity issues can be handled here.
encoder(encode, _Table, Term) ->
Term;
encoder(decode, _Table, Term) ->
Term.
%%
get_meta(K, #{metadata := Meta}) ->
maps:get(K, Meta, undefined).
set_meta(K, V, Rec = #{metadata := Meta}) ->
check_sequence(Rec#{metadata => maps:put(K, V, Meta), ?set_dirty}).
%%
gen_get(Field, Key, Rec) ->
check_sequence(Rec),
pmap_get(Key, maps:get(Field, Rec)).
gen_fold(Field, Fun, Acc, Rec) ->
check_sequence(Rec),
pmap_fold(Fun, Acc, maps:get(Field, Rec)).
gen_put(Field, Key, Val, Rec) ->
check_sequence(Rec),
maps:update_with(
Field,
fun(PMap) -> pmap_put(Key, Val, PMap) end,
Rec#{?set_dirty}
).
gen_del(Field, Key, Rec) ->
check_sequence(Rec),
maps:update_with(
Field,
fun(PMap) -> pmap_del(Key, PMap) end,
Rec#{?set_dirty}
).
%%
read_subscriptions(SessionId) ->
Records = kv_pmap_restore(?subscription_tab, SessionId),
lists:foldl(
fun({{TopicFilter, SubId}, Subscription}, Acc) ->
emqx_topic_gbt:insert(TopicFilter, SubId, Subscription, Acc)
end,
emqx_topic_gbt:new(),
Records
).
%%
%% @doc Open a PMAP and fill the clean area with the data from DB.
%% This functtion should be ran in a transaction.
-spec pmap_open(atom(), emqx_persistent_session_ds:id()) -> pmap(_K, _V).
pmap_open(Table, SessionId) ->
Clean = maps:from_list(kv_pmap_restore(Table, SessionId)),
#pmap{
table = Table,
cache = Clean,
dirty = #{}
}.
-spec pmap_get(K, pmap(K, V)) -> V | undefined.
pmap_get(K, #pmap{cache = Cache}) ->
maps:get(K, Cache, undefined).
-spec pmap_put(K, V, pmap(K, V)) -> pmap(K, V).
pmap_put(K, V, Pmap = #pmap{dirty = Dirty, cache = Cache}) ->
Pmap#pmap{
cache = maps:put(K, V, Cache),
dirty = Dirty#{K => dirty}
}.
-spec pmap_del(K, pmap(K, V)) -> pmap(K, V).
pmap_del(
Key,
Pmap = #pmap{dirty = Dirty, cache = Cache}
) ->
Pmap#pmap{
cache = maps:remove(Key, Cache),
dirty = Dirty#{Key => del}
}.
-spec pmap_fold(fun((K, V, A) -> A), A, pmap(K, V)) -> A.
pmap_fold(Fun, Acc, #pmap{cache = Cache}) ->
maps:fold(Fun, Acc, Cache).
-spec pmap_commit(emqx_persistent_session_ds:id(), pmap(K, V)) -> pmap(K, V).
pmap_commit(
SessionId, Pmap = #pmap{table = Tab, dirty = Dirty, cache = Cache}
) ->
maps:foreach(
fun
(K, del) ->
kv_pmap_delete(Tab, SessionId, K);
(K, dirty) ->
V = maps:get(K, Cache),
kv_pmap_persist(Tab, SessionId, K, V)
end,
Dirty
),
Pmap#pmap{
dirty = #{}
}.
-spec pmap_format(pmap(_K, _V)) -> map().
pmap_format(#pmap{cache = Cache}) ->
Cache.
%% Functions dealing with set tables:
kv_persist(Tab, SessionId, Val0) ->
Val = encoder(encode, Tab, Val0),
mnesia:write(Tab, #kv{k = SessionId, v = Val}, write).
kv_restore(Tab, SessionId) ->
[encoder(decode, Tab, V) || #kv{v = V} <- mnesia:read(Tab, SessionId)].
%% Functions dealing with bags:
%% @doc Create a mnesia table for the PMAP:
-spec create_kv_pmap_table(atom()) -> ok.
create_kv_pmap_table(Table) ->
mria:create_table(Table, [
{type, ordered_set},
{rlog_shard, ?DS_MRIA_SHARD},
{storage, rocksdb_copies},
{record_name, kv},
{attributes, record_info(fields, kv)}
]).
kv_pmap_persist(Tab, SessionId, Key, Val0) ->
%% Write data to mnesia:
Val = encoder(encode, Tab, Val0),
mnesia:write(Tab, #kv{k = {SessionId, Key}, v = Val}, write).
kv_pmap_restore(Table, SessionId) ->
MS = [{#kv{k = {SessionId, '$1'}, v = '$2'}, [], [{{'$1', '$2'}}]}],
Objs = mnesia:select(Table, MS, read),
[{K, encoder(decode, Table, V)} || {K, V} <- Objs].
kv_pmap_delete(Table, SessionId) ->
MS = [{#kv{k = {SessionId, '$1'}, _ = '_'}, [], ['$1']}],
Keys = mnesia:select(Table, MS, read),
[mnesia:delete(Table, {SessionId, K}, write) || K <- Keys],
ok.
kv_pmap_delete(Table, SessionId, Key) ->
%% Note: this match spec uses a fixed primary key, so it doesn't
%% require a table scan, and the transaction doesn't grab the
%% whole table lock:
mnesia:delete(Table, {SessionId, Key}, write).
%%
transaction(Fun) ->
mria:async_dirty(?DS_MRIA_SHARD, Fun).
ro_transaction(Fun) ->
mria:async_dirty(?DS_MRIA_SHARD, Fun).
%% transaction(Fun) ->
%% case mnesia:is_transaction() of
%% true ->
%% Fun();
%% false ->
%% {atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun),
%% Res
%% end.
%% ro_transaction(Fun) ->
%% {atomic, Res} = mria:ro_transaction(?DS_MRIA_SHARD, Fun),
%% Res.
-compile({inline, check_sequence/1}).
-ifdef(CHECK_SEQNO).
do_seqno() ->
case erlang:get(?MODULE) of
undefined ->
put(?MODULE, 0),
0;
N ->
put(?MODULE, N + 1),
N + 1
end.
check_sequence(A = #{'_' := N}) ->
N = erlang:get(?MODULE),
A.
-else.
check_sequence(A) ->
A.
-endif.

380
apps/emqx/src/emqx_persistent_session_ds_stream_scheduler.erl Normal file
View File

@ -0,0 +1,380 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_persistent_session_ds_stream_scheduler).
%% API:
-export([find_new_streams/1, find_replay_streams/1, is_fully_acked/2]).
-export([renew_streams/1, on_unsubscribe/2]).
%% behavior callbacks:
-export([]).
%% internal exports:
-export([]).
-export_type([]).
-include_lib("emqx/include/logger.hrl").
-include("emqx_mqtt.hrl").
-include("emqx_persistent_session_ds.hrl").
%%================================================================================
%% Type declarations
%%================================================================================
%%================================================================================
%% API functions
%%================================================================================
%% @doc Find the streams that have uncommitted (in-flight) messages.
%% Return them in the order they were previously replayed.
-spec find_replay_streams(emqx_persistent_session_ds_state:t()) ->
[{emqx_persistent_session_ds_state:stream_key(), emqx_persistent_session_ds:stream_state()}].
find_replay_streams(S) ->
Comm1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S),
Comm2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S),
%% 1. Find the streams that aren't fully acked
Streams = emqx_persistent_session_ds_state:fold_streams(
fun(Key, Stream, Acc) ->
case is_fully_acked(Comm1, Comm2, Stream) of
false ->
[{Key, Stream} | Acc];
true ->
Acc
end
end,
[],
S
),
lists:sort(fun compare_streams/2, Streams).
%% @doc Find streams from which the new messages can be fetched.
%%
%% Currently it amounts to the streams that don't have any inflight
%% messages, since for performance reasons we keep only one record of
%% in-flight messages per stream, and we don't want to overwrite these
%% records prematurely.
%%
%% This function is non-detereministic: it randomizes the order of
%% streams to ensure fair replay of different topics.
-spec find_new_streams(emqx_persistent_session_ds_state:t()) ->
[{emqx_persistent_session_ds_state:stream_key(), emqx_persistent_session_ds:stream_state()}].
find_new_streams(S) ->
%% FIXME: this function is currently very sensitive to the
%% consistency of the packet IDs on both broker and client side.
%%
%% If the client fails to properly ack packets due to a bug, or a
%% network issue, or if the state of streams and seqno tables ever
%% become de-synced, then this function will return an empty list,
%% and the replay cannot progress.
%%
%% In other words, this function is not robust, and we should find
%% some way to get the replays un-stuck at the cost of potentially
%% losing messages during replay (or just kill the stuck channel
%% after timeout?)
Comm1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S),
Comm2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S),
shuffle(
emqx_persistent_session_ds_state:fold_streams(
fun
(_Key, #srs{it_end = end_of_stream}, Acc) ->
Acc;
(Key, Stream, Acc) ->
case is_fully_acked(Comm1, Comm2, Stream) andalso not Stream#srs.unsubscribed of
true ->
[{Key, Stream} | Acc];
false ->
Acc
end
end,
[],
S
)
).
%% @doc This function makes the session aware of the new streams.
%%
%% It has the following properties:
%%
%% 1. For each RankX, it keeps only the streams with the same RankY.
%%
%% 2. For each RankX, it never advances RankY until _all_ streams with
%% the same RankX are replayed.
%%
%% 3. Once all streams with the given rank are replayed, it advances
%% the RankY to the smallest known RankY that is greater than replayed
%% RankY.
%%
%% 4. If the RankX has never been replayed, it selects the streams
%% with the smallest RankY.
%%
%% This way, messages from the same topic/shard are never reordered.
-spec renew_streams(emqx_persistent_session_ds_state:t()) -> emqx_persistent_session_ds_state:t().
renew_streams(S0) ->
S1 = remove_unsubscribed_streams(S0),
S2 = remove_fully_replayed_streams(S1),
emqx_persistent_session_ds_subs:fold(
fun
(Key, #{start_time := StartTime, id := SubId, deleted := false}, Acc) ->
TopicFilter = emqx_topic:words(Key),
Streams = select_streams(
SubId,
emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime),
Acc
),
lists:foldl(
fun(I, Acc1) ->
ensure_iterator(TopicFilter, StartTime, SubId, I, Acc1)
end,
Acc,
Streams
);
(_Key, _DeletedSubscription, Acc) ->
Acc
end,
S2,
S2
).
-spec on_unsubscribe(
emqx_persistent_session_ds:subscription_id(), emqx_persistent_session_ds_state:t()
) ->
emqx_persistent_session_ds_state:t().
on_unsubscribe(SubId, S0) ->
%% NOTE: this function only marks the streams for deletion,
%% instead of outright deleting them.
%%
%% It's done for two reasons:
%%
%% - MQTT standard states that the broker MUST process acks for
%% all sent messages, and it MAY keep on sending buffered
%% messages:
%% https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Toc3901186
%%
%% - Deleting the streams may lead to gaps in the sequence number
%% series, and lead to problems with acknowledgement tracking, we
%% avoid that by delaying the deletion.
%%
%% When the stream is marked for deletion, the session won't fetch
%% _new_ batches from it. Actual deletion is done by
%% `renew_streams', when it detects that all in-flight messages
%% from the stream have been acked by the client.
emqx_persistent_session_ds_state:fold_streams(
fun(Key, Srs, Acc) ->
case Key of
{SubId, _Stream} ->
%% This stream belongs to a deleted subscription.
%% Mark for deletion:
emqx_persistent_session_ds_state:put_stream(
Key, Srs#srs{unsubscribed = true}, Acc
);
_ ->
Acc
end
end,
S0,
S0
).
-spec is_fully_acked(
emqx_persistent_session_ds:stream_state(), emqx_persistent_session_ds_state:t()
) -> boolean().
is_fully_acked(Srs, S) ->
CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S),
CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S),
is_fully_acked(CommQos1, CommQos2, Srs).
%%================================================================================
%% Internal functions
%%================================================================================
ensure_iterator(TopicFilter, StartTime, SubId, {{RankX, RankY}, Stream}, S) ->
Key = {SubId, Stream},
case emqx_persistent_session_ds_state:get_stream(Key, S) of
undefined ->
?SLOG(debug, #{
msg => new_stream, key => Key, stream => Stream
}),
{ok, Iterator} = emqx_ds:make_iterator(
?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime
),
NewStreamState = #srs{
rank_x = RankX,
rank_y = RankY,
it_begin = Iterator,
it_end = Iterator
},
emqx_persistent_session_ds_state:put_stream(Key, NewStreamState, S);
#srs{} ->
S
end.
select_streams(SubId, Streams0, S) ->
TopicStreamGroups = maps:groups_from_list(fun({{X, _}, _}) -> X end, Streams0),
maps:fold(
fun(RankX, Streams, Acc) ->
select_streams(SubId, RankX, Streams, S) ++ Acc
end,
[],
TopicStreamGroups
).
select_streams(SubId, RankX, Streams0, S) ->
%% 1. Find the streams with the rank Y greater than the recorded one:
Streams1 =
case emqx_persistent_session_ds_state:get_rank({SubId, RankX}, S) of
undefined ->
Streams0;
ReplayedY ->
[I || I = {{_, Y}, _} <- Streams0, Y > ReplayedY]
end,
%% 2. Sort streams by rank Y:
Streams = lists:sort(
fun({{_, Y1}, _}, {{_, Y2}, _}) ->
Y1 =< Y2
end,
Streams1
),
%% 3. Select streams with the least rank Y:
case Streams of
[] ->
[];
[{{_, MinRankY}, _} | _] ->
lists:takewhile(fun({{_, Y}, _}) -> Y =:= MinRankY end, Streams)
end.
%% @doc Remove fully acked streams for the deleted subscriptions.
-spec remove_unsubscribed_streams(emqx_persistent_session_ds_state:t()) ->
emqx_persistent_session_ds_state:t().
remove_unsubscribed_streams(S0) ->
CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S0),
CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S0),
emqx_persistent_session_ds_state:fold_streams(
fun(Key, ReplayState, S1) ->
case
ReplayState#srs.unsubscribed andalso is_fully_acked(CommQos1, CommQos2, ReplayState)
of
true ->
emqx_persistent_session_ds_state:del_stream(Key, S1);
false ->
S1
end
end,
S0,
S0
).
%% @doc Advance RankY for each RankX that doesn't have any unreplayed
%% streams.
%%
%% Drop streams with the fully replayed rank. This function relies on
%% the fact that all streams with the same RankX have also the same
%% RankY.
-spec remove_fully_replayed_streams(emqx_persistent_session_ds_state:t()) ->
emqx_persistent_session_ds_state:t().
remove_fully_replayed_streams(S0) ->
CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S0),
CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S0),
%% 1. For each subscription, find the X ranks that were fully replayed:
Groups = emqx_persistent_session_ds_state:fold_streams(
fun({SubId, _Stream}, StreamState = #srs{rank_x = RankX, rank_y = RankY}, Acc) ->
Key = {SubId, RankX},
case {is_fully_replayed(CommQos1, CommQos2, StreamState), Acc} of
{_, #{Key := false}} ->
Acc;
{true, #{Key := {true, RankY}}} ->
Acc;
{true, #{Key := {true, _RankYOther}}} ->
%% assert, should never happen
error(multiple_rank_y_for_rank_x);
{true, #{}} ->
Acc#{Key => {true, RankY}};
{false, #{}} ->
Acc#{Key => false}
end
end,
#{},
S0
),
%% 2. Advance rank y for each fully replayed set of streams:
S1 = maps:fold(
fun
(Key, {true, RankY}, Acc) ->
emqx_persistent_session_ds_state:put_rank(Key, RankY, Acc);
(_, _, Acc) ->
Acc
end,
S0,
Groups
),
%% 3. Remove the fully replayed streams:
emqx_persistent_session_ds_state:fold_streams(
fun(Key = {SubId, _Stream}, #srs{rank_x = RankX, rank_y = RankY}, Acc) ->
case emqx_persistent_session_ds_state:get_rank({SubId, RankX}, Acc) of
undefined ->
Acc;
MinRankY when RankY =< MinRankY ->
?SLOG(debug, #{
msg => del_fully_preplayed_stream,
key => Key,
rank => {RankX, RankY},
min => MinRankY
}),
emqx_persistent_session_ds_state:del_stream(Key, Acc);
_ ->
Acc
end
end,
S1,
S1
).
%% @doc Compare the streams by the order in which they were replayed.
compare_streams(
{_KeyA, #srs{first_seqno_qos1 = A1, first_seqno_qos2 = A2}},
{_KeyB, #srs{first_seqno_qos1 = B1, first_seqno_qos2 = B2}}
) ->
case A1 =:= B1 of
true ->
A2 =< B2;
false ->
A1 < B1
end.
is_fully_replayed(Comm1, Comm2, S = #srs{it_end = It}) ->
It =:= end_of_stream andalso is_fully_acked(Comm1, Comm2, S).
is_fully_acked(_, _, #srs{
first_seqno_qos1 = Q1, last_seqno_qos1 = Q1, first_seqno_qos2 = Q2, last_seqno_qos2 = Q2
}) ->
%% Streams where the last chunk doesn't contain any QoS1 and 2
%% messages are considered fully acked:
true;
is_fully_acked(Comm1, Comm2, #srs{last_seqno_qos1 = S1, last_seqno_qos2 = S2}) ->
(Comm1 >= S1) andalso (Comm2 >= S2).
-spec shuffle([A]) -> [A].
shuffle(L0) ->
L1 = lists:map(
fun(A) ->
%% maybe topic/stream prioritization could be introduced here?
{rand:uniform(), A}
end,
L0
),
L2 = lists:sort(L1),
{_, L} = lists:unzip(L2),
L.

154
apps/emqx/src/emqx_persistent_session_ds_subs.erl Normal file
View File

@ -0,0 +1,154 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%% @doc This module encapsulates the data related to the client's
%% subscriptions. It tries to reppresent the subscriptions as if they
%% were a simple key-value map.
%%
%% In reality, however, the session has to retain old the
%% subscriptions for longer to ensure the consistency of message
%% replay.
-module(emqx_persistent_session_ds_subs).
%% API:
-export([on_subscribe/3, on_unsubscribe/3, gc/1, lookup/2, to_map/1, fold/3, fold_all/3]).
-export_type([]).
%%================================================================================
%% Type declarations
%%================================================================================
%%================================================================================
%% API functions
%%================================================================================
%% @doc Process a new subscription
-spec on_subscribe(
emqx_persistent_session_ds:topic_filter(),
emqx_persistent_session_ds:subscription(),
emqx_persistent_session_ds_state:t()
) ->
emqx_persistent_session_ds_state:t().
on_subscribe(TopicFilter, Subscription, S) ->
emqx_persistent_session_ds_state:put_subscription(TopicFilter, [], Subscription, S).
%% @doc Process UNSUBSCRIBE
-spec on_unsubscribe(
emqx_persistent_session_ds:topic_filter(),
emqx_persistent_session_ds:subscription(),
emqx_persistent_session_ds_state:t()
) ->
emqx_persistent_session_ds_state:t().
on_unsubscribe(TopicFilter, Subscription0, S0) ->
%% Note: we cannot delete the subscription immediately, since its
%% metadata can be used during replay (see `process_batch'). We
%% instead mark it as deleted, and let `subscription_gc' function
%% dispatch it later:
Subscription = Subscription0#{deleted => true},
emqx_persistent_session_ds_state:put_subscription(TopicFilter, [], Subscription, S0).
%% @doc Remove subscriptions that have been marked for deletion, and
%% that don't have any unacked messages:
-spec gc(emqx_persistent_session_ds_state:t()) -> emqx_persistent_session_ds_state:t().
gc(S0) ->
fold_all(
fun(TopicFilter, #{id := SubId, deleted := Deleted}, Acc) ->
case Deleted andalso has_no_unacked_streams(SubId, S0) of
true ->
emqx_persistent_session_ds_state:del_subscription(TopicFilter, [], Acc);
false ->
Acc
end
end,
S0,
S0
).
%% @doc Fold over active subscriptions:
-spec lookup(emqx_persistent_session_ds:topic_filter(), emqx_persistent_session_ds_state:t()) ->
emqx_persistent_session_ds:subscription() | undefined.
lookup(TopicFilter, S) ->
Subs = emqx_persistent_session_ds_state:get_subscriptions(S),
case emqx_topic_gbt:lookup(TopicFilter, [], Subs, undefined) of
#{deleted := true} ->
undefined;
Sub ->
Sub
end.
%% @doc Convert active subscriptions to a map, for information
%% purpose:
-spec to_map(emqx_persistent_session_ds_state:t()) -> map().
to_map(S) ->
fold(
fun(TopicFilter, #{props := Props}, Acc) -> Acc#{TopicFilter => Props} end,
#{},
S
).
%% @doc Fold over active subscriptions:
-spec fold(
fun((emqx_types:topic(), emqx_persistent_session_ds:subscription(), Acc) -> Acc),
Acc,
emqx_persistent_session_ds_state:t()
) ->
Acc.
fold(Fun, AccIn, S) ->
fold_all(
fun(TopicFilter, Sub = #{deleted := Deleted}, Acc) ->
case Deleted of
true -> Acc;
false -> Fun(TopicFilter, Sub, Acc)
end
end,
AccIn,
S
).
%% @doc Fold over all subscriptions, including inactive ones:
-spec fold_all(
fun((emqx_types:topic(), emqx_persistent_session_ds:subscription(), Acc) -> Acc),
Acc,
emqx_persistent_session_ds_state:t()
) ->
Acc.
fold_all(Fun, AccIn, S) ->
Subs = emqx_persistent_session_ds_state:get_subscriptions(S),
emqx_topic_gbt:fold(
fun(Key, Sub, Acc) -> Fun(emqx_topic_gbt:get_topic(Key), Sub, Acc) end,
AccIn,
Subs
).
%%================================================================================
%% Internal functions
%%================================================================================
-spec has_no_unacked_streams(
emqx_persistent_session_ds:subscription_id(), emqx_persistent_session_ds_state:t()
) -> boolean().
has_no_unacked_streams(SubId, S) ->
emqx_persistent_session_ds_state:fold_streams(
fun
({SID, _Stream}, Srs, Acc) when SID =:= SubId ->
emqx_persistent_session_ds_stream_scheduler:is_fully_acked(Srs, S) andalso Acc;
(_StreamKey, _Srs, Acc) ->
Acc
end,
true,
S
).

7
apps/emqx/src/emqx_persistent_session_ds_sup.erl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -48,13 +48,14 @@ init(Opts) ->
do_init(_Opts) ->
SupFlags = #{
strategy => rest_for_one,
strategy => one_for_one,
intensity => 10,
period => 2,
auto_shutdown => never
},
CoreChildren = [
worker(gc_worker, emqx_persistent_session_ds_gc_worker, [])
worker(session_gc_worker, emqx_persistent_session_ds_gc_worker, []),
worker(message_gc_worker, emqx_persistent_message_ds_gc_worker, [])
],
Children =
case mria_rlog:role() of

64
apps/emqx/src/emqx_pool.erl
View File

@ -28,11 +28,15 @@
submit/1,
submit/2,
async_submit/1,
async_submit/2
async_submit/2,
submit_to_pool/2,
submit_to_pool/3,
async_submit_to_pool/2,
async_submit_to_pool/3
]).
-ifdef(TEST).
-export([worker/0, flush_async_tasks/0]).
-export([worker/0, flush_async_tasks/0, flush_async_tasks/1]).
-endif.
%% gen_server callbacks
@ -57,7 +61,7 @@
-spec start_link(atom(), pos_integer()) -> startlink_ret().
start_link(Pool, Id) ->
gen_server:start_link(
{local, emqx_utils:proc_name(?MODULE, Id)},
{local, emqx_utils:proc_name(Pool, Id)},
?MODULE,
[Pool, Id],
[{hibernate_after, 1000}]
@ -66,32 +70,48 @@ start_link(Pool, Id) ->
%% @doc Submit work to the pool.
-spec submit(task()) -> any().
submit(Task) ->
call({submit, Task}).
submit_to_pool(?POOL, Task).
-spec submit(fun(), list(any())) -> any().
submit(Fun, Args) ->
call({submit, {Fun, Args}}).
%% @private
call(Req) ->
gen_server:call(worker(), Req, infinity).
submit_to_pool(?POOL, Fun, Args).
%% @doc Submit work to the pool asynchronously.
-spec async_submit(task()) -> ok.
async_submit(Task) ->
cast({async_submit, Task}).
async_submit_to_pool(?POOL, Task).
-spec async_submit(fun(), list(any())) -> ok.
async_submit(Fun, Args) ->
cast({async_submit, {Fun, Args}}).
async_submit_to_pool(?POOL, Fun, Args).
-spec submit_to_pool(any(), task()) -> any().
submit_to_pool(Pool, Task) ->
call(Pool, {submit, Task}).
-spec submit_to_pool(any(), fun(), list(any())) -> any().
submit_to_pool(Pool, Fun, Args) ->
call(Pool, {submit, {Fun, Args}}).
-spec async_submit_to_pool(any(), task()) -> ok.
async_submit_to_pool(Pool, Task) ->
cast(Pool, {async_submit, Task}).
-spec async_submit_to_pool(any(), fun(), list(any())) -> ok.
async_submit_to_pool(Pool, Fun, Args) ->
cast(Pool, {async_submit, {Fun, Args}}).
%% @private
cast(Msg) ->
gen_server:cast(worker(), Msg).
call(Pool, Req) ->
gen_server:call(worker(Pool), Req, infinity).
%% @private
worker() ->
gproc_pool:pick_worker(?POOL).
cast(Pool, Msg) ->
gen_server:cast(worker(Pool), Msg).
%% @private
worker(Pool) ->
gproc_pool:pick_worker(Pool).
%%--------------------------------------------------------------------
%% gen_server callbacks
@ -146,15 +166,25 @@ run(Fun) when is_function(Fun) ->
Fun().
-ifdef(TEST).
worker() ->
worker(?POOL).
flush_async_tasks() ->
flush_async_tasks(?POOL).
%% This help function creates a large enough number of async tasks
%% to force flush the pool workers.
%% The number of tasks should be large enough to ensure all workers have
%% the chance to work on at least one of the tasks.
flush_async_tasks() ->
flush_async_tasks(Pool) ->
Ref = make_ref(),
Self = self(),
L = lists:seq(1, 997),
lists:foreach(fun(I) -> emqx_pool:async_submit(fun() -> Self ! {done, Ref, I} end, []) end, L),
lists:foreach(
fun(I) -> emqx_pool:async_submit_to_pool(Pool, fun() -> Self ! {done, Ref, I} end, []) end,
L
),
lists:foreach(
fun(I) ->
receive

9
apps/emqx/src/emqx_router.erl
View File

@ -24,9 +24,7 @@
-include_lib("emqx/include/emqx_router.hrl").
%% Mnesia bootstrap
-export([mnesia/1]).
-boot_mnesia({mnesia, [boot]}).
-export([create_tables/0]).
-export([start_link/2]).
@ -123,7 +121,7 @@
%% Mnesia bootstrap
%%--------------------------------------------------------------------
mnesia(boot) ->
create_tables() ->
mria_config:set_dirty_shard(?ROUTE_SHARD, true),
ok = mria:create_table(?ROUTE_TAB, [
{type, bag},
@ -151,7 +149,8 @@ mnesia(boot) ->
{decentralized_counters, true}
]}
]}
]).
]),
[?ROUTE_TAB, ?ROUTE_TAB_FILTERS].
%%--------------------------------------------------------------------
%% Start a router

9
apps/emqx/src/emqx_router_helper.erl
View File

@ -25,9 +25,7 @@
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% Mnesia bootstrap
-export([mnesia/1]).
-boot_mnesia({mnesia, [boot]}).
-export([create_tables/0]).
%% API
-export([
@ -63,7 +61,7 @@
%% Mnesia bootstrap
%%--------------------------------------------------------------------
mnesia(boot) ->
create_tables() ->
ok = mria:create_table(?ROUTING_NODE, [
{type, set},
{rlog_shard, ?ROUTE_SHARD},
@ -71,7 +69,8 @@ mnesia(boot) ->
{record_name, routing_node},
{attributes, record_info(fields, routing_node)},
{storage_properties, [{ets, [{read_concurrency, true}]}]}
]).
]),
[?ROUTING_NODE].
%%--------------------------------------------------------------------
%% API

5
apps/emqx/src/emqx_router_sup.erl
View File

@ -24,6 +24,11 @@
start_link() ->
%% Init and log routing table type
ok = mria:wait_for_tables(
emqx_trie:create_trie() ++
emqx_router:create_tables() ++
emqx_router_helper:create_tables()
),
ok = emqx_router:init_schema(),
supervisor:start_link({local, ?MODULE}, ?MODULE, []).

149
apps/emqx/src/emqx_schema.erl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -94,6 +94,7 @@
non_empty_string/1,
validations/0,
naive_env_interpolation/1,
ensure_unicode_path/2,
validate_server_ssl_opts/1,
validate_tcp_keepalive/1,
parse_tcp_keepalive/1
@ -181,7 +182,7 @@
-define(DEFAULT_MULTIPLIER, 1.5).
-define(DEFAULT_BACKOFF, 0.75).
namespace() -> broker.
namespace() -> emqx.
tags() ->
[<<"EMQX">>].
@ -229,7 +230,7 @@ roots(high) ->
);
roots(medium) ->
[
{"broker",
{broker,
sc(
ref("broker"),
#{
@ -1103,6 +1104,14 @@ fields("ws_opts") ->
sc(
ref("deflate_opts"),
#{}
)},
{"validate_utf8",
sc(
boolean(),
#{
default => true,
desc => ?DESC(fields_ws_opts_validate_utf8)
}
)}
];
fields("tcp_opts") ->
@ -1338,24 +1347,43 @@ fields("deflate_opts") ->
];
fields("broker") ->
[
{"enable_session_registry",
{enable_session_registry,
sc(
boolean(),
#{
default => true,
importance => ?IMPORTANCE_HIGH,
desc => ?DESC(broker_enable_session_registry)
}
)},
{"session_locking_strategy",
{session_history_retain,
sc(
duration_s(),
#{
default => <<"0s">>,
importance => ?IMPORTANCE_LOW,
desc => ?DESC("broker_session_history_retain")
}
)},
{session_locking_strategy,
sc(
hoconsc:enum([local, leader, quorum, all]),
#{
default => quorum,
importance => ?IMPORTANCE_HIDDEN,
desc => ?DESC(broker_session_locking_strategy)
}
)},
shared_subscription_strategy(),
{"shared_dispatch_ack_enabled",
%% moved to under mqtt root
{shared_subscription_strategy,
sc(
string(),
#{
deprecated => {since, "5.1.0"},
importance => ?IMPORTANCE_HIDDEN
}
)},
{shared_dispatch_ack_enabled,
sc(
boolean(),
#{
@ -1365,7 +1393,7 @@ fields("broker") ->
desc => ?DESC(broker_shared_dispatch_ack_enabled)
}
)},
{"route_batch_clean",
{route_batch_clean,
sc(
boolean(),
#{
@ -1374,18 +1402,18 @@ fields("broker") ->
importance => ?IMPORTANCE_HIDDEN
}
)},
{"perf",
{perf,
sc(
ref("broker_perf"),
#{importance => ?IMPORTANCE_HIDDEN}
)},
{"routing",
{routing,
sc(
ref("broker_routing"),
#{importance => ?IMPORTANCE_HIDDEN}
)},
%% FIXME: Need new design for shared subscription group
{"shared_subscription_group",
{shared_subscription_group,
sc(
map(name, ref("shared_subscription_group")),
#{
@ -1801,7 +1829,7 @@ fields("session_persistence") ->
sc(
pos_integer(),
#{
default => 1000,
default => 100,
desc => ?DESC(session_ds_max_batch_size)
}
)},
@ -1854,6 +1882,14 @@ fields("session_persistence") ->
desc => ?DESC(session_ds_session_gc_batch_size)
}
)},
{"message_retention_period",
sc(
timeout_duration(),
#{
default => <<"1d">>,
desc => ?DESC(session_ds_message_retention_period)
}
)},
{"force_persistence",
sc(
boolean(),
@ -1882,6 +1918,16 @@ fields("session_storage_backend_builtin") ->
default => true
}
)},
{"data_dir",
sc(
string(),
#{
desc => ?DESC(session_builtin_data_dir),
mapping => "emqx_durable_storage.db_data_dir",
required => false,
importance => ?IMPORTANCE_LOW
}
)},
{"n_shards",
sc(
pos_integer(),
@ -1897,6 +1943,24 @@ fields("session_storage_backend_builtin") ->
default => 3,
importance => ?IMPORTANCE_HIDDEN
}
)},
{"egress_batch_size",
sc(
pos_integer(),
#{
default => 1000,
mapping => "emqx_durable_storage.egress_batch_size",
importance => ?IMPORTANCE_HIDDEN
}
)},
{"egress_flush_interval",
sc(
timeout_duration_ms(),
#{
default => 100,
mapping => "emqx_durable_storage.egress_flush_interval",
importance => ?IMPORTANCE_HIDDEN
}
)}
].
@ -3595,7 +3659,22 @@ mqtt_general() ->
desc => ?DESC(mqtt_shared_subscription)
}
)},
shared_subscription_strategy(),
{"shared_subscription_strategy",
sc(
hoconsc:enum([
random,
round_robin,
round_robin_per_group,
sticky,
local,
hash_topic,
hash_clientid
]),
#{
default => round_robin,
desc => ?DESC(mqtt_shared_subscription_strategy)
}
)},
{"exclusive_subscription",
sc(
boolean(),
@ -3700,6 +3779,15 @@ mqtt_session() ->
importance => ?IMPORTANCE_LOW
}
)},
{"message_expiry_interval",
sc(
hoconsc:union([duration(), infinity]),
#{
default => infinity,
desc => ?DESC(mqtt_message_expiry_interval),
importance => ?IMPORTANCE_LOW
}
)},
{"max_awaiting_rel",
sc(
hoconsc:union([non_neg_integer(), infinity]),
@ -3792,24 +3880,6 @@ mqtt_session() ->
)}
].
shared_subscription_strategy() ->
{"shared_subscription_strategy",
sc(
hoconsc:enum([
random,
round_robin,
round_robin_per_group,
sticky,
local,
hash_topic,
hash_clientid
]),
#{
default => round_robin,
desc => ?DESC(broker_shared_subscription_strategy)
}
)}.
default_mem_check_interval() ->
case emqx_os_mon:is_os_check_supported() of
true -> <<"60s">>;
@ -3836,3 +3906,20 @@ tags_schema() ->
importance => ?IMPORTANCE_LOW
}
).
ensure_unicode_path(undefined, _) ->
undefined;
ensure_unicode_path(Path, #{make_serializable := true}) ->
%% format back to serializable string
unicode:characters_to_binary(Path, utf8);
ensure_unicode_path(Path, Opts) when is_binary(Path) ->
case unicode:characters_to_list(Path, utf8) of
{R, _, _} when R =:= error orelse R =:= incomplete ->
throw({"bad_file_path_string", Path});
PathStr ->
ensure_unicode_path(PathStr, Opts)
end;
ensure_unicode_path(Path, _) when is_list(Path) ->
Path;
ensure_unicode_path(Path, _) ->
throw({"not_string", Path}).

26
apps/emqx/src/emqx_session.erl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2017-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -135,7 +135,7 @@
-type custom_timer_name() :: atom().
-type message() :: emqx_types:message().
-type publish() :: {maybe(emqx_types:packet_id()), emqx_types:message()}.
-type publish() :: {option(emqx_types:packet_id()), emqx_types:message()}.
-type pubrel() :: {pubrel, emqx_types:packet_id()}.
-type reply() :: publish() | pubrel().
-type replies() :: [reply()] | reply().
@ -409,12 +409,8 @@ enrich_delivers(ClientInfo, Delivers, Session) ->
enrich_delivers(_ClientInfo, [], _UpgradeQoS, _Session) ->
[];
enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) ->
case enrich_deliver(ClientInfo, D, UpgradeQoS, Session) of
[] ->
enrich_delivers(ClientInfo, Rest, UpgradeQoS, Session);
Msg ->
[Msg | enrich_delivers(ClientInfo, Rest, UpgradeQoS, Session)]
end.
enrich_deliver(ClientInfo, D, UpgradeQoS, Session) ++
enrich_delivers(ClientInfo, Rest, UpgradeQoS, Session).
enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) ->
SubOpts =
@ -435,13 +431,15 @@ enrich_message(
_ = emqx_session_events:handle_event(ClientInfo, {dropped, Msg, no_local}),
[];
enrich_message(_ClientInfo, MsgIn, SubOpts = #{}, UpgradeQoS) ->
maps:fold(
fun(SubOpt, V, Msg) -> enrich_subopts(SubOpt, V, Msg, UpgradeQoS) end,
MsgIn,
SubOpts
);
[
maps:fold(
fun(SubOpt, V, Msg) -> enrich_subopts(SubOpt, V, Msg, UpgradeQoS) end,
MsgIn,
SubOpts
)
];
enrich_message(_ClientInfo, Msg, undefined, _UpgradeQoS) ->
Msg.
[Msg].
enrich_subopts(nl, 1, Msg, _) ->
emqx_message:set_flag(nl, Msg);

4
apps/emqx/src/emqx_session_events.erl
View File

@ -62,10 +62,10 @@ handle_event(ClientInfo, {dropped, Msg, #{reason := queue_full, logctx := Ctx}})
ok = emqx_metrics:inc('delivery.dropped.queue_full'),
ok = inc_pd('send_msg.dropped', 1),
ok = inc_pd('send_msg.dropped.queue_full', 1),
?SLOG(
?SLOG_THROTTLE(
warning,
Ctx#{
msg => "dropped_msg_due_to_mqueue_is_full",
msg => dropped_msg_due_to_mqueue_is_full,
payload => Msg#message.payload
},
#{topic => Msg#message.topic}

8
apps/emqx/src/emqx_session_mem.erl
View File

@ -468,12 +468,12 @@ dequeue(ClientInfo, Session = #session{inflight = Inflight, mqueue = Q}) ->
dequeue(_ClientInfo, 0, Msgs, Q) ->
{lists:reverse(Msgs), Q};
dequeue(ClientInfo, Cnt, Msgs, Q) ->
dequeue(ClientInfo = #{zone := Zone}, Cnt, Msgs, Q) ->
case emqx_mqueue:out(Q) of
{empty, _Q} ->
dequeue(ClientInfo, 0, Msgs, Q);
{{value, Msg}, Q1} ->
case emqx_message:is_expired(Msg) of
case emqx_message:is_expired(Msg, Zone) of
true ->
_ = emqx_session_events:handle_event(ClientInfo, {expired, Msg}),
dequeue(ClientInfo, Cnt, Msgs, Q1);
@ -619,14 +619,14 @@ retry_delivery(
end.
do_retry_delivery(
ClientInfo,
ClientInfo = #{zone := Zone},
PacketId,
#inflight_data{phase = wait_ack, message = Msg} = Data,
Now,
Acc,
Inflight
) ->
case emqx_message:is_expired(Msg) of
case emqx_message:is_expired(Msg, Zone) of
true ->
_ = emqx_session_events:handle_event(ClientInfo, {expired, Msg}),
{Acc, emqx_inflight:delete(PacketId, Inflight)};

9
apps/emqx/src/emqx_shared_sub.erl
View File

@ -25,9 +25,7 @@
-include("types.hrl").
%% Mnesia bootstrap
-export([mnesia/1]).
-boot_mnesia({mnesia, [boot]}).
-export([create_tables/0]).
%% APIs
-export([start_link/0]).
@ -107,14 +105,15 @@
%% Mnesia bootstrap
%%--------------------------------------------------------------------
mnesia(boot) ->
create_tables() ->
ok = mria:create_table(?TAB, [
{type, bag},
{rlog_shard, ?SHARED_SUB_SHARD},
{storage, ram_copies},
{record_name, emqx_shared_subscription},
{attributes, record_info(fields, emqx_shared_subscription)}
]).
]),
[?TAB].
%%--------------------------------------------------------------------
%% API

10
apps/emqx/src/emqx_stats.erl
View File

@ -62,7 +62,7 @@
-record(update, {name, countdown, interval, func}).
-record(state, {
timer :: maybe(reference()),
timer :: option(reference()),
updates :: [#update{}],
tick_ms :: timeout()
}).
@ -99,7 +99,11 @@
[
'sessions.count',
%% Maximum Number of Concurrent Sessions
'sessions.max'
'sessions.max',
%% Count of Sessions in the cluster
'cluster_sessions.count',
%% Maximum Number of Sessions in the cluster
'cluster_sessions.max'
]
).
@ -164,6 +168,8 @@ names() ->
emqx_connections_max,
emqx_live_connections_count,
emqx_live_connections_max,
emqx_cluster_sessions_count,
emqx_cluster_sessions_max,
emqx_sessions_count,
emqx_sessions_max,
emqx_channels_count,

4
apps/emqx/src/emqx_sys.erl
View File

@ -65,8 +65,8 @@
-import(emqx_utils, [start_timer/2]).
-record(state, {
heartbeat :: maybe(reference()),
ticker :: maybe(reference()),
heartbeat :: option(reference()),
ticker :: option(reference()),
sysdescr :: binary()
}).

1
apps/emqx/src/emqx_sys_sup.erl
View File

@ -22,6 +22,7 @@
-export([init/1]).
start_link() ->
_ = mria:wait_for_tables(emqx_alarm:create_tables()),
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
init([]) ->

20
apps/emqx/src/emqx_topic_gbt.erl
View File

@ -1,5 +1,5 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
@ -39,11 +39,11 @@
-type match(ID) :: key(ID).
-opaque t(ID, Value) :: gb_trees:tree(key(ID), Value).
-opaque t() :: t(_ID, _Value).
-type t() :: t(_ID, _Value).
%% @doc Create a new gb_tree and store it in the persitent_term with the
%% given name.
-spec new() -> t().
-spec new() -> t(_ID, _Value).
new() ->
gb_trees:empty().
@ -54,19 +54,19 @@ size(Gbt) ->
%% @doc Insert a new entry into the index that associates given topic filter to given
%% record ID, and attaches arbitrary record to the entry. This allows users to choose
%% between regular and "materialized" indexes, for example.
-spec insert(emqx_types:topic() | words(), _ID, _Record, t()) -> t().
-spec insert(emqx_types:topic() | words(), ID, Record, t(ID, Record)) -> t(ID, Record).
insert(Filter, ID, Record, Gbt) ->
Key = key(Filter, ID),
gb_trees:enter(Key, Record, Gbt).
%% @doc Delete an entry from the index that associates given topic filter to given
%% record ID. Deleting non-existing entry is not an error.
-spec delete(emqx_types:topic() | words(), _ID, t()) -> t().
-spec delete(emqx_types:topic() | words(), ID, t(ID, Record)) -> t(ID, Record).
delete(Filter, ID, Gbt) ->
Key = key(Filter, ID),
gb_trees:delete_any(Key, Gbt).
-spec lookup(emqx_types:topic() | words(), _ID, t(), Default) -> _Record | Default.
-spec lookup(emqx_types:topic() | words(), ID, t(ID, Record), Default) -> Record | Default.
lookup(Filter, ID, Gbt, Default) ->
Key = key(Filter, ID),
case gb_trees:lookup(Key, Gbt) of
@ -76,7 +76,7 @@ lookup(Filter, ID, Gbt, Default) ->
Default
end.
-spec fold(fun((key(_ID), _Record, Acc) -> Acc), Acc, t()) -> Acc.
-spec fold(fun((key(ID), Record, Acc) -> Acc), Acc, t(ID, Record)) -> Acc.
fold(Fun, Acc, Gbt) ->
Iter = gb_trees:iterator(Gbt),
fold_iter(Fun, Acc, Iter).
@ -91,13 +91,13 @@ fold_iter(Fun, Acc, Iter) ->
%% @doc Match given topic against the index and return the first match, or `false` if
%% no match is found.
-spec match(emqx_types:topic(), t()) -> match(_ID) | false.
-spec match(emqx_types:topic(), t(ID, _Record)) -> match(ID) | false.
match(Topic, Gbt) ->
emqx_trie_search:match(Topic, make_nextf(Gbt)).
%% @doc Match given topic against the index and return _all_ matches.
%% If `unique` option is given, return only unique matches by record ID.
-spec matches(emqx_types:topic(), t(), emqx_trie_search:opts()) -> [match(_ID)].
-spec matches(emqx_types:topic(), t(ID, _Record), emqx_trie_search:opts()) -> [match(ID)].
matches(Topic, Gbt, Opts) ->
emqx_trie_search:matches(Topic, make_nextf(Gbt), Opts).
@ -112,7 +112,7 @@ get_topic(Key) ->
emqx_trie_search:get_topic(Key).
%% @doc Fetch the record associated with the match.
-spec get_record(match(_ID), t()) -> _Record.
-spec get_record(match(ID), t(ID, Record)) -> Record.
get_record(Key, Gbt) ->
gb_trees:get(Key, Gbt).

11
apps/emqx/src/emqx_trie.erl
View File

@ -20,13 +20,11 @@
%% Mnesia bootstrap
-export([
mnesia/1,
create_trie/0,
wait_for_tables/0,
create_session_trie/1
]).
-boot_mnesia({mnesia, [boot]}).
%% Trie APIs
-export([
insert/1,
@ -65,8 +63,8 @@
%%--------------------------------------------------------------------
%% @doc Create or replicate topics table.
-spec mnesia(boot | copy) -> ok.
mnesia(boot) ->
-spec create_trie() -> [mria:table()].
create_trie() ->
%% Optimize storage
StoreProps = [
{ets, [
@ -80,7 +78,8 @@ mnesia(boot) ->
{attributes, record_info(fields, ?TRIE)},
{type, ordered_set},
{storage_properties, StoreProps}
]).
]),
[?TRIE].
create_session_trie(Type) ->
Storage =

25
apps/emqx/src/emqx_types.erl
View File

@ -100,6 +100,7 @@
-export_type([
banned/0,
banned_who/0,
command/0
]).
@ -173,7 +174,7 @@
atom() => term()
}.
-type clientinfo() :: #{
zone := maybe(zone()),
zone := option(zone()),
protocol := protocol(),
peerhost := peerhost(),
sockport := non_neg_integer(),
@ -181,9 +182,9 @@
username := username(),
is_bridge := boolean(),
is_superuser := boolean(),
mountpoint := maybe(binary()),
ws_cookie => maybe(list()),
password => maybe(binary()),
mountpoint := option(binary()),
ws_cookie => option(list()),
password => option(binary()),
auth_result => auth_result(),
anonymous => boolean(),
cn => binary(),
@ -191,8 +192,8 @@
atom() => term()
}.
-type clientid() :: binary() | atom().
-type username() :: maybe(binary()).
-type password() :: maybe(binary()).
-type username() :: option(binary()).
-type password() :: option(binary()).
-type peerhost() :: inet:ip_address().
-type peername() ::
{inet:ip_address(), inet:port_number()}
@ -222,8 +223,8 @@
-type packet_id() :: 1..16#FFFF.
-type alias_id() :: 0..16#FFFF.
-type topic_aliases() :: #{
inbound => maybe(map()),
outbound => maybe(map())
inbound => option(map()),
outbound => option(map())
}.
-type properties() :: #{atom() => term()}.
-type topic_filters() :: list({topic(), subopts()}).
@ -246,6 +247,14 @@
}.
-type banned() :: #banned{}.
-type banned_who() ::
{clientid, binary()}
| {peerhost, inet:ip_address()}
| {username, binary()}
| {clientid_re, {_RE :: tuple(), binary()}}
| {username_re, {_RE :: tuple(), binary()}}
| {peerhost_net, esockd_cidr:cidr()}.
-type deliver() :: {deliver, topic(), message()}.
-type delivery() :: #delivery{}.
-type deliver_result() :: ok | {ok, non_neg_integer()} | {error, term()}.

9
apps/emqx/src/emqx_ws_connection.erl
View File

@ -76,15 +76,15 @@
%% Channel
channel :: emqx_channel:channel(),
%% GC State
gc_state :: maybe(emqx_gc:gc_state()),
gc_state :: option(emqx_gc:gc_state()),
%% Postponed Packets|Cmds|Events
postponed :: list(emqx_types:packet() | ws_cmd() | tuple()),
%% Stats Timer
stats_timer :: disabled | maybe(reference()),
stats_timer :: disabled | option(reference()),
%% Idle Timeout
idle_timeout :: timeout(),
%% Idle Timer
idle_timer :: maybe(reference()),
idle_timer :: option(reference()),
%% Zone name
zone :: atom(),
%% Listener Type and Name
@ -205,7 +205,8 @@ init(Req, #{listener := {Type, Listener}} = Opts) ->
compress => get_ws_opts(Type, Listener, compress),
deflate_opts => get_ws_opts(Type, Listener, deflate_opts),
max_frame_size => get_ws_opts(Type, Listener, max_frame_size),
idle_timeout => get_ws_opts(Type, Listener, idle_timeout)
idle_timeout => get_ws_opts(Type, Listener, idle_timeout),
validate_utf8 => get_ws_opts(Type, Listener, validate_utf8)
},
case check_origin_header(Req, Opts) of
{error, Reason} ->

95
apps/emqx/test/emqx_banned_SUITE.erl
View File

@ -34,7 +34,7 @@ end_per_suite(Config) ->
t_add_delete(_) ->
Banned = #banned{
who = {clientid, <<"TestClient">>},
who = emqx_banned:who(clientid, <<"TestClient">>),
by = <<"banned suite">>,
reason = <<"test">>,
at = erlang:system_time(second),
@ -47,54 +47,91 @@ t_add_delete(_) ->
emqx_banned:create(Banned#banned{until = erlang:system_time(second) + 100}),
?assertEqual(1, emqx_banned:info(size)),
ok = emqx_banned:delete({clientid, <<"TestClient">>}),
ok = emqx_banned:delete(emqx_banned:who(clientid, <<"TestClient">>)),
?assertEqual(0, emqx_banned:info(size)).
t_check(_) ->
{ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient">>}}),
{ok, _} = emqx_banned:create(#banned{who = {username, <<"BannedUser">>}}),
{ok, _} = emqx_banned:create(#banned{who = {peerhost, {192, 168, 0, 1}}}),
?assertEqual(3, emqx_banned:info(size)),
ClientInfo1 = #{
{ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(clientid, <<"BannedClient">>)}),
{ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(username, <<"BannedUser">>)}),
{ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(peerhost, {192, 168, 0, 1})}),
{ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(peerhost, <<"192.168.0.2">>)}),
{ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(clientid_re, <<"BannedClientRE.*">>)}),
{ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(username_re, <<"BannedUserRE.*">>)}),
{ok, _} = emqx_banned:create(#banned{who = emqx_banned:who(peerhost_net, <<"192.168.3.0/24">>)}),
?assertEqual(7, emqx_banned:info(size)),
ClientInfoBannedClientId = #{
clientid => <<"BannedClient">>,
username => <<"user">>,
peerhost => {127, 0, 0, 1}
},
ClientInfo2 = #{
ClientInfoBannedUsername = #{
clientid => <<"client">>,
username => <<"BannedUser">>,
peerhost => {127, 0, 0, 1}
},
ClientInfo3 = #{
ClientInfoBannedAddr1 = #{
clientid => <<"client">>,
username => <<"user">>,
peerhost => {192, 168, 0, 1}
},
ClientInfo4 = #{
ClientInfoBannedAddr2 = #{
clientid => <<"client">>,
username => <<"user">>,
peerhost => {192, 168, 0, 2}
},
ClientInfoBannedClientIdRE = #{
clientid => <<"BannedClientRE1">>,
username => <<"user">>,
peerhost => {127, 0, 0, 1}
},
ClientInfoBannedUsernameRE = #{
clientid => <<"client">>,
username => <<"BannedUserRE1">>,
peerhost => {127, 0, 0, 1}
},
ClientInfoBannedAddrNet = #{
clientid => <<"client">>,
username => <<"user">>,
peerhost => {192, 168, 3, 1}
},
ClientInfoValidFull = #{
clientid => <<"client">>,
username => <<"user">>,
peerhost => {127, 0, 0, 1}
},
ClientInfo5 = #{},
ClientInfo6 = #{clientid => <<"client1">>},
?assert(emqx_banned:check(ClientInfo1)),
?assert(emqx_banned:check(ClientInfo2)),
?assert(emqx_banned:check(ClientInfo3)),
?assertNot(emqx_banned:check(ClientInfo4)),
?assertNot(emqx_banned:check(ClientInfo5)),
?assertNot(emqx_banned:check(ClientInfo6)),
ok = emqx_banned:delete({clientid, <<"BannedClient">>}),
ok = emqx_banned:delete({username, <<"BannedUser">>}),
ok = emqx_banned:delete({peerhost, {192, 168, 0, 1}}),
?assertNot(emqx_banned:check(ClientInfo1)),
?assertNot(emqx_banned:check(ClientInfo2)),
?assertNot(emqx_banned:check(ClientInfo3)),
?assertNot(emqx_banned:check(ClientInfo4)),
ClientInfoValidEmpty = #{},
ClientInfoValidOnlyClientId = #{clientid => <<"client1">>},
?assert(emqx_banned:check(ClientInfoBannedClientId)),
?assert(emqx_banned:check(ClientInfoBannedUsername)),
?assert(emqx_banned:check(ClientInfoBannedAddr1)),
?assert(emqx_banned:check(ClientInfoBannedAddr2)),
?assert(emqx_banned:check(ClientInfoBannedClientIdRE)),
?assert(emqx_banned:check(ClientInfoBannedUsernameRE)),
?assert(emqx_banned:check(ClientInfoBannedAddrNet)),
?assertNot(emqx_banned:check(ClientInfoValidFull)),
?assertNot(emqx_banned:check(ClientInfoValidEmpty)),
?assertNot(emqx_banned:check(ClientInfoValidOnlyClientId)),
ok = emqx_banned:delete(emqx_banned:who(clientid, <<"BannedClient">>)),
ok = emqx_banned:delete(emqx_banned:who(username, <<"BannedUser">>)),
ok = emqx_banned:delete(emqx_banned:who(peerhost, {192, 168, 0, 1})),
ok = emqx_banned:delete(emqx_banned:who(peerhost, <<"192.168.0.2">>)),
ok = emqx_banned:delete(emqx_banned:who(clientid_re, <<"BannedClientRE.*">>)),
ok = emqx_banned:delete(emqx_banned:who(username_re, <<"BannedUserRE.*">>)),
ok = emqx_banned:delete(emqx_banned:who(peerhost_net, <<"192.168.3.0/24">>)),
?assertNot(emqx_banned:check(ClientInfoBannedClientId)),
?assertNot(emqx_banned:check(ClientInfoBannedUsername)),
?assertNot(emqx_banned:check(ClientInfoBannedAddr1)),
?assertNot(emqx_banned:check(ClientInfoBannedAddr2)),
?assertNot(emqx_banned:check(ClientInfoBannedClientIdRE)),
?assertNot(emqx_banned:check(ClientInfoBannedUsernameRE)),
?assertNot(emqx_banned:check(ClientInfoBannedAddrNet)),
?assertNot(emqx_banned:check(ClientInfoValidFull)),
?assertEqual(0, emqx_banned:info(size)).
t_unused(_) ->
Who1 = {clientid, <<"BannedClient1">>},
Who2 = {clientid, <<"BannedClient2">>},
Who1 = emqx_banned:who(clientid, <<"BannedClient1">>),
Who2 = emqx_banned:who(clientid, <<"BannedClient2">>),
?assertMatch(
{ok, _},
@ -123,7 +160,7 @@ t_kick(_) ->
snabbkaffe:start_trace(),
Now = erlang:system_time(second),
Who = {clientid, ClientId},
Who = emqx_banned:who(clientid, ClientId),
emqx_banned:create(#{
who => Who,
@ -194,7 +231,7 @@ t_session_taken(_) ->
Publish(),
Now = erlang:system_time(second),
Who = {clientid, ClientId2},
Who = emqx_banned:who(clientid, ClientId2),
emqx_banned:create(#{
who => Who,
by => <<"test">>,

27
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -427,19 +427,32 @@ t_handle_in_auth(_) ->
t_handle_in_frame_error(_) ->
IdleChannel = channel(#{conn_state => idle}),
{shutdown, #{shutdown_count := frame_error, reason := frame_too_large}, _Chan} =
emqx_channel:handle_in({frame_error, frame_too_large}, IdleChannel),
{shutdown, #{shutdown_count := frame_too_large, cause := frame_too_large}, _Chan} =
emqx_channel:handle_in({frame_error, #{cause => frame_too_large}}, IdleChannel),
ConnectingChan = channel(#{conn_state => connecting}),
ConnackPacket = ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE),
{shutdown, #{shutdown_count := frame_error, reason := frame_too_large}, ConnackPacket, _} =
emqx_channel:handle_in({frame_error, frame_too_large}, ConnectingChan),
{shutdown,
#{
shutdown_count := frame_too_large,
cause := frame_too_large,
limit := 100,
received := 101
},
ConnackPacket,
_} =
emqx_channel:handle_in(
{frame_error, #{cause => frame_too_large, received => 101, limit => 100}},
ConnectingChan
),
DisconnectPacket = ?DISCONNECT_PACKET(?RC_PACKET_TOO_LARGE),
ConnectedChan = channel(#{conn_state => connected}),
{ok, [{outgoing, DisconnectPacket}, {close, frame_too_large}], _} =
emqx_channel:handle_in({frame_error, frame_too_large}, ConnectedChan),
?assertMatch(
{ok, [{outgoing, DisconnectPacket}, {close, frame_too_large}], _},
emqx_channel:handle_in({frame_error, #{cause => frame_too_large}}, ConnectedChan)
),
DisconnectedChan = channel(#{conn_state => disconnected}),
{ok, DisconnectedChan} =
emqx_channel:handle_in({frame_error, frame_too_large}, DisconnectedChan).
emqx_channel:handle_in({frame_error, #{cause => frame_too_large}}, DisconnectedChan).
t_handle_in_expected_packet(_) ->
Packet = ?DISCONNECT_PACKET(?RC_PROTOCOL_ERROR),

29
apps/emqx/test/emqx_client_SUITE.erl
View File

@ -72,7 +72,7 @@ groups() ->
t_dollar_topics,
t_sub_non_utf8_topic
]},
{mqttv5, [non_parallel_tests], [t_basic_with_props_v5]},
{mqttv5, [non_parallel_tests], [t_basic_with_props_v5, t_v5_receive_maximim_in_connack]},
{others, [non_parallel_tests], [
t_username_as_clientid,
t_certcn_as_clientid_default_config_tls,
@ -103,14 +103,14 @@ end_per_testcase(_Case, _Config) ->
%%--------------------------------------------------------------------
t_basic_v3(_) ->
t_basic([{proto_ver, v3}]).
run_basic([{proto_ver, v3}]).
%%--------------------------------------------------------------------
%% Test cases for MQTT v4
%%--------------------------------------------------------------------
t_basic_v4(_Config) ->
t_basic([{proto_ver, v4}]).
run_basic([{proto_ver, v4}]).
t_cm(_) ->
emqx_config:put_zone_conf(default, [mqtt, idle_timeout], 1000),
@ -335,19 +335,30 @@ t_sub_non_utf8_topic(_) ->
%% Test cases for MQTT v5
%%--------------------------------------------------------------------
t_basic_with_props_v5(_) ->
t_basic([
v5_conn_props(ReceiveMaximum) ->
[
{proto_ver, v5},
{properties, #{'Receive-Maximum' => 4}}
]).
{properties, #{'Receive-Maximum' => ReceiveMaximum}}
].
t_basic_with_props_v5(_) ->
run_basic(v5_conn_props(4)).
t_v5_receive_maximim_in_connack(_) ->
ReceiveMaximum = 7,
{ok, C} = emqtt:start_link(v5_conn_props(ReceiveMaximum)),
{ok, Props} = emqtt:connect(C),
?assertMatch(#{'Receive-Maximum' := ReceiveMaximum}, Props),
ok = emqtt:disconnect(C),
ok.
%%--------------------------------------------------------------------
%% General test cases.
%%--------------------------------------------------------------------
t_basic(_Opts) ->
run_basic(Opts) ->
Topic = nth(1, ?TOPICS),
{ok, C} = emqtt:start_link([{proto_ver, v4}]),
{ok, C} = emqtt:start_link(Opts),
{ok, _} = emqtt:connect(C),
{ok, _, [1]} = emqtt:subscribe(C, Topic, qos1),
{ok, _, [2]} = emqtt:subscribe(C, Topic, qos2),

24
apps/emqx/test/emqx_cm_SUITE.erl
View File

@ -221,7 +221,7 @@ t_open_session_race_condition(_) ->
end,
%% sync
ignored = gen_server:call(?CM, ignore, infinity),
ok = emqx_pool:flush_async_tasks(),
ok = emqx_pool:flush_async_tasks(?CM_POOL),
?assertEqual([], emqx_cm:lookup_channels(ClientId)).
t_kick_session_discard_normal(_) ->
@ -343,29 +343,9 @@ test_stepdown_session(Action, Reason) ->
end,
% sync
ignored = gen_server:call(?CM, ignore, infinity),
ok = flush_emqx_pool(),
ok = emqx_pool:flush_async_tasks(?CM_POOL),
?assertEqual([], emqx_cm:lookup_channels(ClientId)).
%% Channel deregistration is delegated to emqx_pool as a sync tasks.
%% The emqx_pool is pool of workers, and there is no way to know
%% which worker was picked for the last deregistration task.
%% This help function creates a large enough number of async tasks
%% to sync with the pool workers.
%% The number of tasks should be large enough to ensure all workers have
%% the chance to work on at least one of the tasks.
flush_emqx_pool() ->
Self = self(),
L = lists:seq(1, 1000),
lists:foreach(fun(I) -> emqx_pool:async_submit(fun() -> Self ! {done, I} end, []) end, L),
lists:foreach(
fun(I) ->
receive
{done, I} -> ok
end
end,
L
).
t_discard_session_race(_) ->
ClientId = rand_client_id(),
?check_trace(

100
apps/emqx/test/emqx_cm_registry_keeper_SUITE.erl Normal file
View File

@ -0,0 +1,100 @@
%%--------------------------------------------------------------------
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
-module(emqx_cm_registry_keeper_SUITE).
-compile(export_all).
-compile(nowarn_export_all).
-include_lib("eunit/include/eunit.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-include("emqx_cm.hrl").
%%--------------------------------------------------------------------
%% CT callbacks
%%--------------------------------------------------------------------
all() -> emqx_common_test_helpers:all(?MODULE).
init_per_suite(Config) ->
AppConfig = "broker.session_history_retain = 2s",
Apps = emqx_cth_suite:start(
[{emqx, #{config => AppConfig}}],
#{work_dir => emqx_cth_suite:work_dir(Config)}
),
[{apps, Apps} | Config].
end_per_suite(Config) ->
emqx_cth_suite:stop(proplists:get_value(apps, Config)).
init_per_testcase(_TestCase, Config) ->
Config.
end_per_testcase(_TestCase, Config) ->
Config.
t_cleanup_after_retain(_) ->
Pid = spawn(fun() ->
receive
stop -> ok
end
end),
ClientId = <<"clientid">>,
ClientId2 = <<"clientid2">>,
emqx_cm_registry:register_channel({ClientId, Pid}),
emqx_cm_registry:register_channel({ClientId2, Pid}),
?assertEqual([Pid], emqx_cm_registry:lookup_channels(ClientId)),
?assertEqual([Pid], emqx_cm_registry:lookup_channels(ClientId2)),
?assertEqual(2, emqx_cm_registry_keeper:count(0)),
T0 = erlang:system_time(seconds),
exit(Pid, kill),
%% lookup_channel chesk if the channel is still alive
?assertEqual([], emqx_cm_registry:lookup_channels(ClientId)),
?assertEqual([], emqx_cm_registry:lookup_channels(ClientId2)),
%% simulate a DOWN message which causes emqx_cm to call clean_down
%% to clean the channels for real
ok = emqx_cm:clean_down({Pid, ClientId}),
ok = emqx_cm:clean_down({Pid, ClientId2}),
?assertEqual(2, emqx_cm_registry_keeper:count(T0)),
?assertEqual(2, emqx_cm_registry_keeper:count(0)),
?retry(_Interval = 1000, _Attempts = 4, begin
?assertEqual(0, emqx_cm_registry_keeper:count(T0)),
?assertEqual(0, emqx_cm_registry_keeper:count(0))
end),
ok.
%% count is cached when the number of entries is greater than 1000
t_count_cache(_) ->
Pid = self(),
ClientsCount = 999,
ClientIds = lists:map(fun erlang:integer_to_binary/1, lists:seq(1, ClientsCount)),
Channels = lists:map(fun(ClientId) -> {ClientId, Pid} end, ClientIds),
lists:foreach(
fun emqx_cm_registry:register_channel/1,
Channels
),
T0 = erlang:system_time(seconds),
?assertEqual(ClientsCount, emqx_cm_registry_keeper:count(0)),
?assertEqual(ClientsCount, emqx_cm_registry_keeper:count(T0)),
%% insert another one to trigger the cache threshold
emqx_cm_registry:register_channel({<<"-1">>, Pid}),
?assertEqual(ClientsCount + 1, emqx_cm_registry_keeper:count(0)),
?assertEqual(ClientsCount, emqx_cm_registry_keeper:count(T0)),
mnesia:clear_table(?CHAN_REG_TAB),
ok.
channel(Id, Pid) ->
#channel{chid = Id, pid = Pid}.

Some files were not shown because too many files have changed in this diff Show More