Compare commits
19 Commits
master
...
release-58
Author | SHA1 | Date |
---|---|---|
![]() |
9d6954cf60 | |
![]() |
9a2f878017 | |
![]() |
c35661f484 | |
![]() |
9abdff60a1 | |
![]() |
a849e6df4c | |
![]() |
c00b178b57 | |
![]() |
f085973778 | |
![]() |
a8882bd7fd | |
![]() |
7711307909 | |
![]() |
6849801293 | |
![]() |
6c4cfeed92 | |
![]() |
de9e619c96 | |
![]() |
8d88d14f0a | |
![]() |
ff72d55491 | |
![]() |
42e4a635e0 | |
![]() |
26ddc403c8 | |
![]() |
4971fd3eaf | |
![]() |
5b15886836 | |
![]() |
10dadbad3b |
|
@ -10,7 +10,7 @@ services:
|
|||
nofile: 1024
|
||||
image: openldap
|
||||
#ports:
|
||||
# - "389:389"
|
||||
# - 389:389
|
||||
volumes:
|
||||
- ./certs/ca.crt:/etc/certs/ca.crt
|
||||
restart: always
|
||||
|
|
|
@ -51,7 +51,7 @@ runs:
|
|||
echo "SELF_HOSTED=false" >> $GITHUB_OUTPUT
|
||||
;;
|
||||
esac
|
||||
- uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
|
||||
- uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1
|
||||
id: cache
|
||||
if: steps.prepare.outputs.SELF_HOSTED != 'true'
|
||||
with:
|
||||
|
|
|
@ -152,7 +152,7 @@ jobs:
|
|||
echo "PROFILE=${PROFILE}" | tee -a .env
|
||||
echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env
|
||||
zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip .
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
path: ${{ matrix.profile }}.zip
|
||||
|
|
|
@ -163,7 +163,7 @@ jobs:
|
|||
echo "PROFILE=${PROFILE}" | tee -a .env
|
||||
echo "PKG_VSN=$(./pkg-vsn.sh ${PROFILE})" | tee -a .env
|
||||
zip -ryq -x@.github/workflows/.zipignore $PROFILE.zip .
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
path: ${{ matrix.profile }}.zip
|
||||
|
|
|
@ -83,7 +83,7 @@ jobs:
|
|||
id: build
|
||||
run: |
|
||||
make ${{ matrix.profile }}-tgz
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: "${{ matrix.profile }}-${{ matrix.arch }}.tar.gz"
|
||||
path: "_packages/emqx*/emqx-*.tar.gz"
|
||||
|
@ -110,7 +110,7 @@ jobs:
|
|||
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
|
||||
with:
|
||||
ref: ${{ github.event.inputs.ref }}
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
pattern: "${{ matrix.profile[0] }}-*.tar.gz"
|
||||
path: _packages
|
||||
|
@ -122,25 +122,24 @@ jobs:
|
|||
run: |
|
||||
ls -lR _packages/$PROFILE
|
||||
mv _packages/$PROFILE/*.tar.gz ./
|
||||
|
||||
- name: Enable containerd image store on Docker Engine
|
||||
run: |
|
||||
echo "$(sudo cat /etc/docker/daemon.json | jq '. += {"features": {"containerd-snapshotter": true}}')" > daemon.json
|
||||
echo "$(jq '. += {"features": {"containerd-snapshotter": true}}' /etc/docker/daemon.json)" > daemon.json
|
||||
sudo mv daemon.json /etc/docker/daemon.json
|
||||
sudo systemctl restart docker
|
||||
|
||||
- uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
|
||||
- uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
|
||||
- uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
|
||||
- uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
|
||||
|
||||
- name: Login to hub.docker.com
|
||||
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
|
||||
uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
|
||||
if: inputs.publish && contains(matrix.profile[1], 'docker.io')
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USER }}
|
||||
password: ${{ secrets.DOCKER_HUB_TOKEN }}
|
||||
|
||||
- name: Login to AWS ECR
|
||||
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
|
||||
uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
|
||||
if: inputs.publish && contains(matrix.profile[1], 'public.ecr.aws')
|
||||
with:
|
||||
registry: public.ecr.aws
|
||||
|
|
|
@ -51,7 +51,7 @@ jobs:
|
|||
if: always()
|
||||
run: |
|
||||
docker save $_EMQX_DOCKER_IMAGE_TAG | gzip > $EMQX_NAME-docker-$PKG_VSN.tar.gz
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: "${{ env.EMQX_NAME }}-docker"
|
||||
path: "${{ env.EMQX_NAME }}-docker-${{ env.PKG_VSN }}.tar.gz"
|
||||
|
|
|
@ -95,7 +95,7 @@ jobs:
|
|||
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
|
||||
apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
|
||||
apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: success()
|
||||
with:
|
||||
name: ${{ matrix.profile }}-${{ matrix.os }}-${{ matrix.otp }}
|
||||
|
@ -180,7 +180,7 @@ jobs:
|
|||
--builder $BUILDER \
|
||||
--elixir $IS_ELIXIR \
|
||||
--pkgtype pkg
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: ${{ matrix.profile }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.with_elixir == 'yes' && '-elixir' || '' }}-${{ matrix.builder }}-${{ matrix.otp }}-${{ matrix.elixir }}
|
||||
path: _packages/${{ matrix.profile }}/
|
||||
|
@ -198,7 +198,7 @@ jobs:
|
|||
profile:
|
||||
- ${{ inputs.profile }}
|
||||
steps:
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
pattern: "${{ matrix.profile }}-*"
|
||||
path: packages/${{ matrix.profile }}
|
||||
|
|
|
@ -54,7 +54,7 @@ jobs:
|
|||
- name: build pkg
|
||||
run: |
|
||||
./scripts/buildx.sh --profile "$PROFILE" --pkgtype pkg --builder "$BUILDER"
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: success()
|
||||
with:
|
||||
name: ${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.os }}
|
||||
|
@ -102,7 +102,7 @@ jobs:
|
|||
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
|
||||
apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
|
||||
apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: success()
|
||||
with:
|
||||
name: ${{ matrix.profile }}-${{ matrix.os }}
|
||||
|
|
|
@ -41,13 +41,13 @@ jobs:
|
|||
- name: build pkg
|
||||
run: |
|
||||
./scripts/buildx.sh --profile $PROFILE --pkgtype pkg --elixir $ELIXIR --arch $ARCH
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
|
||||
path: _packages/${{ matrix.profile[0] }}/*
|
||||
retention-days: 7
|
||||
compression-level: 0
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: "${{ matrix.profile[0] }}-schema-dump-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
|
||||
path: |
|
||||
|
@ -84,7 +84,7 @@ jobs:
|
|||
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
|
||||
apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
|
||||
apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: ${{ matrix.os }}
|
||||
path: _packages/**/*
|
||||
|
|
|
@ -37,7 +37,7 @@ jobs:
|
|||
- run: ./scripts/check-elixir-deps-discrepancies.exs
|
||||
- run: ./scripts/check-elixir-applications.exs
|
||||
- name: Upload produced lock files
|
||||
uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: failure()
|
||||
with:
|
||||
name: ${{ matrix.profile }}_produced_lock_files
|
||||
|
|
|
@ -52,7 +52,7 @@ jobs:
|
|||
id: package_file
|
||||
run: |
|
||||
echo "PACKAGE_FILE=$(find _packages/emqx -name 'emqx-*.deb' | head -n 1 | xargs basename)" >> $GITHUB_OUTPUT
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: emqx-ubuntu20.04
|
||||
path: _packages/emqx/${{ steps.package_file.outputs.PACKAGE_FILE }}
|
||||
|
@ -77,7 +77,7 @@ jobs:
|
|||
repository: emqx/tf-emqx-performance-test
|
||||
path: tf-emqx-performance-test
|
||||
ref: v0.2.3
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: emqx-ubuntu20.04
|
||||
path: tf-emqx-performance-test/
|
||||
|
@ -113,13 +113,13 @@ jobs:
|
|||
working-directory: ./tf-emqx-performance-test
|
||||
run: |
|
||||
terraform destroy -auto-approve
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: success()
|
||||
with:
|
||||
name: metrics
|
||||
path: |
|
||||
"./tf-emqx-performance-test/*.tar.gz"
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: failure()
|
||||
with:
|
||||
name: terraform
|
||||
|
@ -148,7 +148,7 @@ jobs:
|
|||
repository: emqx/tf-emqx-performance-test
|
||||
path: tf-emqx-performance-test
|
||||
ref: v0.2.3
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: emqx-ubuntu20.04
|
||||
path: tf-emqx-performance-test/
|
||||
|
@ -184,13 +184,13 @@ jobs:
|
|||
working-directory: ./tf-emqx-performance-test
|
||||
run: |
|
||||
terraform destroy -auto-approve
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: success()
|
||||
with:
|
||||
name: metrics
|
||||
path: |
|
||||
"./tf-emqx-performance-test/*.tar.gz"
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: failure()
|
||||
with:
|
||||
name: terraform
|
||||
|
@ -220,7 +220,7 @@ jobs:
|
|||
repository: emqx/tf-emqx-performance-test
|
||||
path: tf-emqx-performance-test
|
||||
ref: v0.2.3
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: emqx-ubuntu20.04
|
||||
path: tf-emqx-performance-test/
|
||||
|
@ -257,13 +257,13 @@ jobs:
|
|||
working-directory: ./tf-emqx-performance-test
|
||||
run: |
|
||||
terraform destroy -auto-approve
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: success()
|
||||
with:
|
||||
name: metrics
|
||||
path: |
|
||||
"./tf-emqx-performance-test/*.tar.gz"
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: failure()
|
||||
with:
|
||||
name: terraform
|
||||
|
@ -294,7 +294,7 @@ jobs:
|
|||
repository: emqx/tf-emqx-performance-test
|
||||
path: tf-emqx-performance-test
|
||||
ref: v0.2.3
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: emqx-ubuntu20.04
|
||||
path: tf-emqx-performance-test/
|
||||
|
@ -330,13 +330,13 @@ jobs:
|
|||
working-directory: ./tf-emqx-performance-test
|
||||
run: |
|
||||
terraform destroy -auto-approve
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: success()
|
||||
with:
|
||||
name: metrics
|
||||
path: |
|
||||
"./tf-emqx-performance-test/*.tar.gz"
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: failure()
|
||||
with:
|
||||
name: terraform
|
||||
|
|
|
@ -25,7 +25,7 @@ jobs:
|
|||
- emqx
|
||||
- emqx-enterprise
|
||||
steps:
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
- name: extract artifact
|
||||
|
@ -40,7 +40,7 @@ jobs:
|
|||
if: failure()
|
||||
run: |
|
||||
cat _build/${{ matrix.profile }}/rel/emqx/log/erlang.log.*
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: failure()
|
||||
with:
|
||||
name: conftest-logs-${{ matrix.profile }}
|
||||
|
|
|
@ -35,7 +35,7 @@ jobs:
|
|||
source env.sh
|
||||
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
|
||||
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: ${{ env.EMQX_NAME }}-docker
|
||||
path: /tmp
|
||||
|
@ -90,7 +90,7 @@ jobs:
|
|||
fi
|
||||
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
|
||||
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: ${{ env.EMQX_NAME }}-docker
|
||||
path: /tmp
|
||||
|
|
|
@ -95,7 +95,7 @@ jobs:
|
|||
echo "Suites: $SUITES"
|
||||
./rebar3 as standalone_test ct --name 'test@127.0.0.1' -v --readable=true --suite="$SUITES"
|
||||
fi
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: failure()
|
||||
with:
|
||||
name: logs-emqx-app-tests-${{ matrix.type }}
|
||||
|
|
|
@ -44,7 +44,7 @@ jobs:
|
|||
source env.sh
|
||||
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
|
||||
echo "EMQX_TAG=$PKG_VSN" >> "$GITHUB_ENV"
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: "${{ env.EMQX_NAME }}-docker"
|
||||
path: /tmp
|
||||
|
|
|
@ -31,7 +31,7 @@ jobs:
|
|||
else
|
||||
wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz $ARCHIVE_URL
|
||||
fi
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: apache-jmeter.tgz
|
||||
path: /tmp/apache-jmeter.tgz
|
||||
|
@ -58,7 +58,7 @@ jobs:
|
|||
source env.sh
|
||||
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
|
||||
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: emqx-docker
|
||||
path: /tmp
|
||||
|
@ -95,7 +95,7 @@ jobs:
|
|||
echo "check logs failed"
|
||||
exit 1
|
||||
fi
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: always()
|
||||
with:
|
||||
name: jmeter_logs-advanced_feat-${{ matrix.scripts_type }}
|
||||
|
@ -127,7 +127,7 @@ jobs:
|
|||
source env.sh
|
||||
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
|
||||
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: emqx-docker
|
||||
path: /tmp
|
||||
|
@ -175,7 +175,7 @@ jobs:
|
|||
if: failure()
|
||||
run: |
|
||||
docker compose -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml logs --no-color > ./jmeter_logs/emqx.log
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: always()
|
||||
with:
|
||||
name: jmeter_logs-pgsql_authn_authz-${{ matrix.scripts_type }}_${{ matrix.pgsql_tag }}
|
||||
|
@ -204,7 +204,7 @@ jobs:
|
|||
source env.sh
|
||||
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
|
||||
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: emqx-docker
|
||||
path: /tmp
|
||||
|
@ -248,7 +248,7 @@ jobs:
|
|||
echo "check logs failed"
|
||||
exit 1
|
||||
fi
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: always()
|
||||
with:
|
||||
name: jmeter_logs-mysql_authn_authz-${{ matrix.scripts_type }}_${{ matrix.mysql_tag }}
|
||||
|
@ -273,7 +273,7 @@ jobs:
|
|||
source env.sh
|
||||
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
|
||||
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: emqx-docker
|
||||
path: /tmp
|
||||
|
@ -313,7 +313,7 @@ jobs:
|
|||
echo "check logs failed"
|
||||
exit 1
|
||||
fi
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: always()
|
||||
with:
|
||||
name: jmeter_logs-JWT_authn-${{ matrix.scripts_type }}
|
||||
|
@ -339,7 +339,7 @@ jobs:
|
|||
source env.sh
|
||||
PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
|
||||
echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: emqx-docker
|
||||
path: /tmp
|
||||
|
@ -370,7 +370,7 @@ jobs:
|
|||
echo "check logs failed"
|
||||
exit 1
|
||||
fi
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: always()
|
||||
with:
|
||||
name: jmeter_logs-built_in_database_authn_authz-${{ matrix.scripts_type }}
|
||||
|
|
|
@ -25,7 +25,7 @@ jobs:
|
|||
run:
|
||||
shell: bash
|
||||
steps:
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: emqx-enterprise
|
||||
- name: extract artifact
|
||||
|
@ -45,7 +45,7 @@ jobs:
|
|||
run: |
|
||||
export PROFILE='emqx-enterprise'
|
||||
make emqx-enterprise-tgz
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
name: Upload built emqx and test scenario
|
||||
with:
|
||||
name: relup_tests_emqx_built
|
||||
|
@ -72,7 +72,7 @@ jobs:
|
|||
run:
|
||||
shell: bash
|
||||
steps:
|
||||
- uses: erlef/setup-beam@b9c58b0450cd832ccdb3c17cc156a47065d2114f # v1.18.1
|
||||
- uses: erlef/setup-beam@a6e26b22319003294c58386b6f25edbc7336819a # v1.18.0
|
||||
with:
|
||||
otp-version: 26.2.5
|
||||
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
|
||||
|
@ -88,7 +88,7 @@ jobs:
|
|||
./configure
|
||||
make
|
||||
echo "$(pwd)/bin" >> $GITHUB_PATH
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
name: Download built emqx and test scenario
|
||||
with:
|
||||
name: relup_tests_emqx_built
|
||||
|
@ -111,7 +111,7 @@ jobs:
|
|||
docker logs node2.emqx.io | tee lux_logs/emqx2.log
|
||||
exit 1
|
||||
fi
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
name: Save debug data
|
||||
if: failure()
|
||||
with:
|
||||
|
|
|
@ -46,7 +46,7 @@ jobs:
|
|||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
|
||||
|
@ -90,7 +90,7 @@ jobs:
|
|||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
- name: extract artifact
|
||||
|
@ -133,7 +133,7 @@ jobs:
|
|||
if: failure()
|
||||
run: tar -czf logs.tar.gz _build/test/logs
|
||||
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: failure()
|
||||
with:
|
||||
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-sg${{ matrix.suitegroup }}
|
||||
|
@ -164,7 +164,7 @@ jobs:
|
|||
CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-sg${{ matrix.suitegroup }}
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
- name: extract artifact
|
||||
|
@ -193,7 +193,7 @@ jobs:
|
|||
if: failure()
|
||||
run: tar -czf logs.tar.gz _build/test/logs
|
||||
|
||||
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
- uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
if: failure()
|
||||
with:
|
||||
name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-sg${{ matrix.suitegroup }}
|
||||
|
|
|
@ -30,7 +30,7 @@ jobs:
|
|||
persist-credentials: false
|
||||
|
||||
- name: "Run analysis"
|
||||
uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
|
||||
uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3
|
||||
with:
|
||||
results_file: results.sarif
|
||||
results_format: sarif
|
||||
|
@ -40,7 +40,7 @@ jobs:
|
|||
publish_results: true
|
||||
|
||||
- name: "Upload artifact"
|
||||
uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
|
||||
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
|
||||
with:
|
||||
name: SARIF file
|
||||
path: results.sarif
|
||||
|
|
|
@ -19,7 +19,7 @@ jobs:
|
|||
- emqx-enterprise
|
||||
runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
|
||||
steps:
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
pattern: "${{ matrix.profile }}-schema-dump-*-x64"
|
||||
merge-multiple: true
|
||||
|
|
|
@ -30,7 +30,7 @@ jobs:
|
|||
include: ${{ fromJson(inputs.ct-matrix) }}
|
||||
container: "${{ inputs.builder }}"
|
||||
steps:
|
||||
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
|
||||
- uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
|
||||
with:
|
||||
name: ${{ matrix.profile }}
|
||||
- name: extract artifact
|
||||
|
|
|
@ -34,7 +34,7 @@ jobs:
|
|||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
|
||||
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
|
|
|
@ -683,7 +683,6 @@ end).
|
|||
|
||||
-define(FRAME_PARSE_ERROR, frame_parse_error).
|
||||
-define(FRAME_SERIALIZE_ERROR, frame_serialize_error).
|
||||
|
||||
-define(THROW_FRAME_ERROR(Reason), erlang:throw({?FRAME_PARSE_ERROR, Reason})).
|
||||
-define(THROW_SERIALIZE_ERROR(Reason), erlang:throw({?FRAME_SERIALIZE_ERROR, Reason})).
|
||||
|
||||
|
|
|
@ -91,7 +91,7 @@
|
|||
?_DO_TRACE(Tag, Msg, Meta),
|
||||
?SLOG(
|
||||
Level,
|
||||
(Meta)#{msg => Msg, tag => Tag},
|
||||
(emqx_trace_formatter:format_meta_map(Meta))#{msg => Msg, tag => Tag},
|
||||
#{is_trace => false}
|
||||
)
|
||||
end).
|
||||
|
|
|
@ -31,7 +31,7 @@
|
|||
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.12.0"}}},
|
||||
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.5"}}},
|
||||
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
|
||||
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.2"}}},
|
||||
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.3"}}},
|
||||
{emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
|
||||
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
|
||||
{recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
{application, emqx, [
|
||||
{id, "emqx"},
|
||||
{description, "EMQX Core"},
|
||||
{vsn, "5.3.4"},
|
||||
{vsn, "5.3.3"},
|
||||
{modules, []},
|
||||
{registered, []},
|
||||
{applications, [
|
||||
|
|
|
@ -146,9 +146,7 @@
|
|||
-type replies() :: emqx_types:packet() | reply() | [reply()].
|
||||
|
||||
-define(IS_MQTT_V5, #channel{conninfo = #{proto_ver := ?MQTT_PROTO_V5}}).
|
||||
-define(IS_CONNECTED_OR_REAUTHENTICATING(ConnState),
|
||||
((ConnState == connected) orelse (ConnState == reauthenticating))
|
||||
).
|
||||
|
||||
-define(IS_COMMON_SESSION_TIMER(N),
|
||||
((N == retry_delivery) orelse (N == expire_awaiting_rel))
|
||||
).
|
||||
|
@ -339,7 +337,7 @@ take_conn_info_fields(Fields, ClientInfo, ConnInfo) ->
|
|||
| {shutdown, Reason :: term(), channel()}
|
||||
| {shutdown, Reason :: term(), replies(), channel()}.
|
||||
handle_in(?CONNECT_PACKET(), Channel = #channel{conn_state = ConnState}) when
|
||||
?IS_CONNECTED_OR_REAUTHENTICATING(ConnState)
|
||||
ConnState =:= connected orelse ConnState =:= reauthenticating
|
||||
->
|
||||
handle_out(disconnect, ?RC_PROTOCOL_ERROR, Channel);
|
||||
handle_in(?CONNECT_PACKET(), Channel = #channel{conn_state = connecting}) ->
|
||||
|
@ -569,8 +567,29 @@ handle_in(
|
|||
process_disconnect(ReasonCode, Properties, NChannel);
|
||||
handle_in(?AUTH_PACKET(), Channel) ->
|
||||
handle_out(disconnect, ?RC_IMPLEMENTATION_SPECIFIC_ERROR, Channel);
|
||||
handle_in({frame_error, Reason}, Channel) ->
|
||||
handle_frame_error(Reason, Channel);
|
||||
handle_in({frame_error, Reason}, Channel = #channel{conn_state = idle}) ->
|
||||
shutdown(shutdown_count(frame_error, Reason), Channel);
|
||||
handle_in(
|
||||
{frame_error, #{cause := frame_too_large} = R}, Channel = #channel{conn_state = connecting}
|
||||
) ->
|
||||
shutdown(
|
||||
shutdown_count(frame_error, R), ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE), Channel
|
||||
);
|
||||
handle_in({frame_error, Reason}, Channel = #channel{conn_state = connecting}) ->
|
||||
shutdown(shutdown_count(frame_error, Reason), ?CONNACK_PACKET(?RC_MALFORMED_PACKET), Channel);
|
||||
handle_in(
|
||||
{frame_error, #{cause := frame_too_large}}, Channel = #channel{conn_state = ConnState}
|
||||
) when
|
||||
ConnState =:= connected orelse ConnState =:= reauthenticating
|
||||
->
|
||||
handle_out(disconnect, {?RC_PACKET_TOO_LARGE, frame_too_large}, Channel);
|
||||
handle_in({frame_error, Reason}, Channel = #channel{conn_state = ConnState}) when
|
||||
ConnState =:= connected orelse ConnState =:= reauthenticating
|
||||
->
|
||||
handle_out(disconnect, {?RC_MALFORMED_PACKET, Reason}, Channel);
|
||||
handle_in({frame_error, Reason}, Channel = #channel{conn_state = disconnected}) ->
|
||||
?SLOG(error, #{msg => "malformed_mqtt_message", reason => Reason}),
|
||||
{ok, Channel};
|
||||
handle_in(Packet, Channel) ->
|
||||
?SLOG(error, #{msg => "disconnecting_due_to_unexpected_message", packet => Packet}),
|
||||
handle_out(disconnect, ?RC_PROTOCOL_ERROR, Channel).
|
||||
|
@ -1002,68 +1021,6 @@ not_nacked({deliver, _Topic, Msg}) ->
|
|||
true
|
||||
end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% Handle Frame Error
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
handle_frame_error(
|
||||
Reason = #{cause := frame_too_large},
|
||||
Channel = #channel{conn_state = ConnState, conninfo = ConnInfo}
|
||||
) when
|
||||
?IS_CONNECTED_OR_REAUTHENTICATING(ConnState)
|
||||
->
|
||||
ShutdownCount = shutdown_count(frame_error, Reason),
|
||||
case proto_ver(Reason, ConnInfo) of
|
||||
?MQTT_PROTO_V5 ->
|
||||
handle_out(disconnect, {?RC_PACKET_TOO_LARGE, frame_too_large}, Channel);
|
||||
_ ->
|
||||
shutdown(ShutdownCount, Channel)
|
||||
end;
|
||||
%% Only send CONNACK with reason code `frame_too_large` for MQTT-v5.0 when connecting,
|
||||
%% otherwise DONOT send any CONNACK or DISCONNECT packet.
|
||||
handle_frame_error(
|
||||
Reason,
|
||||
Channel = #channel{conn_state = ConnState, conninfo = ConnInfo}
|
||||
) when
|
||||
is_map(Reason) andalso
|
||||
(ConnState == idle orelse ConnState == connecting)
|
||||
->
|
||||
ShutdownCount = shutdown_count(frame_error, Reason),
|
||||
ProtoVer = proto_ver(Reason, ConnInfo),
|
||||
NChannel = Channel#channel{conninfo = ConnInfo#{proto_ver => ProtoVer}},
|
||||
case ProtoVer of
|
||||
?MQTT_PROTO_V5 ->
|
||||
shutdown(ShutdownCount, ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE), NChannel);
|
||||
_ ->
|
||||
shutdown(ShutdownCount, NChannel)
|
||||
end;
|
||||
handle_frame_error(
|
||||
Reason,
|
||||
Channel = #channel{conn_state = connecting}
|
||||
) ->
|
||||
shutdown(
|
||||
shutdown_count(frame_error, Reason),
|
||||
?CONNACK_PACKET(?RC_MALFORMED_PACKET),
|
||||
Channel
|
||||
);
|
||||
handle_frame_error(
|
||||
Reason,
|
||||
Channel = #channel{conn_state = ConnState}
|
||||
) when
|
||||
?IS_CONNECTED_OR_REAUTHENTICATING(ConnState)
|
||||
->
|
||||
handle_out(
|
||||
disconnect,
|
||||
{?RC_MALFORMED_PACKET, Reason},
|
||||
Channel
|
||||
);
|
||||
handle_frame_error(
|
||||
Reason,
|
||||
Channel = #channel{conn_state = disconnected}
|
||||
) ->
|
||||
?SLOG(error, #{msg => "malformed_mqtt_message", reason => Reason}),
|
||||
{ok, Channel}.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% Handle outgoing packet
|
||||
%%--------------------------------------------------------------------
|
||||
|
@ -1332,7 +1289,7 @@ handle_info(
|
|||
session = Session
|
||||
}
|
||||
) when
|
||||
?IS_CONNECTED_OR_REAUTHENTICATING(ConnState)
|
||||
ConnState =:= connected orelse ConnState =:= reauthenticating
|
||||
->
|
||||
{Intent, Session1} = session_disconnect(ClientInfo, ConnInfo, Session),
|
||||
Channel1 = ensure_disconnected(Reason, maybe_publish_will_msg(sock_closed, Channel)),
|
||||
|
@ -2679,7 +2636,8 @@ save_alias(outbound, AliasId, Topic, TopicAliases = #{outbound := Aliases}) ->
|
|||
NAliases = maps:put(Topic, AliasId, Aliases),
|
||||
TopicAliases#{outbound => NAliases}.
|
||||
|
||||
-compile({inline, [reply/2, shutdown/2, shutdown/3]}).
|
||||
-compile({inline, [reply/2, shutdown/2, shutdown/3, sp/1, flag/1]}).
|
||||
|
||||
reply(Reply, Channel) ->
|
||||
{reply, Reply, Channel}.
|
||||
|
||||
|
@ -2715,13 +2673,13 @@ disconnect_and_shutdown(
|
|||
?IS_MQTT_V5 =
|
||||
#channel{conn_state = ConnState}
|
||||
) when
|
||||
?IS_CONNECTED_OR_REAUTHENTICATING(ConnState)
|
||||
ConnState =:= connected orelse ConnState =:= reauthenticating
|
||||
->
|
||||
NChannel = ensure_disconnected(Reason, Channel),
|
||||
shutdown(Reason, Reply, ?DISCONNECT_PACKET(reason_code(Reason)), NChannel);
|
||||
%% mqtt v3/v4 connected sessions
|
||||
disconnect_and_shutdown(Reason, Reply, Channel = #channel{conn_state = ConnState}) when
|
||||
?IS_CONNECTED_OR_REAUTHENTICATING(ConnState)
|
||||
ConnState =:= connected orelse ConnState =:= reauthenticating
|
||||
->
|
||||
NChannel = ensure_disconnected(Reason, Channel),
|
||||
shutdown(Reason, Reply, NChannel);
|
||||
|
@ -2764,13 +2722,6 @@ is_durable_session(#channel{session = Session}) ->
|
|||
false
|
||||
end.
|
||||
|
||||
proto_ver(#{proto_ver := ProtoVer}, _ConnInfo) ->
|
||||
ProtoVer;
|
||||
proto_ver(_Reason, #{proto_ver := ProtoVer}) ->
|
||||
ProtoVer;
|
||||
proto_ver(_, _) ->
|
||||
?MQTT_PROTO_V4.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% For CT tests
|
||||
%%--------------------------------------------------------------------
|
||||
|
|
|
@ -621,16 +621,16 @@ save_to_config_map(Conf, RawConf) ->
|
|||
?MODULE:put_raw(RawConf).
|
||||
|
||||
-spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}.
|
||||
save_to_override_conf(_, undefined, _) ->
|
||||
save_to_override_conf(_HasDeprecatedFile, undefined, _) ->
|
||||
ok;
|
||||
save_to_override_conf(true, RawConf, Opts) ->
|
||||
save_to_override_conf(true = _HasDeprecatedFile, RawConf, Opts) ->
|
||||
case deprecated_conf_file(Opts) of
|
||||
undefined ->
|
||||
ok;
|
||||
FileName ->
|
||||
backup_and_write(FileName, hocon_pp:do(RawConf, Opts))
|
||||
end;
|
||||
save_to_override_conf(false, RawConf, Opts) ->
|
||||
save_to_override_conf(false = _HasDeprecatedFile, RawConf, Opts) ->
|
||||
case cluster_hocon_file() of
|
||||
undefined ->
|
||||
ok;
|
||||
|
|
|
@ -783,8 +783,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
|
|||
input_bytes => Data,
|
||||
parsed_packets => Packets
|
||||
}),
|
||||
NState = enrich_state(Reason, State),
|
||||
{[{frame_error, Reason} | Packets], NState};
|
||||
{[{frame_error, Reason} | Packets], State};
|
||||
error:Reason:Stacktrace ->
|
||||
?LOG(error, #{
|
||||
at_state => emqx_frame:describe_state(ParseState),
|
||||
|
@ -1228,12 +1227,6 @@ inc_counter(Key, Inc) ->
|
|||
_ = emqx_pd:inc_counter(Key, Inc),
|
||||
ok.
|
||||
|
||||
enrich_state(#{parse_state := NParseState}, State) ->
|
||||
Serialize = emqx_frame:serialize_opts(NParseState),
|
||||
State#state{parse_state = NParseState, serialize = Serialize};
|
||||
enrich_state(_, State) ->
|
||||
State.
|
||||
|
||||
set_tcp_keepalive({quic, _Listener}) ->
|
||||
ok;
|
||||
set_tcp_keepalive({Type, Id}) ->
|
||||
|
|
|
@ -267,50 +267,28 @@ packet(Header, Variable) ->
|
|||
packet(Header, Variable, Payload) ->
|
||||
#mqtt_packet{header = Header, variable = Variable, payload = Payload}.
|
||||
|
||||
parse_connect(FrameBin, Options = #{strict_mode := StrictMode}) ->
|
||||
{ProtoName, Rest0} = parse_utf8_string_with_cause(FrameBin, StrictMode, invalid_proto_name),
|
||||
%% No need to parse and check proto_ver if proto_name is invalid, check it first
|
||||
%% And the matching check of `proto_name` and `proto_ver` fields will be done in `emqx_packet:check_proto_ver/2`
|
||||
_ = validate_proto_name(ProtoName),
|
||||
{IsBridge, ProtoVer, Rest2} = parse_connect_proto_ver(Rest0),
|
||||
NOptions = Options#{version => ProtoVer},
|
||||
try
|
||||
do_parse_connect(ProtoName, IsBridge, ProtoVer, Rest2, StrictMode)
|
||||
catch
|
||||
throw:{?FRAME_PARSE_ERROR, ReasonM} when is_map(ReasonM) ->
|
||||
?PARSE_ERR(
|
||||
ReasonM#{
|
||||
proto_ver => ProtoVer,
|
||||
proto_name => ProtoName,
|
||||
parse_state => ?NONE(NOptions)
|
||||
}
|
||||
);
|
||||
throw:{?FRAME_PARSE_ERROR, Reason} ->
|
||||
?PARSE_ERR(
|
||||
#{
|
||||
cause => Reason,
|
||||
proto_ver => ProtoVer,
|
||||
proto_name => ProtoName,
|
||||
parse_state => ?NONE(NOptions)
|
||||
}
|
||||
)
|
||||
end.
|
||||
parse_connect(FrameBin, StrictMode) ->
|
||||
{ProtoName, Rest} = parse_utf8_string_with_cause(FrameBin, StrictMode, invalid_proto_name),
|
||||
case ProtoName of
|
||||
<<"MQTT">> ->
|
||||
ok;
|
||||
<<"MQIsdp">> ->
|
||||
ok;
|
||||
_ ->
|
||||
%% from spec: the server MAY send disconnect with reason code 0x84
|
||||
%% we chose to close socket because the client is likely not talking MQTT anyway
|
||||
?PARSE_ERR(#{
|
||||
cause => invalid_proto_name,
|
||||
expected => <<"'MQTT' or 'MQIsdp'">>,
|
||||
received => ProtoName
|
||||
})
|
||||
end,
|
||||
parse_connect2(ProtoName, Rest, StrictMode).
|
||||
|
||||
do_parse_connect(
|
||||
parse_connect2(
|
||||
ProtoName,
|
||||
IsBridge,
|
||||
ProtoVer,
|
||||
<<
|
||||
UsernameFlagB:1,
|
||||
PasswordFlagB:1,
|
||||
WillRetainB:1,
|
||||
WillQoS:2,
|
||||
WillFlagB:1,
|
||||
CleanStart:1,
|
||||
Reserved:1,
|
||||
KeepAlive:16/big,
|
||||
Rest/binary
|
||||
>>,
|
||||
<<BridgeTag:4, ProtoVer:4, UsernameFlagB:1, PasswordFlagB:1, WillRetainB:1, WillQoS:2,
|
||||
WillFlagB:1, CleanStart:1, Reserved:1, KeepAlive:16/big, Rest2/binary>>,
|
||||
StrictMode
|
||||
) ->
|
||||
_ = validate_connect_reserved(Reserved),
|
||||
|
@ -325,14 +303,14 @@ do_parse_connect(
|
|||
UsernameFlag = bool(UsernameFlagB),
|
||||
PasswordFlag = bool(PasswordFlagB)
|
||||
),
|
||||
{Properties, Rest3} = parse_properties(Rest, ProtoVer, StrictMode),
|
||||
{Properties, Rest3} = parse_properties(Rest2, ProtoVer, StrictMode),
|
||||
{ClientId, Rest4} = parse_utf8_string_with_cause(Rest3, StrictMode, invalid_clientid),
|
||||
ConnPacket = #mqtt_packet_connect{
|
||||
proto_name = ProtoName,
|
||||
proto_ver = ProtoVer,
|
||||
%% For bridge mode, non-standard implementation
|
||||
%% Invented by mosquitto, named 'try_private': https://mosquitto.org/man/mosquitto-conf-5.html
|
||||
is_bridge = IsBridge,
|
||||
is_bridge = (BridgeTag =:= 8),
|
||||
clean_start = bool(CleanStart),
|
||||
will_flag = WillFlag,
|
||||
will_qos = WillQoS,
|
||||
|
@ -365,16 +343,16 @@ do_parse_connect(
|
|||
unexpected_trailing_bytes => size(Rest7)
|
||||
})
|
||||
end;
|
||||
do_parse_connect(_ProtoName, _IsBridge, _ProtoVer, Bin, _StrictMode) ->
|
||||
%% sent less than 24 bytes
|
||||
parse_connect2(_ProtoName, Bin, _StrictMode) ->
|
||||
%% sent less than 32 bytes
|
||||
?PARSE_ERR(#{cause => malformed_connect, header_bytes => Bin}).
|
||||
|
||||
parse_packet(
|
||||
#mqtt_packet_header{type = ?CONNECT},
|
||||
FrameBin,
|
||||
Options
|
||||
#{strict_mode := StrictMode}
|
||||
) ->
|
||||
parse_connect(FrameBin, Options);
|
||||
parse_connect(FrameBin, StrictMode);
|
||||
parse_packet(
|
||||
#mqtt_packet_header{type = ?CONNACK},
|
||||
<<AckFlags:8, ReasonCode:8, Rest/binary>>,
|
||||
|
@ -538,12 +516,6 @@ parse_packet_id(<<PacketId:16/big, Rest/binary>>) ->
|
|||
parse_packet_id(_) ->
|
||||
?PARSE_ERR(invalid_packet_id).
|
||||
|
||||
parse_connect_proto_ver(<<BridgeTag:4, ProtoVer:4, Rest/binary>>) ->
|
||||
{_IsBridge = (BridgeTag =:= 8), ProtoVer, Rest};
|
||||
parse_connect_proto_ver(Bin) ->
|
||||
%% sent less than 1 bytes or empty
|
||||
?PARSE_ERR(#{cause => malformed_connect, header_bytes => Bin}).
|
||||
|
||||
parse_properties(Bin, Ver, _StrictMode) when Ver =/= ?MQTT_PROTO_V5 ->
|
||||
{#{}, Bin};
|
||||
%% TODO: version mess?
|
||||
|
@ -767,8 +739,6 @@ serialize_fun(#{version := Ver, max_size := MaxSize, strict_mode := StrictMode})
|
|||
initial_serialize_opts(Opts) ->
|
||||
maps:merge(?DEFAULT_OPTIONS, Opts).
|
||||
|
||||
serialize_opts(?NONE(Options)) ->
|
||||
maps:merge(?DEFAULT_OPTIONS, Options);
|
||||
serialize_opts(#mqtt_packet_connect{proto_ver = ProtoVer, properties = ConnProps}) ->
|
||||
MaxSize = get_property('Maximum-Packet-Size', ConnProps, ?MAX_PACKET_SIZE),
|
||||
#{version => ProtoVer, max_size => MaxSize, strict_mode => false}.
|
||||
|
@ -1187,34 +1157,18 @@ validate_subqos([3 | _]) -> ?PARSE_ERR(bad_subqos);
|
|||
validate_subqos([_ | T]) -> validate_subqos(T);
|
||||
validate_subqos([]) -> ok.
|
||||
|
||||
%% from spec: the server MAY send disconnect with reason code 0x84
|
||||
%% we chose to close socket because the client is likely not talking MQTT anyway
|
||||
validate_proto_name(<<"MQTT">>) ->
|
||||
ok;
|
||||
validate_proto_name(<<"MQIsdp">>) ->
|
||||
ok;
|
||||
validate_proto_name(ProtoName) ->
|
||||
?PARSE_ERR(#{
|
||||
cause => invalid_proto_name,
|
||||
expected => <<"'MQTT' or 'MQIsdp'">>,
|
||||
received => ProtoName
|
||||
}).
|
||||
|
||||
%% MQTT-v3.1.1-[MQTT-3.1.2-3], MQTT-v5.0-[MQTT-3.1.2-3]
|
||||
-compile({inline, [validate_connect_reserved/1]}).
|
||||
validate_connect_reserved(0) -> ok;
|
||||
validate_connect_reserved(1) -> ?PARSE_ERR(reserved_connect_flag).
|
||||
|
||||
-compile({inline, [validate_connect_will/3]}).
|
||||
%% MQTT-v3.1.1-[MQTT-3.1.2-13], MQTT-v5.0-[MQTT-3.1.2-11]
|
||||
validate_connect_will(false, _, WillQoS) when WillQoS > 0 -> ?PARSE_ERR(invalid_will_qos);
|
||||
validate_connect_will(false, _, WillQos) when WillQos > 0 -> ?PARSE_ERR(invalid_will_qos);
|
||||
%% MQTT-v3.1.1-[MQTT-3.1.2-14], MQTT-v5.0-[MQTT-3.1.2-12]
|
||||
validate_connect_will(true, _, WillQoS) when WillQoS > 2 -> ?PARSE_ERR(invalid_will_qos);
|
||||
%% MQTT-v3.1.1-[MQTT-3.1.2-15], MQTT-v5.0-[MQTT-3.1.2-13]
|
||||
validate_connect_will(false, WillRetain, _) when WillRetain -> ?PARSE_ERR(invalid_will_retain);
|
||||
validate_connect_will(_, _, _) -> ok.
|
||||
|
||||
-compile({inline, [validate_connect_password_flag/4]}).
|
||||
%% MQTT-v3.1
|
||||
%% Username flag and password flag are not strongly related
|
||||
%% https://public.dhe.ibm.com/software/dw/webservices/ws-mqtt/mqtt-v3r1.html#connect
|
||||
|
@ -1229,7 +1183,6 @@ validate_connect_password_flag(true, ?MQTT_PROTO_V5, _, _) ->
|
|||
validate_connect_password_flag(_, _, _, _) ->
|
||||
ok.
|
||||
|
||||
-compile({inline, [bool/1]}).
|
||||
bool(0) -> false;
|
||||
bool(1) -> true.
|
||||
|
||||
|
|
|
@ -432,7 +432,7 @@ do_start_listener(Type, Name, Id, #{bind := ListenOn} = Opts) when ?ESOCKD_LISTE
|
|||
esockd:open(
|
||||
Id,
|
||||
ListenOn,
|
||||
merge_default(esockd_opts(Id, Type, Name, Opts, _OldOpts = undefined))
|
||||
merge_default(esockd_opts(Id, Type, Name, Opts))
|
||||
);
|
||||
%% Start MQTT/WS listener
|
||||
do_start_listener(Type, Name, Id, Opts) when ?COWBOY_LISTENER(Type) ->
|
||||
|
@ -476,7 +476,7 @@ do_update_listener(Type, Name, OldConf, NewConf = #{bind := ListenOn}) when
|
|||
Id = listener_id(Type, Name),
|
||||
case maps:get(bind, OldConf) of
|
||||
ListenOn ->
|
||||
esockd:set_options({Id, ListenOn}, esockd_opts(Id, Type, Name, NewConf, OldConf));
|
||||
esockd:set_options({Id, ListenOn}, esockd_opts(Id, Type, Name, NewConf));
|
||||
_Different ->
|
||||
%% TODO
|
||||
%% Again, we're not strictly required to drop live connections in this case.
|
||||
|
@ -588,7 +588,7 @@ perform_listener_change(update, {{Type, Name, ConfOld}, {_, _, ConfNew}}) ->
|
|||
perform_listener_change(stop, {Type, Name, Conf}) ->
|
||||
stop_listener(Type, Name, Conf).
|
||||
|
||||
esockd_opts(ListenerId, Type, Name, Opts0, OldOpts) ->
|
||||
esockd_opts(ListenerId, Type, Name, Opts0) ->
|
||||
Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0),
|
||||
Limiter = limiter(Opts0),
|
||||
Opts2 =
|
||||
|
@ -620,7 +620,7 @@ esockd_opts(ListenerId, Type, Name, Opts0, OldOpts) ->
|
|||
tcp ->
|
||||
Opts3#{tcp_options => tcp_opts(Opts0)};
|
||||
ssl ->
|
||||
OptsWithCRL = inject_crl_config(Opts0, OldOpts),
|
||||
OptsWithCRL = inject_crl_config(Opts0),
|
||||
OptsWithSNI = inject_sni_fun(ListenerId, OptsWithCRL),
|
||||
OptsWithRootFun = inject_root_fun(OptsWithSNI),
|
||||
OptsWithVerifyFun = inject_verify_fun(OptsWithRootFun),
|
||||
|
@ -996,7 +996,7 @@ inject_sni_fun(_ListenerId, Conf) ->
|
|||
Conf.
|
||||
|
||||
inject_crl_config(
|
||||
Conf = #{ssl_options := #{enable_crl_check := true} = SSLOpts}, _OldOpts
|
||||
Conf = #{ssl_options := #{enable_crl_check := true} = SSLOpts}
|
||||
) ->
|
||||
HTTPTimeout = emqx_config:get([crl_cache, http_timeout], timer:seconds(15)),
|
||||
Conf#{
|
||||
|
@ -1006,16 +1006,7 @@ inject_crl_config(
|
|||
crl_cache => {emqx_ssl_crl_cache, {internal, [{http, HTTPTimeout}]}}
|
||||
}
|
||||
};
|
||||
inject_crl_config(#{ssl_options := SSLOpts0} = Conf0, #{} = OldOpts) ->
|
||||
%% Note: we must set crl options to `undefined' to unset them. Otherwise,
|
||||
%% `esockd' will retain such options when `esockd:merge_opts/2' is called and the SSL
|
||||
%% options were previously enabled.
|
||||
WasEnabled = emqx_utils_maps:deep_get([ssl_options, enable_crl_check], OldOpts, false),
|
||||
Undefine = fun(Acc, K) -> emqx_utils_maps:put_if(Acc, K, undefined, WasEnabled) end,
|
||||
SSLOpts1 = Undefine(SSLOpts0, crl_check),
|
||||
SSLOpts = Undefine(SSLOpts1, crl_cache),
|
||||
Conf0#{ssl_options := SSLOpts};
|
||||
inject_crl_config(Conf, undefined = _OldOpts) ->
|
||||
inject_crl_config(Conf) ->
|
||||
Conf.
|
||||
|
||||
maybe_unregister_ocsp_stapling_refresh(
|
||||
|
|
|
@ -105,7 +105,7 @@ format(Msg, Meta, Config) ->
|
|||
maybe_format_msg(undefined, _Meta, _Config) ->
|
||||
#{};
|
||||
maybe_format_msg({report, Report0} = Msg, #{report_cb := Cb} = Meta, Config) ->
|
||||
Report = emqx_logger_textfmt:try_encode_meta(Report0, Config),
|
||||
Report = emqx_logger_textfmt:try_encode_payload(Report0, Config),
|
||||
case is_map(Report) andalso Cb =:= ?DEFAULT_FORMATTER of
|
||||
true ->
|
||||
%% reporting a map without a customised format function
|
||||
|
|
|
@ -20,7 +20,7 @@
|
|||
|
||||
-export([format/2]).
|
||||
-export([check_config/1]).
|
||||
-export([try_format_unicode/1, try_encode_meta/2]).
|
||||
-export([try_format_unicode/1, try_encode_payload/2]).
|
||||
%% Used in the other log formatters
|
||||
-export([evaluate_lazy_values_if_dbg_level/1, evaluate_lazy_values/1]).
|
||||
|
||||
|
@ -111,7 +111,7 @@ is_list_report_acceptable(_) ->
|
|||
enrich_report(ReportRaw0, Meta, Config) ->
|
||||
%% clientid and peername always in emqx_conn's process metadata.
|
||||
%% topic and username can be put in meta using ?SLOG/3, or put in msg's report by ?SLOG/2
|
||||
ReportRaw = try_encode_meta(ReportRaw0, Config),
|
||||
ReportRaw = try_encode_payload(ReportRaw0, Config),
|
||||
Topic =
|
||||
case maps:get(topic, Meta, undefined) of
|
||||
undefined -> maps:get(topic, ReportRaw, undefined);
|
||||
|
@ -180,22 +180,9 @@ enrich_topic({Fmt, Args}, #{topic := Topic}) when is_list(Fmt) ->
|
|||
enrich_topic(Msg, _) ->
|
||||
Msg.
|
||||
|
||||
try_encode_meta(Report, Config) ->
|
||||
lists:foldl(
|
||||
fun(Meta, Acc) ->
|
||||
try_encode_meta(Meta, Acc, Config)
|
||||
end,
|
||||
Report,
|
||||
[payload, packet]
|
||||
).
|
||||
|
||||
try_encode_meta(payload, #{payload := Payload} = Report, #{payload_encode := Encode}) ->
|
||||
try_encode_payload(#{payload := Payload} = Report, #{payload_encode := Encode}) ->
|
||||
Report#{payload := encode_payload(Payload, Encode)};
|
||||
try_encode_meta(packet, #{packet := Packet} = Report, #{payload_encode := Encode}) when
|
||||
is_tuple(Packet)
|
||||
->
|
||||
Report#{packet := emqx_packet:format(Packet, Encode)};
|
||||
try_encode_meta(_, Report, _Config) ->
|
||||
try_encode_payload(Report, _Config) ->
|
||||
Report.
|
||||
|
||||
encode_payload(Payload, text) ->
|
||||
|
@ -203,5 +190,4 @@ encode_payload(Payload, text) ->
|
|||
encode_payload(_Payload, hidden) ->
|
||||
"******";
|
||||
encode_payload(Payload, hex) ->
|
||||
Bin = emqx_utils_conv:bin(Payload),
|
||||
binary:encode_hex(Bin).
|
||||
binary:encode_hex(Payload).
|
||||
|
|
|
@ -51,6 +51,7 @@
|
|||
]).
|
||||
|
||||
-export([
|
||||
format/1,
|
||||
format/2
|
||||
]).
|
||||
|
||||
|
@ -480,6 +481,10 @@ will_msg(#mqtt_packet_connect{
|
|||
headers = #{username => Username, properties => Props}
|
||||
}.
|
||||
|
||||
%% @doc Format packet
|
||||
-spec format(emqx_types:packet()) -> iolist().
|
||||
format(Packet) -> format(Packet, emqx_trace_handler:payload_encode()).
|
||||
|
||||
%% @doc Format packet
|
||||
-spec format(emqx_types:packet(), hex | text | hidden) -> iolist().
|
||||
format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, PayloadEncode) ->
|
||||
|
|
|
@ -56,11 +56,6 @@
|
|||
cold_get_subscription/2
|
||||
]).
|
||||
|
||||
-export([
|
||||
format_lease_events/1,
|
||||
format_stream_progresses/1
|
||||
]).
|
||||
|
||||
-define(schedule_subscribe, schedule_subscribe).
|
||||
-define(schedule_unsubscribe, schedule_unsubscribe).
|
||||
|
||||
|
@ -241,14 +236,14 @@ schedule_subscribe(
|
|||
ScheduledActions1 = ScheduledActions0#{
|
||||
ShareTopicFilter => ScheduledAction#{type => {?schedule_subscribe, SubOpts}}
|
||||
},
|
||||
?tp(debug, shared_subs_schedule_subscribe_override, #{
|
||||
?tp(warning, shared_subs_schedule_subscribe_override, #{
|
||||
share_topic_filter => ShareTopicFilter,
|
||||
new_type => {?schedule_subscribe, SubOpts},
|
||||
old_action => format_schedule_action(ScheduledAction)
|
||||
}),
|
||||
SharedSubS0#{scheduled_actions := ScheduledActions1};
|
||||
_ ->
|
||||
?tp(debug, shared_subs_schedule_subscribe_new, #{
|
||||
?tp(warning, shared_subs_schedule_subscribe_new, #{
|
||||
share_topic_filter => ShareTopicFilter, subopts => SubOpts
|
||||
}),
|
||||
Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_subscribe(
|
||||
|
@ -299,7 +294,7 @@ schedule_unsubscribe(
|
|||
ScheduledActions1 = ScheduledActions0#{
|
||||
ShareTopicFilter => ScheduledAction1
|
||||
},
|
||||
?tp(debug, shared_subs_schedule_unsubscribe_override, #{
|
||||
?tp(warning, shared_subs_schedule_unsubscribe_override, #{
|
||||
share_topic_filter => ShareTopicFilter,
|
||||
new_type => ?schedule_unsubscribe,
|
||||
old_action => format_schedule_action(ScheduledAction0)
|
||||
|
@ -314,7 +309,7 @@ schedule_unsubscribe(
|
|||
progresses => []
|
||||
}
|
||||
},
|
||||
?tp(debug, shared_subs_schedule_unsubscribe_new, #{
|
||||
?tp(warning, shared_subs_schedule_unsubscribe_new, #{
|
||||
share_topic_filter => ShareTopicFilter,
|
||||
stream_keys => format_stream_keys(StreamKeys)
|
||||
}),
|
||||
|
@ -339,7 +334,7 @@ renew_streams(S0, #{agent := Agent0, scheduled_actions := ScheduledActions} = Sh
|
|||
Agent0
|
||||
),
|
||||
StreamLeaseEvents =/= [] andalso
|
||||
?tp(debug, shared_subs_new_stream_lease_events, #{
|
||||
?tp(warning, shared_subs_new_stream_lease_events, #{
|
||||
stream_lease_events => format_lease_events(StreamLeaseEvents)
|
||||
}),
|
||||
S1 = lists:foldl(
|
||||
|
@ -506,7 +501,7 @@ run_scheduled_action(
|
|||
Progresses1 = stream_progresses(S, StreamKeysToWait0 -- StreamKeysToWait1) ++ Progresses0,
|
||||
case StreamKeysToWait1 of
|
||||
[] ->
|
||||
?tp(debug, shared_subs_schedule_action_complete, #{
|
||||
?tp(warning, shared_subs_schedule_action_complete, #{
|
||||
share_topic_filter => ShareTopicFilter,
|
||||
progresses => format_stream_progresses(Progresses1),
|
||||
type => Type
|
||||
|
@ -530,7 +525,7 @@ run_scheduled_action(
|
|||
end;
|
||||
_ ->
|
||||
Action1 = Action#{stream_keys_to_wait => StreamKeysToWait1, progresses => Progresses1},
|
||||
?tp(debug, shared_subs_schedule_action_continue, #{
|
||||
?tp(warning, shared_subs_schedule_action_continue, #{
|
||||
share_topic_filter => ShareTopicFilter,
|
||||
new_action => format_schedule_action(Action1)
|
||||
}),
|
||||
|
|
|
@ -62,7 +62,7 @@
|
|||
streams := [{pid(), quicer:stream_handle()}],
|
||||
%% New stream opts
|
||||
stream_opts := map(),
|
||||
%% If connection is resumed from session ticket
|
||||
%% If conneciton is resumed from session ticket
|
||||
is_resumed => boolean(),
|
||||
%% mqtt message serializer config
|
||||
serialize => undefined,
|
||||
|
@ -70,8 +70,8 @@
|
|||
}.
|
||||
-type cb_ret() :: quicer_lib:cb_ret().
|
||||
|
||||
%% @doc Data streams initializations are started in parallel with control streams, data streams are blocked
|
||||
%% for the activation from control stream after it is accepted as a legit connection.
|
||||
%% @doc Data streams initializions are started in parallel with control streams, data streams are blocked
|
||||
%% for the activation from control stream after it is accepted as a legit conneciton.
|
||||
%% For security, the initial number of allowed data streams from client should be limited by
|
||||
%% 'peer_bidi_stream_count` & 'peer_unidi_stream_count`
|
||||
-spec activate_data_streams(pid(), {
|
||||
|
@ -80,7 +80,7 @@
|
|||
activate_data_streams(ConnOwner, {PS, Serialize, Channel}) ->
|
||||
gen_server:call(ConnOwner, {activate_data_streams, {PS, Serialize, Channel}}, infinity).
|
||||
|
||||
%% @doc connection owner init callback
|
||||
%% @doc conneciton owner init callback
|
||||
-spec init(map()) -> {ok, cb_state()}.
|
||||
init(#{stream_opts := SOpts} = S) when is_list(SOpts) ->
|
||||
init(S#{stream_opts := maps:from_list(SOpts)});
|
||||
|
|
|
@ -589,14 +589,6 @@ ensure_valid_options(Options, Versions) ->
|
|||
|
||||
ensure_valid_options([], _, Acc) ->
|
||||
lists:reverse(Acc);
|
||||
ensure_valid_options([{K, undefined} | T], Versions, Acc) when
|
||||
K =:= crl_check;
|
||||
K =:= crl_cache
|
||||
->
|
||||
%% Note: we must set crl options to `undefined' to unset them. Otherwise,
|
||||
%% `esockd' will retain such options when `esockd:merge_opts/2' is called and the SSL
|
||||
%% options were previously enabled.
|
||||
ensure_valid_options(T, Versions, [{K, undefined} | Acc]);
|
||||
ensure_valid_options([{_, undefined} | T], Versions, Acc) ->
|
||||
ensure_valid_options(T, Versions, Acc);
|
||||
ensure_valid_options([{_, ""} | T], Versions, Acc) ->
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
-include("emqx_mqtt.hrl").
|
||||
|
||||
-export([format/2]).
|
||||
-export([format_meta_map/1]).
|
||||
|
||||
%% logger_formatter:config/0 is not exported.
|
||||
-type config() :: map().
|
||||
|
@ -42,6 +43,10 @@ format(
|
|||
format(Event, Config) ->
|
||||
emqx_logger_textfmt:format(Event, Config).
|
||||
|
||||
format_meta_map(Meta) ->
|
||||
Encode = emqx_trace_handler:payload_encode(),
|
||||
format_meta_map(Meta, Encode).
|
||||
|
||||
format_meta_map(Meta, Encode) ->
|
||||
format_meta_map(Meta, Encode, [
|
||||
{packet, fun format_packet/2},
|
||||
|
|
|
@ -436,7 +436,6 @@ websocket_handle({Frame, _}, State) ->
|
|||
%% TODO: should not close the ws connection
|
||||
?LOG(error, #{msg => "unexpected_frame", frame => Frame}),
|
||||
shutdown(unexpected_ws_frame, State).
|
||||
|
||||
websocket_info({call, From, Req}, State) ->
|
||||
handle_call(From, Req, State);
|
||||
websocket_info({cast, rate_limit}, State) ->
|
||||
|
@ -738,8 +737,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
|
|||
input_bytes => Data
|
||||
}),
|
||||
FrameError = {frame_error, Reason},
|
||||
NState = enrich_state(Reason, State),
|
||||
{[{incoming, FrameError} | Packets], NState};
|
||||
{[{incoming, FrameError} | Packets], State};
|
||||
error:Reason:Stacktrace ->
|
||||
?LOG(error, #{
|
||||
at_state => emqx_frame:describe_state(ParseState),
|
||||
|
@ -832,7 +830,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
|
|||
?LOG(warning, #{
|
||||
msg => "packet_discarded",
|
||||
reason => "frame_too_large",
|
||||
packet => Packet
|
||||
packet => emqx_packet:format(Packet)
|
||||
}),
|
||||
ok = emqx_metrics:inc('delivery.dropped.too_large'),
|
||||
ok = emqx_metrics:inc('delivery.dropped'),
|
||||
|
@ -1071,13 +1069,6 @@ check_max_connection(Type, Listener) ->
|
|||
{denny, Reason}
|
||||
end
|
||||
end.
|
||||
|
||||
enrich_state(#{parse_state := NParseState}, State) ->
|
||||
Serialize = emqx_frame:serialize_opts(NParseState),
|
||||
State#state{parse_state = NParseState, serialize = Serialize};
|
||||
enrich_state(_, State) ->
|
||||
State.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% For CT tests
|
||||
%%--------------------------------------------------------------------
|
||||
|
|
|
@ -414,32 +414,24 @@ t_handle_in_auth(_) ->
|
|||
emqx_channel:handle_in(?AUTH_PACKET(), Channel).
|
||||
|
||||
t_handle_in_frame_error(_) ->
|
||||
IdleChannelV5 = channel(#{conn_state => idle}),
|
||||
%% no CONNACK packet for v4
|
||||
?assertMatch(
|
||||
{shutdown, #{shutdown_count := frame_too_large, cause := frame_too_large}, _Chan},
|
||||
emqx_channel:handle_in(
|
||||
{frame_error, #{cause => frame_too_large}}, v4(IdleChannelV5)
|
||||
)
|
||||
),
|
||||
|
||||
IdleChannel = channel(#{conn_state => idle}),
|
||||
{shutdown, #{shutdown_count := frame_too_large, cause := frame_too_large}, _Chan} =
|
||||
emqx_channel:handle_in({frame_error, #{cause => frame_too_large}}, IdleChannel),
|
||||
ConnectingChan = channel(#{conn_state => connecting}),
|
||||
ConnackPacket = ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE),
|
||||
?assertMatch(
|
||||
{shutdown,
|
||||
#{
|
||||
shutdown_count := frame_too_large,
|
||||
cause := frame_too_large,
|
||||
limit := 100,
|
||||
received := 101
|
||||
},
|
||||
ConnackPacket, _},
|
||||
{shutdown,
|
||||
#{
|
||||
shutdown_count := frame_too_large,
|
||||
cause := frame_too_large,
|
||||
limit := 100,
|
||||
received := 101
|
||||
},
|
||||
ConnackPacket,
|
||||
_} =
|
||||
emqx_channel:handle_in(
|
||||
{frame_error, #{cause => frame_too_large, received => 101, limit => 100}},
|
||||
ConnectingChan
|
||||
)
|
||||
),
|
||||
|
||||
),
|
||||
DisconnectPacket = ?DISCONNECT_PACKET(?RC_PACKET_TOO_LARGE),
|
||||
ConnectedChan = channel(#{conn_state => connected}),
|
||||
?assertMatch(
|
||||
|
|
|
@ -138,14 +138,13 @@ init_per_testcase(t_refresh_config = TestCase, Config) ->
|
|||
];
|
||||
init_per_testcase(TestCase, Config) when
|
||||
TestCase =:= t_update_listener;
|
||||
TestCase =:= t_update_listener_enable_disable;
|
||||
TestCase =:= t_validations
|
||||
->
|
||||
ct:timetrap({seconds, 30}),
|
||||
ok = snabbkaffe:start_trace(),
|
||||
%% when running emqx standalone tests, we can't use those
|
||||
%% features.
|
||||
case does_module_exist(emqx_mgmt) of
|
||||
case does_module_exist(emqx_management) of
|
||||
true ->
|
||||
DataDir = ?config(data_dir, Config),
|
||||
CRLFile = filename:join([DataDir, "intermediate-revoked.crl.pem"]),
|
||||
|
@ -166,7 +165,7 @@ init_per_testcase(TestCase, Config) when
|
|||
{emqx_conf, #{config => #{listeners => #{ssl => #{default => ListenerConf}}}}},
|
||||
emqx,
|
||||
emqx_management,
|
||||
emqx_mgmt_api_test_util:emqx_dashboard()
|
||||
{emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}
|
||||
],
|
||||
#{work_dir => emqx_cth_suite:work_dir(TestCase, Config)}
|
||||
),
|
||||
|
@ -207,7 +206,6 @@ read_crl(Filename) ->
|
|||
|
||||
end_per_testcase(TestCase, Config) when
|
||||
TestCase =:= t_update_listener;
|
||||
TestCase =:= t_update_listener_enable_disable;
|
||||
TestCase =:= t_validations
|
||||
->
|
||||
Skip = proplists:get_bool(skip_does_not_apply, Config),
|
||||
|
@ -1059,104 +1057,3 @@ do_t_validations(_Config) ->
|
|||
),
|
||||
|
||||
ok.
|
||||
|
||||
%% Checks that if CRL is ever enabled and then disabled, clients can connect, even if they
|
||||
%% would otherwise not have their corresponding CRLs cached and fail with `{bad_crls,
|
||||
%% no_relevant_crls}`.
|
||||
t_update_listener_enable_disable(Config) ->
|
||||
case proplists:get_bool(skip_does_not_apply, Config) of
|
||||
true ->
|
||||
ct:pal("skipping as this test does not apply in this profile"),
|
||||
ok;
|
||||
false ->
|
||||
do_t_update_listener_enable_disable(Config)
|
||||
end.
|
||||
|
||||
do_t_update_listener_enable_disable(Config) ->
|
||||
DataDir = ?config(data_dir, Config),
|
||||
Keyfile = filename:join([DataDir, "server.key.pem"]),
|
||||
Certfile = filename:join([DataDir, "server.cert.pem"]),
|
||||
Cacertfile = filename:join([DataDir, "ca-chain.cert.pem"]),
|
||||
ClientCert = filename:join(DataDir, "client.cert.pem"),
|
||||
ClientKey = filename:join(DataDir, "client.key.pem"),
|
||||
|
||||
ListenerId = "ssl:default",
|
||||
%% Enable CRL
|
||||
{ok, {{_, 200, _}, _, ListenerData0}} = get_listener_via_api(ListenerId),
|
||||
CRLConfig0 =
|
||||
#{
|
||||
<<"ssl_options">> =>
|
||||
#{
|
||||
<<"keyfile">> => Keyfile,
|
||||
<<"certfile">> => Certfile,
|
||||
<<"cacertfile">> => Cacertfile,
|
||||
<<"enable_crl_check">> => true,
|
||||
<<"fail_if_no_peer_cert">> => true
|
||||
}
|
||||
},
|
||||
ListenerData1 = emqx_utils_maps:deep_merge(ListenerData0, CRLConfig0),
|
||||
{ok, {_, _, ListenerData2}} = update_listener_via_api(ListenerId, ListenerData1),
|
||||
?assertMatch(
|
||||
#{
|
||||
<<"ssl_options">> :=
|
||||
#{
|
||||
<<"enable_crl_check">> := true,
|
||||
<<"verify">> := <<"verify_peer">>,
|
||||
<<"fail_if_no_peer_cert">> := true
|
||||
}
|
||||
},
|
||||
ListenerData2
|
||||
),
|
||||
|
||||
%% Disable CRL
|
||||
CRLConfig1 =
|
||||
#{
|
||||
<<"ssl_options">> =>
|
||||
#{
|
||||
<<"keyfile">> => Keyfile,
|
||||
<<"certfile">> => Certfile,
|
||||
<<"cacertfile">> => Cacertfile,
|
||||
<<"enable_crl_check">> => false,
|
||||
<<"fail_if_no_peer_cert">> => true
|
||||
}
|
||||
},
|
||||
ListenerData3 = emqx_utils_maps:deep_merge(ListenerData2, CRLConfig1),
|
||||
redbug:start(
|
||||
[
|
||||
"esockd_server:get_listener_prop -> return",
|
||||
"esockd_server:set_listener_prop -> return",
|
||||
"esockd:merge_opts -> return",
|
||||
"esockd_listener_sup:set_options -> return",
|
||||
"emqx_listeners:inject_crl_config -> return"
|
||||
],
|
||||
[{msgs, 100}]
|
||||
),
|
||||
{ok, {_, _, ListenerData4}} = update_listener_via_api(ListenerId, ListenerData3),
|
||||
?assertMatch(
|
||||
#{
|
||||
<<"ssl_options">> :=
|
||||
#{
|
||||
<<"enable_crl_check">> := false,
|
||||
<<"verify">> := <<"verify_peer">>,
|
||||
<<"fail_if_no_peer_cert">> := true
|
||||
}
|
||||
},
|
||||
ListenerData4
|
||||
),
|
||||
|
||||
%% Now the client that would be blocked tries to connect and should now be allowed.
|
||||
{ok, C} = emqtt:start_link([
|
||||
{ssl, true},
|
||||
{ssl_opts, [
|
||||
{certfile, ClientCert},
|
||||
{keyfile, ClientKey},
|
||||
{verify, verify_none}
|
||||
]},
|
||||
{port, 8883}
|
||||
]),
|
||||
?assertMatch({ok, _}, emqtt:connect(C)),
|
||||
emqtt:stop(C),
|
||||
|
||||
?assertNotReceive({http_get, _}),
|
||||
|
||||
ok.
|
||||
|
|
|
@ -63,7 +63,6 @@ groups() ->
|
|||
t_parse_malformed_properties,
|
||||
t_malformed_connect_header,
|
||||
t_malformed_connect_data,
|
||||
t_malformed_connect_data_proto_ver,
|
||||
t_reserved_connect_flag,
|
||||
t_invalid_clientid,
|
||||
t_undefined_password,
|
||||
|
@ -168,8 +167,6 @@ t_parse_malformed_utf8_string(_) ->
|
|||
ParseState = emqx_frame:initial_parse_state(#{strict_mode => true}),
|
||||
?ASSERT_FRAME_THROW(utf8_string_invalid, emqx_frame:parse(MalformedPacket, ParseState)).
|
||||
|
||||
%% TODO: parse v3 with 0 length clientid
|
||||
|
||||
t_serialize_parse_v3_connect(_) ->
|
||||
Bin =
|
||||
<<16, 37, 0, 6, 77, 81, 73, 115, 100, 112, 3, 2, 0, 60, 0, 23, 109, 111, 115, 113, 112, 117,
|
||||
|
@ -327,7 +324,7 @@ t_serialize_parse_bridge_connect(_) ->
|
|||
header = #mqtt_packet_header{type = ?CONNECT},
|
||||
variable = #mqtt_packet_connect{
|
||||
clientid = <<"C_00:0C:29:2B:77:52">>,
|
||||
proto_ver = ?MQTT_PROTO_V3,
|
||||
proto_ver = 16#03,
|
||||
proto_name = <<"MQIsdp">>,
|
||||
is_bridge = true,
|
||||
will_retain = true,
|
||||
|
@ -689,36 +686,15 @@ t_malformed_connect_header(_) ->
|
|||
).
|
||||
|
||||
t_malformed_connect_data(_) ->
|
||||
ProtoNameWithLen = <<0, 6, "MQIsdp">>,
|
||||
ConnectFlags = <<2#00000000>>,
|
||||
ClientIdwithLen = <<0, 1, "a">>,
|
||||
UnexpectedRestBin = <<0, 1, 2>>,
|
||||
?ASSERT_FRAME_THROW(
|
||||
#{cause := malformed_connect, unexpected_trailing_bytes := 3},
|
||||
emqx_frame:parse(
|
||||
<<16, 18, ProtoNameWithLen/binary, ?MQTT_PROTO_V3, ConnectFlags/binary, 0, 0,
|
||||
ClientIdwithLen/binary, UnexpectedRestBin/binary>>
|
||||
)
|
||||
).
|
||||
|
||||
t_malformed_connect_data_proto_ver(_) ->
|
||||
Proto3NameWithLen = <<0, 6, "MQIsdp">>,
|
||||
?ASSERT_FRAME_THROW(
|
||||
#{cause := malformed_connect, header_bytes := <<>>},
|
||||
emqx_frame:parse(<<16, 8, Proto3NameWithLen/binary>>)
|
||||
),
|
||||
ProtoNameWithLen = <<0, 4, "MQTT">>,
|
||||
?ASSERT_FRAME_THROW(
|
||||
#{cause := malformed_connect, header_bytes := <<>>},
|
||||
emqx_frame:parse(<<16, 6, ProtoNameWithLen/binary>>)
|
||||
#{cause := malformed_connect, unexpected_trailing_bytes := _},
|
||||
emqx_frame:parse(<<16, 15, 0, 6, 77, 81, 73, 115, 100, 112, 3, 0, 0, 0, 0, 0, 0>>)
|
||||
).
|
||||
|
||||
t_reserved_connect_flag(_) ->
|
||||
?assertException(
|
||||
throw,
|
||||
{frame_parse_error, #{
|
||||
cause := reserved_connect_flag, proto_ver := ?MQTT_PROTO_V3, proto_name := <<"MQIsdp">>
|
||||
}},
|
||||
{frame_parse_error, reserved_connect_flag},
|
||||
emqx_frame:parse(<<16, 15, 0, 6, 77, 81, 73, 115, 100, 112, 3, 1, 0, 0, 1, 0, 0>>)
|
||||
).
|
||||
|
||||
|
@ -750,7 +726,7 @@ t_undefined_password(_) ->
|
|||
},
|
||||
variable = #mqtt_packet_connect{
|
||||
proto_name = <<"MQTT">>,
|
||||
proto_ver = ?MQTT_PROTO_V4,
|
||||
proto_ver = 4,
|
||||
is_bridge = false,
|
||||
clean_start = true,
|
||||
will_flag = false,
|
||||
|
@ -798,9 +774,7 @@ t_invalid_will_retain(_) ->
|
|||
54, 75, 78, 112, 57, 0, 6, 68, 103, 55, 87, 87, 87>>,
|
||||
?assertException(
|
||||
throw,
|
||||
{frame_parse_error, #{
|
||||
cause := invalid_will_retain, proto_ver := ?MQTT_PROTO_V5, proto_name := <<"MQTT">>
|
||||
}},
|
||||
{frame_parse_error, invalid_will_retain},
|
||||
emqx_frame:parse(ConnectBin)
|
||||
),
|
||||
ok.
|
||||
|
@ -822,30 +796,22 @@ t_invalid_will_qos(_) ->
|
|||
),
|
||||
?assertException(
|
||||
throw,
|
||||
{frame_parse_error, #{
|
||||
cause := invalid_will_qos, proto_ver := ?MQTT_PROTO_V5, proto_name := <<"MQTT">>
|
||||
}},
|
||||
{frame_parse_error, invalid_will_qos},
|
||||
emqx_frame:parse(ConnectBinFun(Will_F_WillQoS1))
|
||||
),
|
||||
?assertException(
|
||||
throw,
|
||||
{frame_parse_error, #{
|
||||
cause := invalid_will_qos, proto_ver := ?MQTT_PROTO_V5, proto_name := <<"MQTT">>
|
||||
}},
|
||||
{frame_parse_error, invalid_will_qos},
|
||||
emqx_frame:parse(ConnectBinFun(Will_F_WillQoS2))
|
||||
),
|
||||
?assertException(
|
||||
throw,
|
||||
{frame_parse_error, #{
|
||||
cause := invalid_will_qos, proto_ver := ?MQTT_PROTO_V5, proto_name := <<"MQTT">>
|
||||
}},
|
||||
{frame_parse_error, invalid_will_qos},
|
||||
emqx_frame:parse(ConnectBinFun(Will_F_WillQoS3))
|
||||
),
|
||||
?assertException(
|
||||
throw,
|
||||
{frame_parse_error, #{
|
||||
cause := invalid_will_qos, proto_ver := ?MQTT_PROTO_V5, proto_name := <<"MQTT">>
|
||||
}},
|
||||
{frame_parse_error, invalid_will_qos},
|
||||
emqx_frame:parse(ConnectBinFun(Will_T_WillQoS3))
|
||||
),
|
||||
ok.
|
||||
|
|
|
@ -377,60 +377,42 @@ t_will_msg(_) ->
|
|||
|
||||
t_format(_) ->
|
||||
io:format("~ts", [
|
||||
emqx_packet:format(
|
||||
#mqtt_packet{
|
||||
header = #mqtt_packet_header{type = ?CONNACK, retain = true, dup = 0},
|
||||
variable = undefined
|
||||
},
|
||||
text
|
||||
)
|
||||
emqx_packet:format(#mqtt_packet{
|
||||
header = #mqtt_packet_header{type = ?CONNACK, retain = true, dup = 0},
|
||||
variable = undefined
|
||||
})
|
||||
]),
|
||||
io:format("~ts", [
|
||||
emqx_packet:format(#mqtt_packet{
|
||||
header = #mqtt_packet_header{type = ?CONNACK}, variable = 1, payload = <<"payload">>
|
||||
})
|
||||
]),
|
||||
io:format(
|
||||
"~ts",
|
||||
[
|
||||
emqx_packet:format(
|
||||
#mqtt_packet{
|
||||
header = #mqtt_packet_header{type = ?CONNACK},
|
||||
variable = 1,
|
||||
payload = <<"payload">>
|
||||
},
|
||||
text
|
||||
)
|
||||
]
|
||||
),
|
||||
io:format("~ts", [
|
||||
emqx_packet:format(
|
||||
?CONNECT_PACKET(
|
||||
#mqtt_packet_connect{
|
||||
will_flag = true,
|
||||
will_retain = true,
|
||||
will_qos = ?QOS_2,
|
||||
will_topic = <<"topic">>,
|
||||
will_payload = <<"payload">>
|
||||
}
|
||||
),
|
||||
text
|
||||
?CONNECT_PACKET(#mqtt_packet_connect{
|
||||
will_flag = true,
|
||||
will_retain = true,
|
||||
will_qos = ?QOS_2,
|
||||
will_topic = <<"topic">>,
|
||||
will_payload = <<"payload">>
|
||||
})
|
||||
)
|
||||
]),
|
||||
io:format("~ts", [
|
||||
emqx_packet:format(?CONNECT_PACKET(#mqtt_packet_connect{password = password}), text)
|
||||
emqx_packet:format(?CONNECT_PACKET(#mqtt_packet_connect{password = password}))
|
||||
]),
|
||||
io:format("~ts", [emqx_packet:format(?CONNACK_PACKET(?CONNACK_SERVER), text)]),
|
||||
io:format("~ts", [emqx_packet:format(?PUBLISH_PACKET(?QOS_1, 1), text)]),
|
||||
io:format("~ts", [emqx_packet:format(?CONNACK_PACKET(?CONNACK_SERVER))]),
|
||||
io:format("~ts", [emqx_packet:format(?PUBLISH_PACKET(?QOS_1, 1))]),
|
||||
io:format("~ts", [emqx_packet:format(?PUBLISH_PACKET(?QOS_2, <<"topic">>, 10, <<"payload">>))]),
|
||||
io:format("~ts", [emqx_packet:format(?PUBACK_PACKET(?PUBACK, 98))]),
|
||||
io:format("~ts", [emqx_packet:format(?PUBREL_PACKET(99))]),
|
||||
io:format("~ts", [
|
||||
emqx_packet:format(?PUBLISH_PACKET(?QOS_2, <<"topic">>, 10, <<"payload">>), text)
|
||||
emqx_packet:format(?SUBSCRIBE_PACKET(15, [{<<"topic">>, ?QOS_0}, {<<"topic1">>, ?QOS_1}]))
|
||||
]),
|
||||
io:format("~ts", [emqx_packet:format(?PUBACK_PACKET(?PUBACK, 98), text)]),
|
||||
io:format("~ts", [emqx_packet:format(?PUBREL_PACKET(99), text)]),
|
||||
io:format("~ts", [
|
||||
emqx_packet:format(
|
||||
?SUBSCRIBE_PACKET(15, [{<<"topic">>, ?QOS_0}, {<<"topic1">>, ?QOS_1}]), text
|
||||
)
|
||||
]),
|
||||
io:format("~ts", [emqx_packet:format(?SUBACK_PACKET(40, [?QOS_0, ?QOS_1]), text)]),
|
||||
io:format("~ts", [emqx_packet:format(?UNSUBSCRIBE_PACKET(89, [<<"t">>, <<"t2">>]), text)]),
|
||||
io:format("~ts", [emqx_packet:format(?UNSUBACK_PACKET(90), text)]),
|
||||
io:format("~ts", [emqx_packet:format(?DISCONNECT_PACKET(128), text)]).
|
||||
io:format("~ts", [emqx_packet:format(?SUBACK_PACKET(40, [?QOS_0, ?QOS_1]))]),
|
||||
io:format("~ts", [emqx_packet:format(?UNSUBSCRIBE_PACKET(89, [<<"t">>, <<"t2">>]))]),
|
||||
io:format("~ts", [emqx_packet:format(?UNSUBACK_PACKET(90))]),
|
||||
io:format("~ts", [emqx_packet:format(?DISCONNECT_PACKET(128))]).
|
||||
|
||||
t_parse_empty_publish(_) ->
|
||||
%% 52: 0011(type=PUBLISH) 0100 (QoS=2)
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_auth, [
|
||||
{description, "EMQX Authentication and authorization"},
|
||||
{vsn, "0.3.4"},
|
||||
{vsn, "0.3.3"},
|
||||
{modules, []},
|
||||
{registered, [emqx_auth_sup]},
|
||||
{applications, [
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_auth_http, [
|
||||
{description, "EMQX External HTTP API Authentication and Authorization"},
|
||||
{vsn, "0.3.1"},
|
||||
{vsn, "0.3.0"},
|
||||
{registered, []},
|
||||
{mod, {emqx_auth_http_app, []}},
|
||||
{applications, [
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_auth_jwt, [
|
||||
{description, "EMQX JWT Authentication and Authorization"},
|
||||
{vsn, "0.3.3"},
|
||||
{vsn, "0.3.2"},
|
||||
{registered, []},
|
||||
{mod, {emqx_auth_jwt_app, []}},
|
||||
{applications, [
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_auth_mnesia, [
|
||||
{description, "EMQX Buitl-in Database Authentication and Authorization"},
|
||||
{vsn, "0.1.7"},
|
||||
{vsn, "0.1.6"},
|
||||
{registered, []},
|
||||
{mod, {emqx_auth_mnesia_app, []}},
|
||||
{applications, [
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_auth_mongodb, [
|
||||
{description, "EMQX MongoDB Authentication and Authorization"},
|
||||
{vsn, "0.2.2"},
|
||||
{vsn, "0.2.1"},
|
||||
{registered, []},
|
||||
{mod, {emqx_auth_mongodb_app, []}},
|
||||
{applications, [
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_auth_mysql, [
|
||||
{description, "EMQX MySQL Authentication and Authorization"},
|
||||
{vsn, "0.2.2"},
|
||||
{vsn, "0.2.1"},
|
||||
{registered, []},
|
||||
{mod, {emqx_auth_mysql_app, []}},
|
||||
{applications, [
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_auth_postgresql, [
|
||||
{description, "EMQX PostgreSQL Authentication and Authorization"},
|
||||
{vsn, "0.2.2"},
|
||||
{vsn, "0.2.1"},
|
||||
{registered, []},
|
||||
{mod, {emqx_auth_postgresql_app, []}},
|
||||
{applications, [
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_auth_redis, [
|
||||
{description, "EMQX Redis Authentication and Authorization"},
|
||||
{vsn, "0.2.2"},
|
||||
{vsn, "0.2.1"},
|
||||
{registered, []},
|
||||
{mod, {emqx_auth_redis_app, []}},
|
||||
{applications, [
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_bridge, [
|
||||
{description, "EMQX bridges"},
|
||||
{vsn, "0.2.4"},
|
||||
{vsn, "0.2.3"},
|
||||
{registered, [emqx_bridge_sup]},
|
||||
{mod, {emqx_bridge_app, []}},
|
||||
{applications, [
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{application, emqx_bridge_gcp_pubsub, [
|
||||
{description, "EMQX Enterprise GCP Pub/Sub Bridge"},
|
||||
{vsn, "0.3.3"},
|
||||
{vsn, "0.3.2"},
|
||||
{registered, []},
|
||||
{applications, [
|
||||
kernel,
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{application, emqx_bridge_http, [
|
||||
{description, "EMQX HTTP Bridge and Connector Application"},
|
||||
{vsn, "0.3.4"},
|
||||
{vsn, "0.3.3"},
|
||||
{registered, []},
|
||||
{applications, [kernel, stdlib, emqx_resource, ehttpc]},
|
||||
{env, [
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_bridge_kafka, [
|
||||
{description, "EMQX Enterprise Kafka Bridge"},
|
||||
{vsn, "0.3.4"},
|
||||
{vsn, "0.3.3"},
|
||||
{registered, [emqx_bridge_kafka_consumer_sup]},
|
||||
{applications, [
|
||||
kernel,
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_bridge_mqtt, [
|
||||
{description, "EMQX MQTT Broker Bridge"},
|
||||
{vsn, "0.2.4"},
|
||||
{vsn, "0.2.3"},
|
||||
{registered, []},
|
||||
{applications, [
|
||||
kernel,
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{application, emqx_bridge_pulsar, [
|
||||
{description, "EMQX Pulsar Bridge"},
|
||||
{vsn, "0.2.4"},
|
||||
{vsn, "0.2.3"},
|
||||
{registered, []},
|
||||
{applications, [
|
||||
kernel,
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{application, emqx_bridge_rabbitmq, [
|
||||
{description, "EMQX Enterprise RabbitMQ Bridge"},
|
||||
{vsn, "0.2.3"},
|
||||
{vsn, "0.2.2"},
|
||||
{registered, []},
|
||||
{mod, {emqx_bridge_rabbitmq_app, []}},
|
||||
{applications, [
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{application, emqx_bridge_s3, [
|
||||
{description, "EMQX Enterprise S3 Bridge"},
|
||||
{vsn, "0.1.6"},
|
||||
{vsn, "0.1.5"},
|
||||
{registered, []},
|
||||
{applications, [
|
||||
kernel,
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{application, emqx_bridge_sqlserver, [
|
||||
{description, "EMQX Enterprise SQL Server Bridge"},
|
||||
{vsn, "0.2.4"},
|
||||
{vsn, "0.2.3"},
|
||||
{registered, []},
|
||||
{applications, [kernel, stdlib, emqx_resource, odbc]},
|
||||
{env, [
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{application, emqx_bridge_syskeeper, [
|
||||
{description, "EMQX Enterprise Data bridge for Syskeeper"},
|
||||
{vsn, "0.1.5"},
|
||||
{vsn, "0.1.4"},
|
||||
{registered, []},
|
||||
{applications, [
|
||||
kernel,
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{application, emqx_conf, [
|
||||
{description, "EMQX configuration management"},
|
||||
{vsn, "0.2.4"},
|
||||
{vsn, "0.2.3"},
|
||||
{registered, []},
|
||||
{mod, {emqx_conf_app, []}},
|
||||
{applications, [kernel, stdlib]},
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_connector, [
|
||||
{description, "EMQX Data Integration Connectors"},
|
||||
{vsn, "0.3.4"},
|
||||
{vsn, "0.3.3"},
|
||||
{registered, []},
|
||||
{mod, {emqx_connector_app, []}},
|
||||
{applications, [
|
||||
|
|
|
@ -125,7 +125,6 @@ create(Type, Name, Conf0, Opts) ->
|
|||
TypeBin = bin(Type),
|
||||
ResourceId = resource_id(Type, Name),
|
||||
Conf = Conf0#{connector_type => TypeBin, connector_name => Name},
|
||||
_ = emqx_alarm:ensure_deactivated(ResourceId),
|
||||
{ok, _Data} = emqx_resource:create_local(
|
||||
ResourceId,
|
||||
?CONNECTOR_RESOURCE_GROUP,
|
||||
|
@ -133,6 +132,7 @@ create(Type, Name, Conf0, Opts) ->
|
|||
parse_confs(TypeBin, Name, Conf),
|
||||
parse_opts(Conf, Opts)
|
||||
),
|
||||
_ = emqx_alarm:ensure_deactivated(ResourceId),
|
||||
ok.
|
||||
|
||||
update(ConnectorId, {OldConf, Conf}) ->
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
{application, emqx_dashboard, [
|
||||
{description, "EMQX Web Dashboard"},
|
||||
% strict semver, bump manually!
|
||||
{vsn, "5.1.4"},
|
||||
{vsn, "5.1.3"},
|
||||
{modules, []},
|
||||
{registered, [emqx_dashboard_sup]},
|
||||
{applications, [
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{application, emqx_dashboard_sso, [
|
||||
{description, "EMQX Dashboard Single Sign-On"},
|
||||
{vsn, "0.1.6"},
|
||||
{vsn, "0.1.5"},
|
||||
{registered, [emqx_dashboard_sso_sup]},
|
||||
{applications, [
|
||||
kernel,
|
||||
|
|
|
@ -689,18 +689,9 @@ all() ->
|
|||
|
||||
groups() ->
|
||||
TCs = emqx_common_test_helpers:all(?MODULE),
|
||||
%% TODO: Remove once builtin-local supports preconditions + atomic batches.
|
||||
BuiltinLocalTCs =
|
||||
TCs --
|
||||
[
|
||||
t_09_atomic_store_batch,
|
||||
t_11_batch_preconditions,
|
||||
t_12_batch_precondition_conflicts
|
||||
],
|
||||
BuiltinRaftTCs = TCs,
|
||||
[
|
||||
{builtin_local, BuiltinLocalTCs},
|
||||
{builtin_raft, BuiltinRaftTCs}
|
||||
{builtin_local, TCs},
|
||||
{builtin_raft, TCs}
|
||||
].
|
||||
|
||||
init_per_group(builtin_local, Config) ->
|
||||
|
|
|
@ -49,7 +49,9 @@
|
|||
%% Internal exports:
|
||||
-export([
|
||||
do_next/3,
|
||||
do_delete_next/4
|
||||
do_delete_next/4,
|
||||
%% Used by batch serializer
|
||||
make_batch/3
|
||||
]).
|
||||
|
||||
-export_type([db_opts/0, shard/0, iterator/0, delete_iterator/0]).
|
||||
|
@ -88,7 +90,10 @@
|
|||
#{
|
||||
backend := builtin_local,
|
||||
storage := emqx_ds_storage_layer:prototype(),
|
||||
n_shards := pos_integer()
|
||||
n_shards := pos_integer(),
|
||||
%% Inherited from `emqx_ds:generic_db_opts()`.
|
||||
force_monotonic_timestamps => boolean(),
|
||||
atomic_batches => boolean()
|
||||
}.
|
||||
|
||||
-type generation_rank() :: {shard(), emqx_ds_storage_layer:gen_id()}.
|
||||
|
@ -193,9 +198,17 @@ drop_db(DB) ->
|
|||
),
|
||||
emqx_ds_builtin_local_meta:drop_db(DB).
|
||||
|
||||
-spec store_batch(emqx_ds:db(), [emqx_types:message()], emqx_ds:message_store_opts()) ->
|
||||
-spec store_batch(emqx_ds:db(), emqx_ds:batch(), emqx_ds:message_store_opts()) ->
|
||||
emqx_ds:store_batch_result().
|
||||
store_batch(DB, Messages, Opts) ->
|
||||
store_batch(DB, Batch, Opts) ->
|
||||
case emqx_ds_builtin_local_meta:db_config(DB) of
|
||||
#{atomic_batches := true} ->
|
||||
store_batch_atomic(DB, Batch, Opts);
|
||||
_ ->
|
||||
store_batch_buffered(DB, Batch, Opts)
|
||||
end.
|
||||
|
||||
store_batch_buffered(DB, Messages, Opts) ->
|
||||
try
|
||||
emqx_ds_buffer:store_batch(DB, Messages, Opts)
|
||||
catch
|
||||
|
@ -203,6 +216,34 @@ store_batch(DB, Messages, Opts) ->
|
|||
{error, recoverable, Reason}
|
||||
end.
|
||||
|
||||
store_batch_atomic(DB, Batch, Opts) ->
|
||||
Shards = shards_of_batch(DB, Batch),
|
||||
case Shards of
|
||||
[Shard] ->
|
||||
emqx_ds_builtin_local_batch_serializer:store_batch_atomic(DB, Shard, Batch, Opts);
|
||||
[] ->
|
||||
ok;
|
||||
[_ | _] ->
|
||||
{error, unrecoverable, atomic_batch_spans_multiple_shards}
|
||||
end.
|
||||
|
||||
shards_of_batch(DB, #dsbatch{operations = Operations, preconditions = Preconditions}) ->
|
||||
shards_of_batch(DB, Preconditions, shards_of_batch(DB, Operations, []));
|
||||
shards_of_batch(DB, Operations) ->
|
||||
shards_of_batch(DB, Operations, []).
|
||||
|
||||
shards_of_batch(DB, [Operation | Rest], Acc) ->
|
||||
case shard_of_operation(DB, Operation, clientid, #{}) of
|
||||
Shard when Shard =:= hd(Acc) ->
|
||||
shards_of_batch(DB, Rest, Acc);
|
||||
Shard when Acc =:= [] ->
|
||||
shards_of_batch(DB, Rest, [Shard]);
|
||||
ShardAnother ->
|
||||
[ShardAnother | Acc]
|
||||
end;
|
||||
shards_of_batch(_DB, [], Acc) ->
|
||||
Acc.
|
||||
|
||||
-record(bs, {options :: emqx_ds:create_db_opts()}).
|
||||
-type buffer_state() :: #bs{}.
|
||||
|
||||
|
|
|
@ -0,0 +1,122 @@
|
|||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%
|
||||
%% Licensed under the Apache License, Version 2.0 (the "License");
|
||||
%% you may not use this file except in compliance with the License.
|
||||
%% You may obtain a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing, software
|
||||
%% distributed under the License is distributed on an "AS IS" BASIS,
|
||||
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
%% See the License for the specific language governing permissions and
|
||||
%% limitations under the License.
|
||||
%%--------------------------------------------------------------------
|
||||
-module(emqx_ds_builtin_local_batch_serializer).
|
||||
|
||||
-include_lib("emqx_durable_storage/include/emqx_ds.hrl").
|
||||
|
||||
%% API
|
||||
-export([
|
||||
start_link/3,
|
||||
|
||||
store_batch_atomic/4
|
||||
]).
|
||||
|
||||
%% `gen_server' API
|
||||
-export([
|
||||
init/1,
|
||||
handle_call/3,
|
||||
handle_cast/2
|
||||
]).
|
||||
|
||||
%%------------------------------------------------------------------------------
|
||||
%% Type declarations
|
||||
%%------------------------------------------------------------------------------
|
||||
|
||||
-define(name(DB, SHARD), {n, l, {?MODULE, DB, SHARD}}).
|
||||
-define(via(DB, SHARD), {via, gproc, ?name(DB, SHARD)}).
|
||||
|
||||
-record(store_batch_atomic, {batch :: emqx_ds:batch(), opts :: emqx_ds:message_store_opts()}).
|
||||
|
||||
%%------------------------------------------------------------------------------
|
||||
%% API
|
||||
%%------------------------------------------------------------------------------
|
||||
|
||||
start_link(DB, Shard, _Opts) ->
|
||||
gen_server:start_link(?via(DB, Shard), ?MODULE, [DB, Shard], []).
|
||||
|
||||
store_batch_atomic(DB, Shard, Batch, Opts) ->
|
||||
gen_server:call(?via(DB, Shard), #store_batch_atomic{batch = Batch, opts = Opts}, infinity).
|
||||
|
||||
%%------------------------------------------------------------------------------
|
||||
%% `gen_server' API
|
||||
%%------------------------------------------------------------------------------
|
||||
|
||||
init([DB, Shard]) ->
|
||||
process_flag(message_queue_data, off_heap),
|
||||
State = #{
|
||||
db => DB,
|
||||
shard => Shard
|
||||
},
|
||||
{ok, State}.
|
||||
|
||||
handle_call(#store_batch_atomic{batch = Batch, opts = StoreOpts}, _From, State) ->
|
||||
ShardId = shard_id(State),
|
||||
DBOpts = db_config(State),
|
||||
Result = do_store_batch_atomic(ShardId, Batch, DBOpts, StoreOpts),
|
||||
{reply, Result, State};
|
||||
handle_call(Call, _From, State) ->
|
||||
{reply, {error, {unknown_call, Call}}, State}.
|
||||
|
||||
handle_cast(_Cast, State) ->
|
||||
{noreply, State}.
|
||||
|
||||
%%------------------------------------------------------------------------------
|
||||
%% Internal fns
|
||||
%%------------------------------------------------------------------------------
|
||||
|
||||
shard_id(#{db := DB, shard := Shard}) ->
|
||||
{DB, Shard}.
|
||||
|
||||
db_config(#{db := DB}) ->
|
||||
emqx_ds_builtin_local_meta:db_config(DB).
|
||||
|
||||
-spec do_store_batch_atomic(
|
||||
emqx_ds_storage_layer:shard_id(),
|
||||
emqx_ds:dsbatch(),
|
||||
emqx_ds_builtin_local:db_opts(),
|
||||
emqx_ds:message_store_opts()
|
||||
) ->
|
||||
emqx_ds:store_batch_result().
|
||||
do_store_batch_atomic(ShardId, #dsbatch{} = Batch, DBOpts, StoreOpts) ->
|
||||
#dsbatch{
|
||||
operations = Operations0,
|
||||
preconditions = Preconditions
|
||||
} = Batch,
|
||||
case emqx_ds_precondition:verify(emqx_ds_storage_layer, ShardId, Preconditions) of
|
||||
ok ->
|
||||
do_store_operations(ShardId, Operations0, DBOpts, StoreOpts);
|
||||
{precondition_failed, _} = PreconditionFailed ->
|
||||
{error, unrecoverable, PreconditionFailed};
|
||||
Error ->
|
||||
Error
|
||||
end;
|
||||
do_store_batch_atomic(ShardId, Operations, DBOpts, StoreOpts) ->
|
||||
do_store_operations(ShardId, Operations, DBOpts, StoreOpts).
|
||||
|
||||
do_store_operations(ShardId, Operations0, DBOpts, _StoreOpts) ->
|
||||
ForceMonotonic = maps:get(force_monotonic_timestamps, DBOpts),
|
||||
{Latest, Operations} =
|
||||
emqx_ds_builtin_local:make_batch(
|
||||
ForceMonotonic,
|
||||
current_timestamp(ShardId),
|
||||
Operations0
|
||||
),
|
||||
Result = emqx_ds_storage_layer:store_batch(ShardId, Operations, _Options = #{}),
|
||||
emqx_ds_builtin_local_meta:set_current_timestamp(ShardId, Latest),
|
||||
Result.
|
||||
|
||||
current_timestamp(ShardId) ->
|
||||
emqx_ds_builtin_local_meta:current_timestamp(ShardId).
|
|
@ -158,7 +158,8 @@ init({#?shard_sup{db = DB, shard = Shard}, _}) ->
|
|||
Opts = emqx_ds_builtin_local_meta:db_config(DB),
|
||||
Children = [
|
||||
shard_storage_spec(DB, Shard, Opts),
|
||||
shard_buffer_spec(DB, Shard, Opts)
|
||||
shard_buffer_spec(DB, Shard, Opts),
|
||||
shard_batch_serializer_spec(DB, Shard, Opts)
|
||||
],
|
||||
{ok, {SupFlags, Children}}.
|
||||
|
||||
|
@ -208,6 +209,15 @@ shard_buffer_spec(DB, Shard, Options) ->
|
|||
type => worker
|
||||
}.
|
||||
|
||||
shard_batch_serializer_spec(DB, Shard, Opts) ->
|
||||
#{
|
||||
id => {Shard, batch_serializer},
|
||||
start => {emqx_ds_builtin_local_batch_serializer, start_link, [DB, Shard, Opts]},
|
||||
shutdown => 5_000,
|
||||
restart => permanent,
|
||||
type => worker
|
||||
}.
|
||||
|
||||
ensure_started(Res) ->
|
||||
case Res of
|
||||
{ok, _Pid} ->
|
||||
|
|
|
@ -479,10 +479,10 @@ shards_of_batch(_DB, [], Acc) ->
|
|||
%% TODO
|
||||
%% There's a possibility of race condition: storage may shut down right after we
|
||||
%% ask for its status.
|
||||
-define(IF_STORAGE_RUNNING(SHARDID, EXPR),
|
||||
case emqx_ds_storage_layer:shard_info(SHARDID, status) of
|
||||
running -> EXPR;
|
||||
down -> {error, recoverable, storage_down}
|
||||
-define(IF_SHARD_READY(DB, SHARD, EXPR),
|
||||
case emqx_ds_replication_layer_shard:shard_info(DB, SHARD, ready) of
|
||||
true -> EXPR;
|
||||
false -> {error, recoverable, shard_unavailable}
|
||||
end
|
||||
).
|
||||
|
||||
|
@ -525,8 +525,9 @@ do_get_streams_v1(_DB, _Shard, _TopicFilter, _StartTime) ->
|
|||
[{integer(), emqx_ds_storage_layer:stream()}] | emqx_ds:error(storage_down).
|
||||
do_get_streams_v2(DB, Shard, TopicFilter, StartTime) ->
|
||||
ShardId = {DB, Shard},
|
||||
?IF_STORAGE_RUNNING(
|
||||
ShardId,
|
||||
?IF_SHARD_READY(
|
||||
DB,
|
||||
Shard,
|
||||
emqx_ds_storage_layer:get_streams(ShardId, TopicFilter, StartTime)
|
||||
).
|
||||
|
||||
|
@ -552,8 +553,9 @@ do_make_iterator_v1(_DB, _Shard, _Stream, _TopicFilter, _StartTime) ->
|
|||
emqx_ds:make_iterator_result(emqx_ds_storage_layer:iterator()).
|
||||
do_make_iterator_v2(DB, Shard, Stream, TopicFilter, StartTime) ->
|
||||
ShardId = {DB, Shard},
|
||||
?IF_STORAGE_RUNNING(
|
||||
ShardId,
|
||||
?IF_SHARD_READY(
|
||||
DB,
|
||||
Shard,
|
||||
emqx_ds_storage_layer:make_iterator(ShardId, Stream, TopicFilter, StartTime)
|
||||
).
|
||||
|
||||
|
@ -587,8 +589,9 @@ do_update_iterator_v2(DB, Shard, OldIter, DSKey) ->
|
|||
emqx_ds:next_result(emqx_ds_storage_layer:iterator()).
|
||||
do_next_v1(DB, Shard, Iter, BatchSize) ->
|
||||
ShardId = {DB, Shard},
|
||||
?IF_STORAGE_RUNNING(
|
||||
ShardId,
|
||||
?IF_SHARD_READY(
|
||||
DB,
|
||||
Shard,
|
||||
emqx_ds_storage_layer:next(
|
||||
ShardId, Iter, BatchSize, emqx_ds_replication_layer:current_timestamp(DB, Shard)
|
||||
)
|
||||
|
@ -620,8 +623,9 @@ do_add_generation_v2(_DB) ->
|
|||
| emqx_ds:error(storage_down).
|
||||
do_list_generations_with_lifetimes_v3(DB, Shard) ->
|
||||
ShardId = {DB, Shard},
|
||||
?IF_STORAGE_RUNNING(
|
||||
ShardId,
|
||||
?IF_SHARD_READY(
|
||||
DB,
|
||||
Shard,
|
||||
emqx_ds_storage_layer:list_generations_with_lifetimes(ShardId)
|
||||
).
|
||||
|
||||
|
|
|
@ -18,7 +18,8 @@
|
|||
|
||||
%% Dynamic server location API
|
||||
-export([
|
||||
servers/3
|
||||
servers/3,
|
||||
shard_info/3
|
||||
]).
|
||||
|
||||
%% Safe Process Command API
|
||||
|
@ -38,8 +39,10 @@
|
|||
-behaviour(gen_server).
|
||||
-export([
|
||||
init/1,
|
||||
handle_continue/2,
|
||||
handle_call/3,
|
||||
handle_cast/2,
|
||||
handle_info/2,
|
||||
terminate/2
|
||||
]).
|
||||
|
||||
|
@ -52,6 +55,9 @@
|
|||
| {error, servers_unreachable}.
|
||||
|
||||
-define(MEMBERSHIP_CHANGE_TIMEOUT, 30_000).
|
||||
-define(MAX_BOOSTRAP_RETRY_TIMEOUT, 1_000).
|
||||
|
||||
-define(PTERM(DB, SHARD, KEY), {?MODULE, DB, SHARD, KEY}).
|
||||
|
||||
%%
|
||||
|
||||
|
@ -160,6 +166,12 @@ local_site() ->
|
|||
|
||||
%%
|
||||
|
||||
-spec shard_info(emqx_ds:db(), emqx_ds_replication_layer:shard_id(), _Info) -> _Value.
|
||||
shard_info(DB, Shard, ready) ->
|
||||
get_shard_info(DB, Shard, ready, false).
|
||||
|
||||
%%
|
||||
|
||||
-spec process_command([server()], _Command, timeout()) ->
|
||||
{ok, _Result, _Leader :: server()} | server_error().
|
||||
process_command(Servers, Command, Timeout) ->
|
||||
|
@ -324,10 +336,45 @@ ra_overview(Server) ->
|
|||
|
||||
%%
|
||||
|
||||
-record(st, {
|
||||
db :: emqx_ds:db(),
|
||||
shard :: emqx_ds_replication_layer:shard_id(),
|
||||
server :: server(),
|
||||
bootstrapped :: boolean(),
|
||||
stage :: term()
|
||||
}).
|
||||
|
||||
init({DB, Shard, Opts}) ->
|
||||
_ = process_flag(trap_exit, true),
|
||||
ok = start_server(DB, Shard, Opts),
|
||||
{ok, {DB, Shard}}.
|
||||
case start_server(DB, Shard, Opts) of
|
||||
{_New = true, Server} ->
|
||||
NextStage = trigger_election;
|
||||
{_New = false, Server} ->
|
||||
NextStage = wait_leader
|
||||
end,
|
||||
St = #st{
|
||||
db = DB,
|
||||
shard = Shard,
|
||||
server = Server,
|
||||
bootstrapped = false,
|
||||
stage = NextStage
|
||||
},
|
||||
{ok, St, {continue, bootstrap}}.
|
||||
|
||||
handle_continue(bootstrap, St = #st{bootstrapped = true}) ->
|
||||
{noreply, St};
|
||||
handle_continue(bootstrap, St0 = #st{db = DB, shard = Shard, stage = Stage}) ->
|
||||
?tp(emqx_ds_replshard_bootstrapping, #{db => DB, shard => Shard, stage => Stage}),
|
||||
case bootstrap(St0) of
|
||||
St = #st{bootstrapped = true} ->
|
||||
?tp(emqx_ds_replshard_bootstrapped, #{db => DB, shard => Shard}),
|
||||
{noreply, St};
|
||||
St = #st{bootstrapped = false} ->
|
||||
{noreply, St, {continue, bootstrap}};
|
||||
{retry, Timeout, St} ->
|
||||
_TRef = erlang:start_timer(Timeout, self(), bootstrap),
|
||||
{noreply, St}
|
||||
end.
|
||||
|
||||
handle_call(_Call, _From, State) ->
|
||||
{reply, ignored, State}.
|
||||
|
@ -335,7 +382,14 @@ handle_call(_Call, _From, State) ->
|
|||
handle_cast(_Msg, State) ->
|
||||
{noreply, State}.
|
||||
|
||||
terminate(_Reason, {DB, Shard}) ->
|
||||
handle_info({timeout, _TRef, bootstrap}, St) ->
|
||||
{noreply, St, {continue, bootstrap}};
|
||||
handle_info(_Info, State) ->
|
||||
{noreply, State}.
|
||||
|
||||
terminate(_Reason, #st{db = DB, shard = Shard}) ->
|
||||
%% NOTE: Mark as not ready right away.
|
||||
ok = erase_shard_info(DB, Shard),
|
||||
%% NOTE: Timeouts are ignored, it's a best effort attempt.
|
||||
catch prep_stop_server(DB, Shard),
|
||||
LocalServer = get_local_server(DB, Shard),
|
||||
|
@ -343,6 +397,40 @@ terminate(_Reason, {DB, Shard}) ->
|
|||
|
||||
%%
|
||||
|
||||
bootstrap(St = #st{stage = trigger_election, server = Server}) ->
|
||||
ok = trigger_election(Server),
|
||||
St#st{stage = wait_leader};
|
||||
bootstrap(St = #st{stage = wait_leader, server = Server}) ->
|
||||
case current_leader(Server) of
|
||||
Leader = {_, _} ->
|
||||
St#st{stage = {wait_log, Leader}};
|
||||
unknown ->
|
||||
St
|
||||
end;
|
||||
bootstrap(St = #st{stage = {wait_log, Leader}}) ->
|
||||
case ra_overview(Leader) of
|
||||
#{commit_index := RaftIdx} ->
|
||||
St#st{stage = {wait_log_index, RaftIdx}};
|
||||
#{} ->
|
||||
St#st{stage = wait_leader}
|
||||
end;
|
||||
bootstrap(St = #st{stage = {wait_log_index, RaftIdx}, db = DB, shard = Shard, server = Server}) ->
|
||||
Overview = ra_overview(Server),
|
||||
case maps:get(last_applied, Overview, 0) of
|
||||
LastApplied when LastApplied >= RaftIdx ->
|
||||
ok = announce_shard_ready(DB, Shard),
|
||||
St#st{bootstrapped = true, stage = undefined};
|
||||
LastApplied ->
|
||||
%% NOTE
|
||||
%% Blunt estimate of time shard needs to catch up. If this proves to be too long in
|
||||
%% practice, it's could be augmented with handling `recover` -> `follower` Ra
|
||||
%% member state transition.
|
||||
Timeout = min(RaftIdx - LastApplied, ?MAX_BOOSTRAP_RETRY_TIMEOUT),
|
||||
{retry, Timeout, St}
|
||||
end.
|
||||
|
||||
%%
|
||||
|
||||
start_server(DB, Shard, #{replication_options := ReplicationOpts}) ->
|
||||
ClusterName = cluster_name(DB, Shard),
|
||||
LocalServer = local_server(DB, Shard),
|
||||
|
@ -350,7 +438,6 @@ start_server(DB, Shard, #{replication_options := ReplicationOpts}) ->
|
|||
MutableConfig = #{tick_timeout => 100},
|
||||
case ra:restart_server(DB, LocalServer, MutableConfig) of
|
||||
{error, name_not_registered} ->
|
||||
Bootstrap = true,
|
||||
Machine = {module, emqx_ds_replication_layer, #{db => DB, shard => Shard}},
|
||||
LogOpts = maps:with(
|
||||
[
|
||||
|
@ -366,30 +453,34 @@ start_server(DB, Shard, #{replication_options := ReplicationOpts}) ->
|
|||
initial_members => Servers,
|
||||
machine => Machine,
|
||||
log_init_args => LogOpts
|
||||
});
|
||||
}),
|
||||
{_NewServer = true, LocalServer};
|
||||
ok ->
|
||||
Bootstrap = false;
|
||||
{_NewServer = false, LocalServer};
|
||||
{error, {already_started, _}} ->
|
||||
Bootstrap = false
|
||||
end,
|
||||
{_NewServer = false, LocalServer}
|
||||
end.
|
||||
|
||||
trigger_election(Server) ->
|
||||
%% NOTE
|
||||
%% Triggering election is necessary when a new consensus group is being brought up.
|
||||
%% TODO
|
||||
%% It's probably a good idea to rebalance leaders across the cluster from time to
|
||||
%% time. There's `ra:transfer_leadership/2` for that.
|
||||
try Bootstrap andalso ra:trigger_election(LocalServer, _Timeout = 1_000) of
|
||||
false ->
|
||||
ok;
|
||||
ok ->
|
||||
ok
|
||||
try ra:trigger_election(Server) of
|
||||
ok -> ok
|
||||
catch
|
||||
%% TODO
|
||||
%% NOTE
|
||||
%% Tolerating exceptions because server might be occupied with log replay for
|
||||
%% a while.
|
||||
exit:{timeout, _} when not Bootstrap ->
|
||||
exit:{timeout, _} ->
|
||||
?tp(emqx_ds_replshard_trigger_election, #{server => Server, error => timeout}),
|
||||
ok
|
||||
end.
|
||||
|
||||
announce_shard_ready(DB, Shard) ->
|
||||
set_shard_info(DB, Shard, ready, true).
|
||||
|
||||
server_uid(_DB, Shard) ->
|
||||
%% NOTE
|
||||
%% Each new "instance" of a server should have a unique identifier. Otherwise,
|
||||
|
@ -402,6 +493,22 @@ server_uid(_DB, Shard) ->
|
|||
|
||||
%%
|
||||
|
||||
get_shard_info(DB, Shard, K, Default) ->
|
||||
persistent_term:get(?PTERM(DB, Shard, K), Default).
|
||||
|
||||
set_shard_info(DB, Shard, K, V) ->
|
||||
persistent_term:put(?PTERM(DB, Shard, K), V).
|
||||
|
||||
erase_shard_info(DB, Shard) ->
|
||||
lists:foreach(fun(K) -> erase_shard_info(DB, Shard, K) end, [
|
||||
ready
|
||||
]).
|
||||
|
||||
erase_shard_info(DB, Shard, K) ->
|
||||
persistent_term:erase(?PTERM(DB, Shard, K)).
|
||||
|
||||
%%
|
||||
|
||||
prep_stop_server(DB, Shard) ->
|
||||
prep_stop_server(DB, Shard, 5_000).
|
||||
|
||||
|
|
|
@ -131,7 +131,6 @@ t_replication_transfers_snapshots(Config) ->
|
|||
%% Initialize DB on all nodes and wait for it to be online.
|
||||
Opts = opts(Config, #{n_shards => 1, n_sites => 3}),
|
||||
assert_db_open(Nodes, ?DB, Opts),
|
||||
assert_db_stable(Nodes, ?DB),
|
||||
|
||||
%% Stop the DB on the "offline" node.
|
||||
?wait_async_action(
|
||||
|
@ -207,7 +206,6 @@ t_rebalance(Config) ->
|
|||
%% 1. Initialize DB on the first node.
|
||||
Opts = opts(Config, #{n_shards => 16, n_sites => 1, replication_factor => 3}),
|
||||
assert_db_open(Nodes, ?DB, Opts),
|
||||
assert_db_stable(Nodes, ?DB),
|
||||
|
||||
%% 1.1 Kick all sites except S1 from the replica set as
|
||||
%% the initial condition:
|
||||
|
@ -419,7 +417,6 @@ t_rebalance_chaotic_converges(Config) ->
|
|||
|
||||
%% Open DB:
|
||||
assert_db_open(Nodes, ?DB, Opts),
|
||||
assert_db_stable(Nodes, ?DB),
|
||||
|
||||
%% Kick N3 from the replica set as the initial condition:
|
||||
?assertMatch(
|
||||
|
@ -503,7 +500,6 @@ t_rebalance_offline_restarts(Config) ->
|
|||
%% Initialize DB on all 3 nodes.
|
||||
Opts = opts(Config, #{n_shards => 8, n_sites => 3, replication_factor => 3}),
|
||||
assert_db_open(Nodes, ?DB, Opts),
|
||||
assert_db_stable(Nodes, ?DB),
|
||||
|
||||
?retry(
|
||||
1000,
|
||||
|
@ -845,13 +841,11 @@ t_crash_restart_recover(Config) ->
|
|||
?check_trace(
|
||||
begin
|
||||
%% Initialize DB on all nodes.
|
||||
?assertEqual(
|
||||
[{ok, ok} || _ <- Nodes],
|
||||
erpc:multicall(Nodes, emqx_ds, open_db, [?DB, DBOpts])
|
||||
),
|
||||
assert_db_open(Nodes, ?DB, DBOpts),
|
||||
|
||||
%% Apply the test events, including simulated node crashes.
|
||||
NodeStream = emqx_utils_stream:const(N1),
|
||||
StartedAt = erlang:monotonic_time(millisecond),
|
||||
emqx_ds_test_helpers:apply_stream(?DB, NodeStream, Stream, 0),
|
||||
|
||||
%% It's expected to lose few messages when leaders are abruptly killed.
|
||||
|
@ -865,6 +859,10 @@ t_crash_restart_recover(Config) ->
|
|||
ct:pal("Some messages were lost: ~p", [LostMessages]),
|
||||
?assert(length(LostMessages) < NMsgs div 20),
|
||||
|
||||
%% Wait until crashed nodes are ready.
|
||||
SinceStarted = erlang:monotonic_time(millisecond) - StartedAt,
|
||||
wait_db_bootstrapped([N2, N3], ?DB, infinity, SinceStarted),
|
||||
|
||||
%% Verify that all the successfully persisted messages are there.
|
||||
VerifyClient = fun({ClientId, ExpectedStream}) ->
|
||||
Topic = emqx_ds_test_helpers:client_topic(?FUNCTION_NAME, ClientId),
|
||||
|
@ -926,7 +924,8 @@ assert_db_open(Nodes, DB, Opts) ->
|
|||
?assertEqual(
|
||||
[{ok, ok} || _ <- Nodes],
|
||||
erpc:multicall(Nodes, emqx_ds, open_db, [DB, Opts])
|
||||
).
|
||||
),
|
||||
wait_db_bootstrapped(Nodes, ?DB).
|
||||
|
||||
assert_db_stable([Node | _], DB) ->
|
||||
Shards = ds_repl_meta(Node, shards, [DB]),
|
||||
|
@ -935,6 +934,32 @@ assert_db_stable([Node | _], DB) ->
|
|||
db_leadership(Node, DB, Shards)
|
||||
).
|
||||
|
||||
wait_db_bootstrapped(Nodes, DB) ->
|
||||
wait_db_bootstrapped(Nodes, DB, infinity, infinity).
|
||||
|
||||
wait_db_bootstrapped(Nodes, DB, Timeout, BackInTime) ->
|
||||
SRefs = [
|
||||
snabbkaffe:subscribe(
|
||||
?match_event(#{
|
||||
?snk_kind := emqx_ds_replshard_bootstrapped,
|
||||
?snk_meta := #{node := Node},
|
||||
db := DB,
|
||||
shard := Shard
|
||||
}),
|
||||
1,
|
||||
Timeout,
|
||||
BackInTime
|
||||
)
|
||||
|| Node <- Nodes,
|
||||
Shard <- ds_repl_meta(Node, my_shards, [DB])
|
||||
],
|
||||
lists:foreach(
|
||||
fun({ok, SRef}) ->
|
||||
?assertMatch({ok, [_]}, snabbkaffe:receive_events(SRef))
|
||||
end,
|
||||
SRefs
|
||||
).
|
||||
|
||||
%%
|
||||
|
||||
db_leadership(Node, DB, Shards) ->
|
||||
|
|
|
@ -100,7 +100,7 @@ open(TopicSubscriptions, Opts) ->
|
|||
State0 = init_state(Opts),
|
||||
State1 = lists:foldl(
|
||||
fun({ShareTopicFilter, #{}}, State) ->
|
||||
?tp(debug, ds_agent_open_subscription, #{
|
||||
?tp(warning, ds_agent_open_subscription, #{
|
||||
topic_filter => ShareTopicFilter
|
||||
}),
|
||||
add_shared_subscription(State, ShareTopicFilter)
|
||||
|
@ -120,7 +120,7 @@ can_subscribe(_State, _ShareTopicFilter, _SubOpts) ->
|
|||
|
||||
-spec on_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> t().
|
||||
on_subscribe(State0, ShareTopicFilter, _SubOpts) ->
|
||||
?tp(debug, ds_agent_on_subscribe, #{
|
||||
?tp(warning, ds_agent_on_subscribe, #{
|
||||
share_topic_filter => ShareTopicFilter
|
||||
}),
|
||||
add_shared_subscription(State0, ShareTopicFilter).
|
||||
|
@ -163,7 +163,7 @@ on_disconnect(#{groups := Groups0} = State, StreamProgresses) ->
|
|||
|
||||
-spec on_info(t(), term()) -> t().
|
||||
on_info(State, ?leader_lease_streams_match(GroupId, Leader, StreamProgresses, Version)) ->
|
||||
?SLOG(debug, #{
|
||||
?SLOG(info, #{
|
||||
msg => leader_lease_streams,
|
||||
group_id => GroupId,
|
||||
streams => StreamProgresses,
|
||||
|
@ -176,7 +176,7 @@ on_info(State, ?leader_lease_streams_match(GroupId, Leader, StreamProgresses, Ve
|
|||
)
|
||||
end);
|
||||
on_info(State, ?leader_renew_stream_lease_match(GroupId, Version)) ->
|
||||
?SLOG(debug, #{
|
||||
?SLOG(info, #{
|
||||
msg => leader_renew_stream_lease,
|
||||
group_id => GroupId,
|
||||
version => Version
|
||||
|
@ -185,7 +185,7 @@ on_info(State, ?leader_renew_stream_lease_match(GroupId, Version)) ->
|
|||
emqx_ds_shared_sub_group_sm:handle_leader_renew_stream_lease(GSM, Version)
|
||||
end);
|
||||
on_info(State, ?leader_renew_stream_lease_match(GroupId, VersionOld, VersionNew)) ->
|
||||
?SLOG(debug, #{
|
||||
?SLOG(info, #{
|
||||
msg => leader_renew_stream_lease,
|
||||
group_id => GroupId,
|
||||
version_old => VersionOld,
|
||||
|
@ -195,7 +195,7 @@ on_info(State, ?leader_renew_stream_lease_match(GroupId, VersionOld, VersionNew)
|
|||
emqx_ds_shared_sub_group_sm:handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew)
|
||||
end);
|
||||
on_info(State, ?leader_update_streams_match(GroupId, VersionOld, VersionNew, StreamsNew)) ->
|
||||
?SLOG(debug, #{
|
||||
?SLOG(info, #{
|
||||
msg => leader_update_streams,
|
||||
group_id => GroupId,
|
||||
version_old => VersionOld,
|
||||
|
@ -208,7 +208,7 @@ on_info(State, ?leader_update_streams_match(GroupId, VersionOld, VersionNew, Str
|
|||
)
|
||||
end);
|
||||
on_info(State, ?leader_invalidate_match(GroupId)) ->
|
||||
?SLOG(debug, #{
|
||||
?SLOG(info, #{
|
||||
msg => leader_invalidate,
|
||||
group_id => GroupId
|
||||
}),
|
||||
|
@ -245,7 +245,7 @@ delete_shared_subscription(State, ShareTopicFilter, GroupProgress) ->
|
|||
add_shared_subscription(
|
||||
#{session_id := SessionId, groups := Groups0} = State0, ShareTopicFilter
|
||||
) ->
|
||||
?SLOG(debug, #{
|
||||
?SLOG(info, #{
|
||||
msg => agent_add_shared_subscription,
|
||||
share_topic_filter => ShareTopicFilter
|
||||
}),
|
||||
|
|
|
@ -120,7 +120,7 @@ new(#{
|
|||
send_after := SendAfter
|
||||
}) ->
|
||||
?SLOG(
|
||||
debug,
|
||||
info,
|
||||
#{
|
||||
msg => group_sm_new,
|
||||
agent => Agent,
|
||||
|
@ -133,7 +133,7 @@ new(#{
|
|||
agent => Agent,
|
||||
send_after => SendAfter
|
||||
},
|
||||
?tp(debug, group_sm_new, #{
|
||||
?tp(warning, group_sm_new, #{
|
||||
agent => Agent,
|
||||
share_topic_filter => ShareTopicFilter
|
||||
}),
|
||||
|
@ -176,7 +176,7 @@ handle_disconnect(
|
|||
%% Connecting state
|
||||
|
||||
handle_connecting(#{agent := Agent, share_topic_filter := ShareTopicFilter} = GSM) ->
|
||||
?tp(debug, group_sm_enter_connecting, #{
|
||||
?tp(warning, group_sm_enter_connecting, #{
|
||||
agent => Agent,
|
||||
share_topic_filter => ShareTopicFilter
|
||||
}),
|
||||
|
@ -264,13 +264,11 @@ handle_leader_update_streams(
|
|||
VersionNew,
|
||||
StreamProgresses
|
||||
) ->
|
||||
?tp(debug, shared_sub_group_sm_leader_update_streams, #{
|
||||
?tp(warning, shared_sub_group_sm_leader_update_streams, #{
|
||||
id => Id,
|
||||
version_old => VersionOld,
|
||||
version_new => VersionNew,
|
||||
stream_progresses => emqx_persistent_session_ds_shared_subs:format_stream_progresses(
|
||||
StreamProgresses
|
||||
)
|
||||
stream_progresses => emqx_ds_shared_sub_proto:format_stream_progresses(StreamProgresses)
|
||||
}),
|
||||
{AddEvents, Streams1} = lists:foldl(
|
||||
fun(#{stream := Stream, progress := Progress}, {AddEventAcc, StreamsAcc}) ->
|
||||
|
@ -305,11 +303,9 @@ handle_leader_update_streams(
|
|||
maps:keys(Streams1)
|
||||
),
|
||||
StreamLeaseEvents = AddEvents ++ RevokeEvents,
|
||||
?tp(debug, shared_sub_group_sm_leader_update_streams, #{
|
||||
?tp(warning, shared_sub_group_sm_leader_update_streams, #{
|
||||
id => Id,
|
||||
stream_lease_events => emqx_persistent_session_ds_shared_subs:format_lease_events(
|
||||
StreamLeaseEvents
|
||||
)
|
||||
stream_lease_events => emqx_ds_shared_sub_proto:format_lease_events(StreamLeaseEvents)
|
||||
}),
|
||||
transition(
|
||||
GSM,
|
||||
|
@ -435,11 +431,24 @@ handle_leader_invalidate(#{agent := Agent, share_topic_filter := ShareTopicFilte
|
|||
%% Internal API
|
||||
%%-----------------------------------------------------------------------
|
||||
|
||||
handle_state_timeout(#{state := ?connecting} = GSM, find_leader_timeout, _Message) ->
|
||||
handle_state_timeout(
|
||||
#{state := ?connecting, share_topic_filter := ShareTopicFilter} = GSM,
|
||||
find_leader_timeout,
|
||||
_Message
|
||||
) ->
|
||||
?tp(debug, find_leader_timeout, #{share_topic_filter => ShareTopicFilter}),
|
||||
handle_find_leader_timeout(GSM);
|
||||
handle_state_timeout(#{state := ?replaying} = GSM, renew_lease_timeout, _Message) ->
|
||||
handle_state_timeout(
|
||||
#{state := ?replaying} = GSM,
|
||||
renew_lease_timeout,
|
||||
_Message
|
||||
) ->
|
||||
handle_renew_lease_timeout(GSM);
|
||||
handle_state_timeout(GSM, update_stream_state_timeout, _Message) ->
|
||||
handle_state_timeout(
|
||||
GSM,
|
||||
update_stream_state_timeout,
|
||||
_Message
|
||||
) ->
|
||||
?tp(debug, update_stream_state_timeout, #{}),
|
||||
handle_stream_progress(GSM, []).
|
||||
|
||||
|
|
|
@ -164,7 +164,7 @@ handle_event({call, From}, #register{register_fun = Fun}, ?leader_waiting_regist
|
|||
%%--------------------------------------------------------------------
|
||||
%% repalying state
|
||||
handle_event(enter, _OldState, ?leader_active, #{topic := Topic} = _Data) ->
|
||||
?tp(debug, shared_sub_leader_enter_actve, #{topic => Topic}),
|
||||
?tp(warning, shared_sub_leader_enter_actve, #{topic => Topic}),
|
||||
{keep_state_and_data, [
|
||||
{{timeout, #renew_streams{}}, 0, #renew_streams{}},
|
||||
{{timeout, #renew_leases{}}, ?dq_config(leader_renew_lease_interval_ms), #renew_leases{}},
|
||||
|
@ -174,7 +174,7 @@ handle_event(enter, _OldState, ?leader_active, #{topic := Topic} = _Data) ->
|
|||
%% timers
|
||||
%% renew_streams timer
|
||||
handle_event({timeout, #renew_streams{}}, #renew_streams{}, ?leader_active, Data0) ->
|
||||
?tp(debug, shared_sub_leader_timeout, #{timeout => renew_streams}),
|
||||
% ?tp(warning, shared_sub_leader_timeout, #{timeout => renew_streams}),
|
||||
Data1 = renew_streams(Data0),
|
||||
{keep_state, Data1,
|
||||
{
|
||||
|
@ -184,7 +184,7 @@ handle_event({timeout, #renew_streams{}}, #renew_streams{}, ?leader_active, Data
|
|||
}};
|
||||
%% renew_leases timer
|
||||
handle_event({timeout, #renew_leases{}}, #renew_leases{}, ?leader_active, Data0) ->
|
||||
?tp(debug, shared_sub_leader_timeout, #{timeout => renew_leases}),
|
||||
% ?tp(warning, shared_sub_leader_timeout, #{timeout => renew_leases}),
|
||||
Data1 = renew_leases(Data0),
|
||||
{keep_state, Data1,
|
||||
{{timeout, #renew_leases{}}, ?dq_config(leader_renew_lease_interval_ms), #renew_leases{}}};
|
||||
|
@ -279,7 +279,7 @@ renew_streams(
|
|||
Data2 = Data1#{stream_states => NewStreamStates, rank_progress => RankProgress1},
|
||||
Data3 = revoke_streams(Data2),
|
||||
Data4 = assign_streams(Data3),
|
||||
?SLOG(debug, #{
|
||||
?SLOG(info, #{
|
||||
msg => leader_renew_streams,
|
||||
topic_filter => TopicFilter,
|
||||
new_streams => length(NewStreamsWRanks)
|
||||
|
@ -368,7 +368,7 @@ revoke_excess_streams_from_agent(Data0, Agent, DesiredCount) ->
|
|||
false ->
|
||||
AgentState0;
|
||||
true ->
|
||||
?tp(debug, shared_sub_leader_revoke_streams, #{
|
||||
?tp(warning, shared_sub_leader_revoke_streams, #{
|
||||
agent => Agent,
|
||||
agent_stream_count => length(Streams0),
|
||||
revoke_count => RevokeCount,
|
||||
|
@ -421,7 +421,7 @@ assign_lacking_streams(Data0, Agent, DesiredCount) ->
|
|||
false ->
|
||||
Data0;
|
||||
true ->
|
||||
?tp(debug, shared_sub_leader_assign_streams, #{
|
||||
?tp(warning, shared_sub_leader_assign_streams, #{
|
||||
agent => Agent,
|
||||
agent_stream_count => length(Streams0),
|
||||
assign_count => AssignCount,
|
||||
|
@ -449,7 +449,7 @@ select_streams_for_assign(Data0, _Agent, AssignCount) ->
|
|||
%% renew_leases - send lease confirmations to agents
|
||||
|
||||
renew_leases(#{agents := AgentStates} = Data) ->
|
||||
?tp(debug, shared_sub_leader_renew_leases, #{agents => maps:keys(AgentStates)}),
|
||||
?tp(warning, shared_sub_leader_renew_leases, #{agents => maps:keys(AgentStates)}),
|
||||
ok = lists:foreach(
|
||||
fun({Agent, AgentState}) ->
|
||||
renew_lease(Data, Agent, AgentState)
|
||||
|
@ -492,7 +492,7 @@ drop_timeout_agents(#{agents := Agents} = Data) ->
|
|||
(is_integer(NoReplayingDeadline) andalso NoReplayingDeadline < Now)
|
||||
of
|
||||
true ->
|
||||
?SLOG(debug, #{
|
||||
?SLOG(info, #{
|
||||
msg => leader_agent_timeout,
|
||||
now => Now,
|
||||
update_deadline => UpdateDeadline,
|
||||
|
@ -516,14 +516,14 @@ connect_agent(
|
|||
Agent,
|
||||
AgentMetadata
|
||||
) ->
|
||||
?SLOG(debug, #{
|
||||
?SLOG(info, #{
|
||||
msg => leader_agent_connected,
|
||||
agent => Agent,
|
||||
group_id => GroupId
|
||||
}),
|
||||
case Agents of
|
||||
#{Agent := AgentState} ->
|
||||
?tp(debug, shared_sub_leader_agent_already_connected, #{
|
||||
?tp(warning, shared_sub_leader_agent_already_connected, #{
|
||||
agent => Agent
|
||||
}),
|
||||
reconnect_agent(Data, Agent, AgentMetadata, AgentState);
|
||||
|
@ -546,7 +546,7 @@ reconnect_agent(
|
|||
AgentMetadata,
|
||||
#{streams := OldStreams, revoked_streams := OldRevokedStreams} = _OldAgentState
|
||||
) ->
|
||||
?tp(debug, shared_sub_leader_agent_reconnect, #{
|
||||
?tp(warning, shared_sub_leader_agent_reconnect, #{
|
||||
agent => Agent,
|
||||
agent_metadata => AgentMetadata,
|
||||
inherited_streams => OldStreams
|
||||
|
@ -767,7 +767,7 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, Vers
|
|||
disconnect_agent(Data0, Agent, AgentStreamProgresses, Version) ->
|
||||
case get_agent_state(Data0, Agent) of
|
||||
#{version := Version} ->
|
||||
?tp(debug, shared_sub_leader_disconnect_agent, #{
|
||||
?tp(warning, shared_sub_leader_disconnect_agent, #{
|
||||
agent => Agent,
|
||||
version => Version
|
||||
}),
|
||||
|
@ -794,7 +794,7 @@ agent_transition_to_waiting_updating(
|
|||
Streams,
|
||||
RevokedStreams
|
||||
) ->
|
||||
?tp(debug, shared_sub_leader_agent_state_transition, #{
|
||||
?tp(warning, shared_sub_leader_agent_state_transition, #{
|
||||
agent => Agent,
|
||||
old_state => OldState,
|
||||
new_state => ?waiting_updating
|
||||
|
@ -818,7 +818,7 @@ agent_transition_to_waiting_updating(
|
|||
agent_transition_to_waiting_replaying(
|
||||
#{group_id := GroupId} = _Data, Agent, #{state := OldState, version := Version} = AgentState0
|
||||
) ->
|
||||
?tp(debug, shared_sub_leader_agent_state_transition, #{
|
||||
?tp(warning, shared_sub_leader_agent_state_transition, #{
|
||||
agent => Agent,
|
||||
old_state => OldState,
|
||||
new_state => ?waiting_replaying
|
||||
|
@ -833,7 +833,7 @@ agent_transition_to_waiting_replaying(
|
|||
agent_transition_to_initial_waiting_replaying(
|
||||
#{group_id := GroupId} = Data, Agent, AgentMetadata, InitialStreams
|
||||
) ->
|
||||
?tp(debug, shared_sub_leader_agent_state_transition, #{
|
||||
?tp(warning, shared_sub_leader_agent_state_transition, #{
|
||||
agent => Agent,
|
||||
old_state => none,
|
||||
new_state => ?waiting_replaying
|
||||
|
@ -856,7 +856,7 @@ agent_transition_to_initial_waiting_replaying(
|
|||
renew_no_replaying_deadline(AgentState).
|
||||
|
||||
agent_transition_to_replaying(Agent, #{state := ?waiting_replaying} = AgentState) ->
|
||||
?tp(debug, shared_sub_leader_agent_state_transition, #{
|
||||
?tp(warning, shared_sub_leader_agent_state_transition, #{
|
||||
agent => Agent,
|
||||
old_state => ?waiting_replaying,
|
||||
new_state => ?replaying
|
||||
|
@ -868,7 +868,7 @@ agent_transition_to_replaying(Agent, #{state := ?waiting_replaying} = AgentState
|
|||
}.
|
||||
|
||||
agent_transition_to_updating(Agent, #{state := ?waiting_updating} = AgentState0) ->
|
||||
?tp(debug, shared_sub_leader_agent_state_transition, #{
|
||||
?tp(warning, shared_sub_leader_agent_state_transition, #{
|
||||
agent => Agent,
|
||||
old_state => ?waiting_updating,
|
||||
new_state => ?updating
|
||||
|
@ -995,7 +995,7 @@ drop_agent(#{agents := Agents} = Data0, Agent) ->
|
|||
#{streams := Streams, revoked_streams := RevokedStreams} = AgentState,
|
||||
AllStreams = Streams ++ RevokedStreams,
|
||||
Data1 = unassign_streams(Data0, AllStreams),
|
||||
?tp(debug, shared_sub_leader_drop_agent, #{agent => Agent}),
|
||||
?tp(warning, shared_sub_leader_drop_agent, #{agent => Agent}),
|
||||
Data1#{agents => maps:remove(Agent, Agents)}.
|
||||
|
||||
invalidate_agent(#{group_id := GroupId}, Agent) ->
|
||||
|
|
|
@ -55,7 +55,7 @@ set_replayed({{RankX, RankY}, Stream}, State) ->
|
|||
State#{RankX => #{min_y => MinY, ys => Ys2}};
|
||||
_ ->
|
||||
?SLOG(
|
||||
debug,
|
||||
warning,
|
||||
#{
|
||||
msg => leader_rank_progress_double_or_invalid_update,
|
||||
rank_x => RankX,
|
||||
|
|
|
@ -22,6 +22,12 @@
|
|||
]).
|
||||
|
||||
-export([
|
||||
format_stream_progresses/1,
|
||||
format_stream_progress/1,
|
||||
format_stream_key/1,
|
||||
format_stream_keys/1,
|
||||
format_lease_event/1,
|
||||
format_lease_events/1,
|
||||
agent/2
|
||||
]).
|
||||
|
||||
|
@ -51,20 +57,6 @@
|
|||
agent_metadata/0
|
||||
]).
|
||||
|
||||
-define(log_agent_msg(ToLeader, Msg),
|
||||
?tp(debug, shared_sub_proto_msg, #{
|
||||
to_leader => ToLeader,
|
||||
msg => emqx_ds_shared_sub_proto_format:format_agent_msg(Msg)
|
||||
})
|
||||
).
|
||||
|
||||
-define(log_leader_msg(ToAgent, Msg),
|
||||
?tp(debug, shared_sub_proto_msg, #{
|
||||
to_agent => ToAgent,
|
||||
msg => emqx_ds_shared_sub_proto_format:format_leader_msg(Msg)
|
||||
})
|
||||
).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% API
|
||||
%%--------------------------------------------------------------------
|
||||
|
@ -75,7 +67,15 @@
|
|||
agent_connect_leader(ToLeader, FromAgent, AgentMetadata, ShareTopicFilter) when
|
||||
?is_local_leader(ToLeader)
|
||||
->
|
||||
send_agent_msg(ToLeader, ?agent_connect_leader(FromAgent, AgentMetadata, ShareTopicFilter));
|
||||
?tp(warning, shared_sub_proto_msg, #{
|
||||
type => agent_connect_leader,
|
||||
to_leader => ToLeader,
|
||||
from_agent => FromAgent,
|
||||
agent_metadata => AgentMetadata,
|
||||
share_topic_filter => ShareTopicFilter
|
||||
}),
|
||||
_ = erlang:send(ToLeader, ?agent_connect_leader(FromAgent, AgentMetadata, ShareTopicFilter)),
|
||||
ok;
|
||||
agent_connect_leader(ToLeader, FromAgent, AgentMetadata, ShareTopicFilter) ->
|
||||
emqx_ds_shared_sub_proto_v1:agent_connect_leader(
|
||||
?leader_node(ToLeader), ToLeader, FromAgent, AgentMetadata, ShareTopicFilter
|
||||
|
@ -85,7 +85,15 @@ agent_connect_leader(ToLeader, FromAgent, AgentMetadata, ShareTopicFilter) ->
|
|||
agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) when
|
||||
?is_local_leader(ToLeader)
|
||||
->
|
||||
send_agent_msg(ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, Version));
|
||||
?tp(warning, shared_sub_proto_msg, #{
|
||||
type => agent_update_stream_states,
|
||||
to_leader => ToLeader,
|
||||
from_agent => FromAgent,
|
||||
stream_progresses => format_stream_progresses(StreamProgresses),
|
||||
version => Version
|
||||
}),
|
||||
_ = erlang:send(ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, Version)),
|
||||
ok;
|
||||
agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) ->
|
||||
emqx_ds_shared_sub_proto_v1:agent_update_stream_states(
|
||||
?leader_node(ToLeader), ToLeader, FromAgent, StreamProgresses, Version
|
||||
|
@ -97,9 +105,18 @@ agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) ->
|
|||
agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) when
|
||||
?is_local_leader(ToLeader)
|
||||
->
|
||||
send_agent_msg(
|
||||
?tp(warning, shared_sub_proto_msg, #{
|
||||
type => agent_update_stream_states,
|
||||
to_leader => ToLeader,
|
||||
from_agent => FromAgent,
|
||||
stream_progresses => format_stream_progresses(StreamProgresses),
|
||||
version_old => VersionOld,
|
||||
version_new => VersionNew
|
||||
}),
|
||||
_ = erlang:send(
|
||||
ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, VersionOld, VersionNew)
|
||||
);
|
||||
),
|
||||
ok;
|
||||
agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) ->
|
||||
emqx_ds_shared_sub_proto_v1:agent_update_stream_states(
|
||||
?leader_node(ToLeader), ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew
|
||||
|
@ -108,7 +125,15 @@ agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, Ve
|
|||
agent_disconnect(ToLeader, FromAgent, StreamProgresses, Version) when
|
||||
?is_local_leader(ToLeader)
|
||||
->
|
||||
send_agent_msg(ToLeader, ?agent_disconnect(FromAgent, StreamProgresses, Version));
|
||||
?tp(warning, shared_sub_proto_msg, #{
|
||||
type => agent_disconnect,
|
||||
to_leader => ToLeader,
|
||||
from_agent => FromAgent,
|
||||
stream_progresses => format_stream_progresses(StreamProgresses),
|
||||
version => Version
|
||||
}),
|
||||
_ = erlang:send(ToLeader, ?agent_disconnect(FromAgent, StreamProgresses, Version)),
|
||||
ok;
|
||||
agent_disconnect(ToLeader, FromAgent, StreamProgresses, Version) ->
|
||||
emqx_ds_shared_sub_proto_v1:agent_disconnect(
|
||||
?leader_node(ToLeader), ToLeader, FromAgent, StreamProgresses, Version
|
||||
|
@ -119,7 +144,19 @@ agent_disconnect(ToLeader, FromAgent, StreamProgresses, Version) ->
|
|||
-spec leader_lease_streams(agent(), group(), leader(), list(leader_stream_progress()), version()) ->
|
||||
ok.
|
||||
leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) when ?is_local_agent(ToAgent) ->
|
||||
send_leader_msg(ToAgent, ?leader_lease_streams(OfGroup, Leader, Streams, Version));
|
||||
?tp(warning, shared_sub_proto_msg, #{
|
||||
type => leader_lease_streams,
|
||||
to_agent => ToAgent,
|
||||
of_group => OfGroup,
|
||||
leader => Leader,
|
||||
streams => format_stream_progresses(Streams),
|
||||
version => Version
|
||||
}),
|
||||
_ = emqx_persistent_session_ds_shared_subs_agent:send(
|
||||
?agent_pid(ToAgent),
|
||||
?leader_lease_streams(OfGroup, Leader, Streams, Version)
|
||||
),
|
||||
ok;
|
||||
leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) ->
|
||||
emqx_ds_shared_sub_proto_v1:leader_lease_streams(
|
||||
?agent_node(ToAgent), ToAgent, OfGroup, Leader, Streams, Version
|
||||
|
@ -127,7 +164,17 @@ leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) ->
|
|||
|
||||
-spec leader_renew_stream_lease(agent(), group(), version()) -> ok.
|
||||
leader_renew_stream_lease(ToAgent, OfGroup, Version) when ?is_local_agent(ToAgent) ->
|
||||
send_leader_msg(ToAgent, ?leader_renew_stream_lease(OfGroup, Version));
|
||||
?tp(warning, shared_sub_proto_msg, #{
|
||||
type => leader_renew_stream_lease,
|
||||
to_agent => ToAgent,
|
||||
of_group => OfGroup,
|
||||
version => Version
|
||||
}),
|
||||
_ = emqx_persistent_session_ds_shared_subs_agent:send(
|
||||
?agent_pid(ToAgent),
|
||||
?leader_renew_stream_lease(OfGroup, Version)
|
||||
),
|
||||
ok;
|
||||
leader_renew_stream_lease(ToAgent, OfGroup, Version) ->
|
||||
emqx_ds_shared_sub_proto_v1:leader_renew_stream_lease(
|
||||
?agent_node(ToAgent), ToAgent, OfGroup, Version
|
||||
|
@ -135,7 +182,18 @@ leader_renew_stream_lease(ToAgent, OfGroup, Version) ->
|
|||
|
||||
-spec leader_renew_stream_lease(agent(), group(), version(), version()) -> ok.
|
||||
leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) when ?is_local_agent(ToAgent) ->
|
||||
send_leader_msg(ToAgent, ?leader_renew_stream_lease(OfGroup, VersionOld, VersionNew));
|
||||
?tp(warning, shared_sub_proto_msg, #{
|
||||
type => leader_renew_stream_lease,
|
||||
to_agent => ToAgent,
|
||||
of_group => OfGroup,
|
||||
version_old => VersionOld,
|
||||
version_new => VersionNew
|
||||
}),
|
||||
_ = emqx_persistent_session_ds_shared_subs_agent:send(
|
||||
?agent_pid(ToAgent),
|
||||
?leader_renew_stream_lease(OfGroup, VersionOld, VersionNew)
|
||||
),
|
||||
ok;
|
||||
leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) ->
|
||||
emqx_ds_shared_sub_proto_v1:leader_renew_stream_lease(
|
||||
?agent_node(ToAgent), ToAgent, OfGroup, VersionOld, VersionNew
|
||||
|
@ -146,7 +204,19 @@ leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) ->
|
|||
leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) when
|
||||
?is_local_agent(ToAgent)
|
||||
->
|
||||
send_leader_msg(ToAgent, ?leader_update_streams(OfGroup, VersionOld, VersionNew, StreamsNew));
|
||||
?tp(warning, shared_sub_proto_msg, #{
|
||||
type => leader_update_streams,
|
||||
to_agent => ToAgent,
|
||||
of_group => OfGroup,
|
||||
version_old => VersionOld,
|
||||
version_new => VersionNew,
|
||||
streams_new => format_stream_progresses(StreamsNew)
|
||||
}),
|
||||
_ = emqx_persistent_session_ds_shared_subs_agent:send(
|
||||
?agent_pid(ToAgent),
|
||||
?leader_update_streams(OfGroup, VersionOld, VersionNew, StreamsNew)
|
||||
),
|
||||
ok;
|
||||
leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) ->
|
||||
emqx_ds_shared_sub_proto_v1:leader_update_streams(
|
||||
?agent_node(ToAgent), ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew
|
||||
|
@ -154,7 +224,16 @@ leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) ->
|
|||
|
||||
-spec leader_invalidate(agent(), group()) -> ok.
|
||||
leader_invalidate(ToAgent, OfGroup) when ?is_local_agent(ToAgent) ->
|
||||
send_leader_msg(ToAgent, ?leader_invalidate(OfGroup));
|
||||
?tp(warning, shared_sub_proto_msg, #{
|
||||
type => leader_invalidate,
|
||||
to_agent => ToAgent,
|
||||
of_group => OfGroup
|
||||
}),
|
||||
_ = emqx_persistent_session_ds_shared_subs_agent:send(
|
||||
?agent_pid(ToAgent),
|
||||
?leader_invalidate(OfGroup)
|
||||
),
|
||||
ok;
|
||||
leader_invalidate(ToAgent, OfGroup) ->
|
||||
emqx_ds_shared_sub_proto_v1:leader_invalidate(
|
||||
?agent_node(ToAgent), ToAgent, OfGroup
|
||||
|
@ -168,12 +247,41 @@ agent(Id, Pid) ->
|
|||
_ = Id,
|
||||
?agent(Id, Pid).
|
||||
|
||||
send_agent_msg(ToLeader, Msg) ->
|
||||
?log_agent_msg(ToLeader, Msg),
|
||||
_ = erlang:send(ToLeader, Msg),
|
||||
ok.
|
||||
format_stream_progresses(Streams) ->
|
||||
lists:map(
|
||||
fun format_stream_progress/1,
|
||||
Streams
|
||||
).
|
||||
|
||||
send_leader_msg(ToAgent, Msg) ->
|
||||
?log_leader_msg(ToAgent, Msg),
|
||||
_ = emqx_persistent_session_ds_shared_subs_agent:send(?agent_pid(ToAgent), Msg),
|
||||
ok.
|
||||
format_stream_progress(#{stream := Stream, progress := Progress} = Value) ->
|
||||
Value#{stream => format_opaque(Stream), progress => format_progress(Progress)}.
|
||||
|
||||
format_progress(#{iterator := Iterator} = Progress) ->
|
||||
Progress#{iterator => format_opaque(Iterator)}.
|
||||
|
||||
format_stream_key({SubId, Stream}) ->
|
||||
{SubId, format_opaque(Stream)}.
|
||||
|
||||
format_stream_keys(StreamKeys) ->
|
||||
lists:map(
|
||||
fun format_stream_key/1,
|
||||
StreamKeys
|
||||
).
|
||||
|
||||
format_lease_events(Events) ->
|
||||
lists:map(
|
||||
fun format_lease_event/1,
|
||||
Events
|
||||
).
|
||||
|
||||
format_lease_event(#{stream := Stream, progress := Progress} = Event) ->
|
||||
Event#{stream => format_opaque(Stream), progress => format_progress(Progress)};
|
||||
format_lease_event(#{stream := Stream} = Event) ->
|
||||
Event#{stream => format_opaque(Stream)}.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% Helpers
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
format_opaque(Opaque) ->
|
||||
erlang:phash2(Opaque).
|
||||
|
|
|
@ -12,167 +12,146 @@
|
|||
|
||||
%% agent messages, sent from agent side to the leader
|
||||
|
||||
-define(agent_connect_leader_msg, 1).
|
||||
-define(agent_update_stream_states_msg, 2).
|
||||
-define(agent_connect_leader_timeout_msg, 3).
|
||||
-define(agent_renew_stream_lease_timeout_msg, 4).
|
||||
-define(agent_disconnect_msg, 5).
|
||||
|
||||
%% message keys (used used not to send atoms over the network)
|
||||
-define(agent_msg_type, 1).
|
||||
-define(agent_msg_agent, 2).
|
||||
-define(agent_msg_share_topic_filter, 3).
|
||||
-define(agent_msg_agent_metadata, 4).
|
||||
-define(agent_msg_stream_states, 5).
|
||||
-define(agent_msg_version, 6).
|
||||
-define(agent_msg_version_old, 7).
|
||||
-define(agent_msg_version_new, 8).
|
||||
-define(agent_connect_leader_msg, agent_connect_leader).
|
||||
-define(agent_update_stream_states_msg, agent_update_stream_states).
|
||||
-define(agent_connect_leader_timeout_msg, agent_connect_leader_timeout).
|
||||
-define(agent_renew_stream_lease_timeout_msg, agent_renew_stream_lease_timeout).
|
||||
-define(agent_disconnect_msg, agent_disconnect).
|
||||
|
||||
%% Agent messages sent to the leader.
|
||||
%% Leader talks to many agents, `agent` field is used to identify the sender.
|
||||
|
||||
-define(agent_connect_leader(Agent, AgentMetadata, ShareTopicFilter), #{
|
||||
?agent_msg_type => ?agent_connect_leader_msg,
|
||||
?agent_msg_share_topic_filter => ShareTopicFilter,
|
||||
?agent_msg_agent_metadata => AgentMetadata,
|
||||
?agent_msg_agent => Agent
|
||||
type => ?agent_connect_leader_msg,
|
||||
share_topic_filter => ShareTopicFilter,
|
||||
agent_metadata => AgentMetadata,
|
||||
agent => Agent
|
||||
}).
|
||||
|
||||
-define(agent_connect_leader_match(Agent, AgentMetadata, ShareTopicFilter), #{
|
||||
?agent_msg_type := ?agent_connect_leader_msg,
|
||||
?agent_msg_share_topic_filter := ShareTopicFilter,
|
||||
?agent_msg_agent_metadata := AgentMetadata,
|
||||
?agent_msg_agent := Agent
|
||||
type := ?agent_connect_leader_msg,
|
||||
share_topic_filter := ShareTopicFilter,
|
||||
agent_metadata := AgentMetadata,
|
||||
agent := Agent
|
||||
}).
|
||||
|
||||
-define(agent_update_stream_states(Agent, StreamStates, Version), #{
|
||||
?agent_msg_type => ?agent_update_stream_states_msg,
|
||||
?agent_msg_stream_states => StreamStates,
|
||||
?agent_msg_version => Version,
|
||||
?agent_msg_agent => Agent
|
||||
type => ?agent_update_stream_states_msg,
|
||||
stream_states => StreamStates,
|
||||
version => Version,
|
||||
agent => Agent
|
||||
}).
|
||||
|
||||
-define(agent_update_stream_states_match(Agent, StreamStates, Version), #{
|
||||
?agent_msg_type := ?agent_update_stream_states_msg,
|
||||
?agent_msg_stream_states := StreamStates,
|
||||
?agent_msg_version := Version,
|
||||
?agent_msg_agent := Agent
|
||||
type := ?agent_update_stream_states_msg,
|
||||
stream_states := StreamStates,
|
||||
version := Version,
|
||||
agent := Agent
|
||||
}).
|
||||
|
||||
-define(agent_update_stream_states(Agent, StreamStates, VersionOld, VersionNew), #{
|
||||
?agent_msg_type => ?agent_update_stream_states_msg,
|
||||
?agent_msg_stream_states => StreamStates,
|
||||
?agent_msg_version_old => VersionOld,
|
||||
?agent_msg_version_new => VersionNew,
|
||||
?agent_msg_agent => Agent
|
||||
type => ?agent_update_stream_states_msg,
|
||||
stream_states => StreamStates,
|
||||
version_old => VersionOld,
|
||||
version_new => VersionNew,
|
||||
agent => Agent
|
||||
}).
|
||||
|
||||
-define(agent_update_stream_states_match(Agent, StreamStates, VersionOld, VersionNew), #{
|
||||
?agent_msg_type := ?agent_update_stream_states_msg,
|
||||
?agent_msg_stream_states := StreamStates,
|
||||
?agent_msg_version_old := VersionOld,
|
||||
?agent_msg_version_new := VersionNew,
|
||||
?agent_msg_agent := Agent
|
||||
type := ?agent_update_stream_states_msg,
|
||||
stream_states := StreamStates,
|
||||
version_old := VersionOld,
|
||||
version_new := VersionNew,
|
||||
agent := Agent
|
||||
}).
|
||||
|
||||
-define(agent_disconnect(Agent, StreamStates, Version), #{
|
||||
?agent_msg_type => ?agent_disconnect_msg,
|
||||
?agent_msg_stream_states => StreamStates,
|
||||
?agent_msg_version => Version,
|
||||
?agent_msg_agent => Agent
|
||||
type => ?agent_disconnect_msg,
|
||||
stream_states => StreamStates,
|
||||
version => Version,
|
||||
agent => Agent
|
||||
}).
|
||||
|
||||
-define(agent_disconnect_match(Agent, StreamStates, Version), #{
|
||||
?agent_msg_type := ?agent_disconnect_msg,
|
||||
?agent_msg_stream_states := StreamStates,
|
||||
?agent_msg_version := Version,
|
||||
?agent_msg_agent := Agent
|
||||
type := ?agent_disconnect_msg,
|
||||
stream_states := StreamStates,
|
||||
version := Version,
|
||||
agent := Agent
|
||||
}).
|
||||
|
||||
%% leader messages, sent from the leader to the agent
|
||||
%% Agent may have several shared subscriptions, so may talk to several leaders
|
||||
%% `group_id` field is used to identify the leader.
|
||||
|
||||
-define(leader_lease_streams_msg, 101).
|
||||
-define(leader_renew_stream_lease_msg, 102).
|
||||
-define(leader_update_streams, 103).
|
||||
-define(leader_invalidate, 104).
|
||||
|
||||
-define(leader_msg_type, 101).
|
||||
-define(leader_msg_streams, 102).
|
||||
-define(leader_msg_version, 103).
|
||||
-define(leader_msg_version_old, 104).
|
||||
-define(leader_msg_version_new, 105).
|
||||
-define(leader_msg_streams_new, 106).
|
||||
-define(leader_msg_leader, 107).
|
||||
-define(leader_msg_group_id, 108).
|
||||
-define(leader_lease_streams_msg, leader_lease_streams).
|
||||
-define(leader_renew_stream_lease_msg, leader_renew_stream_lease).
|
||||
|
||||
-define(leader_lease_streams(GrouId, Leader, Streams, Version), #{
|
||||
?leader_msg_type => ?leader_lease_streams_msg,
|
||||
?leader_msg_streams => Streams,
|
||||
?leader_msg_version => Version,
|
||||
?leader_msg_leader => Leader,
|
||||
?leader_msg_group_id => GrouId
|
||||
type => ?leader_lease_streams_msg,
|
||||
streams => Streams,
|
||||
version => Version,
|
||||
leader => Leader,
|
||||
group_id => GrouId
|
||||
}).
|
||||
|
||||
-define(leader_lease_streams_match(GroupId, Leader, Streams, Version), #{
|
||||
?leader_msg_type := ?leader_lease_streams_msg,
|
||||
?leader_msg_streams := Streams,
|
||||
?leader_msg_version := Version,
|
||||
?leader_msg_leader := Leader,
|
||||
?leader_msg_group_id := GroupId
|
||||
type := ?leader_lease_streams_msg,
|
||||
streams := Streams,
|
||||
version := Version,
|
||||
leader := Leader,
|
||||
group_id := GroupId
|
||||
}).
|
||||
|
||||
-define(leader_renew_stream_lease(GroupId, Version), #{
|
||||
?leader_msg_type => ?leader_renew_stream_lease_msg,
|
||||
?leader_msg_version => Version,
|
||||
?leader_msg_group_id => GroupId
|
||||
type => ?leader_renew_stream_lease_msg,
|
||||
version => Version,
|
||||
group_id => GroupId
|
||||
}).
|
||||
|
||||
-define(leader_renew_stream_lease_match(GroupId, Version), #{
|
||||
?leader_msg_type := ?leader_renew_stream_lease_msg,
|
||||
?leader_msg_version := Version,
|
||||
?leader_msg_group_id := GroupId
|
||||
type := ?leader_renew_stream_lease_msg,
|
||||
version := Version,
|
||||
group_id := GroupId
|
||||
}).
|
||||
|
||||
-define(leader_renew_stream_lease(GroupId, VersionOld, VersionNew), #{
|
||||
?leader_msg_type => ?leader_renew_stream_lease_msg,
|
||||
?leader_msg_version_old => VersionOld,
|
||||
?leader_msg_version_new => VersionNew,
|
||||
?leader_msg_group_id => GroupId
|
||||
type => ?leader_renew_stream_lease_msg,
|
||||
version_old => VersionOld,
|
||||
version_new => VersionNew,
|
||||
group_id => GroupId
|
||||
}).
|
||||
|
||||
-define(leader_renew_stream_lease_match(GroupId, VersionOld, VersionNew), #{
|
||||
?leader_msg_type := ?leader_renew_stream_lease_msg,
|
||||
?leader_msg_version_old := VersionOld,
|
||||
?leader_msg_version_new := VersionNew,
|
||||
?leader_msg_group_id := GroupId
|
||||
type := ?leader_renew_stream_lease_msg,
|
||||
version_old := VersionOld,
|
||||
version_new := VersionNew,
|
||||
group_id := GroupId
|
||||
}).
|
||||
|
||||
-define(leader_update_streams(GroupId, VersionOld, VersionNew, StreamsNew), #{
|
||||
?leader_msg_type => ?leader_update_streams,
|
||||
?leader_msg_version_old => VersionOld,
|
||||
?leader_msg_version_new => VersionNew,
|
||||
?leader_msg_streams_new => StreamsNew,
|
||||
?leader_msg_group_id => GroupId
|
||||
type => leader_update_streams,
|
||||
version_old => VersionOld,
|
||||
version_new => VersionNew,
|
||||
streams_new => StreamsNew,
|
||||
group_id => GroupId
|
||||
}).
|
||||
|
||||
-define(leader_update_streams_match(GroupId, VersionOld, VersionNew, StreamsNew), #{
|
||||
?leader_msg_type := ?leader_update_streams,
|
||||
?leader_msg_version_old := VersionOld,
|
||||
?leader_msg_version_new := VersionNew,
|
||||
?leader_msg_streams_new := StreamsNew,
|
||||
?leader_msg_group_id := GroupId
|
||||
type := leader_update_streams,
|
||||
version_old := VersionOld,
|
||||
version_new := VersionNew,
|
||||
streams_new := StreamsNew,
|
||||
group_id := GroupId
|
||||
}).
|
||||
|
||||
-define(leader_invalidate(GroupId), #{
|
||||
?leader_msg_type => ?leader_invalidate,
|
||||
?leader_msg_group_id => GroupId
|
||||
type => leader_invalidate,
|
||||
group_id => GroupId
|
||||
}).
|
||||
|
||||
-define(leader_invalidate_match(GroupId), #{
|
||||
?leader_msg_type := ?leader_invalidate,
|
||||
?leader_msg_group_id := GroupId
|
||||
type := leader_invalidate,
|
||||
group_id := GroupId
|
||||
}).
|
||||
|
||||
%% Helpers
|
||||
|
|
|
@ -1,82 +0,0 @@
|
|||
%%--------------------------------------------------------------------
|
||||
%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-module(emqx_ds_shared_sub_proto_format).
|
||||
|
||||
-include("emqx_ds_shared_sub_proto.hrl").
|
||||
|
||||
-export([format_agent_msg/1, format_leader_msg/1]).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% API
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
format_agent_msg(Msg) ->
|
||||
maps:from_list(
|
||||
lists:map(
|
||||
fun({K, V}) ->
|
||||
FormattedKey = agent_msg_key(K),
|
||||
{FormattedKey, format_agent_msg_value(FormattedKey, V)}
|
||||
end,
|
||||
maps:to_list(Msg)
|
||||
)
|
||||
).
|
||||
|
||||
format_leader_msg(Msg) ->
|
||||
maps:from_list(
|
||||
lists:map(
|
||||
fun({K, V}) ->
|
||||
FormattedKey = leader_msg_key(K),
|
||||
{FormattedKey, format_leader_msg_value(FormattedKey, V)}
|
||||
end,
|
||||
maps:to_list(Msg)
|
||||
)
|
||||
).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% Internal functions
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
format_agent_msg_value(agent_msg_type, Type) ->
|
||||
agent_msg_type(Type);
|
||||
format_agent_msg_value(agent_msg_stream_states, StreamStates) ->
|
||||
emqx_persistent_session_ds_shared_subs:format_stream_progresses(StreamStates);
|
||||
format_agent_msg_value(_, Value) ->
|
||||
Value.
|
||||
|
||||
format_leader_msg_value(leader_msg_type, Type) ->
|
||||
leader_msg_type(Type);
|
||||
format_leader_msg_value(leader_msg_streams, Streams) ->
|
||||
emqx_persistent_session_ds_shared_subs:format_lease_events(Streams);
|
||||
format_leader_msg_value(_, Value) ->
|
||||
Value.
|
||||
|
||||
agent_msg_type(?agent_connect_leader_msg) -> agent_connect_leader_msg;
|
||||
agent_msg_type(?agent_update_stream_states_msg) -> agent_update_stream_states_msg;
|
||||
agent_msg_type(?agent_connect_leader_timeout_msg) -> agent_connect_leader_timeout_msg;
|
||||
agent_msg_type(?agent_renew_stream_lease_timeout_msg) -> agent_renew_stream_lease_timeout_msg;
|
||||
agent_msg_type(?agent_disconnect_msg) -> agent_disconnect_msg.
|
||||
|
||||
agent_msg_key(?agent_msg_type) -> agent_msg_type;
|
||||
agent_msg_key(?agent_msg_agent) -> agent_msg_agent;
|
||||
agent_msg_key(?agent_msg_share_topic_filter) -> agent_msg_share_topic_filter;
|
||||
agent_msg_key(?agent_msg_agent_metadata) -> agent_msg_agent_metadata;
|
||||
agent_msg_key(?agent_msg_stream_states) -> agent_msg_stream_states;
|
||||
agent_msg_key(?agent_msg_version) -> agent_msg_version;
|
||||
agent_msg_key(?agent_msg_version_old) -> agent_msg_version_old;
|
||||
agent_msg_key(?agent_msg_version_new) -> agent_msg_version_new.
|
||||
|
||||
leader_msg_type(?leader_lease_streams_msg) -> leader_lease_streams_msg;
|
||||
leader_msg_type(?leader_renew_stream_lease_msg) -> leader_renew_stream_lease_msg;
|
||||
leader_msg_type(?leader_update_streams) -> leader_update_streams;
|
||||
leader_msg_type(?leader_invalidate) -> leader_invalidate.
|
||||
|
||||
leader_msg_key(?leader_msg_type) -> leader_msg_type;
|
||||
leader_msg_key(?leader_msg_streams) -> leader_msg_streams;
|
||||
leader_msg_key(?leader_msg_version) -> leader_msg_version;
|
||||
leader_msg_key(?leader_msg_version_old) -> leader_msg_version_old;
|
||||
leader_msg_key(?leader_msg_version_new) -> leader_msg_version_new;
|
||||
leader_msg_key(?leader_msg_streams_new) -> leader_msg_streams_new;
|
||||
leader_msg_key(?leader_msg_leader) -> leader_msg_leader;
|
||||
leader_msg_key(?leader_msg_group_id) -> leader_msg_group_id.
|
|
@ -113,7 +113,7 @@ do_lookup_leader(Agent, AgentMetadata, ShareTopicFilter, State) ->
|
|||
Pid ->
|
||||
Pid
|
||||
end,
|
||||
?SLOG(debug, #{
|
||||
?SLOG(info, #{
|
||||
msg => lookup_leader,
|
||||
agent => Agent,
|
||||
share_topic_filter => ShareTopicFilter,
|
||||
|
|
|
@ -417,7 +417,7 @@ t_lease_reconnect(_Config) ->
|
|||
|
||||
?assertWaitEvent(
|
||||
{ok, _, _} = emqtt:subscribe(ConnShared, <<"$share/gr2/topic2/#">>, 1),
|
||||
#{?snk_kind := group_sm_find_leader_timeout},
|
||||
#{?snk_kind := find_leader_timeout},
|
||||
5_000
|
||||
),
|
||||
|
||||
|
|
|
@ -55,6 +55,7 @@
|
|||
topic_filter/0,
|
||||
topic/0,
|
||||
batch/0,
|
||||
dsbatch/0,
|
||||
operation/0,
|
||||
deletion/0,
|
||||
precondition/0,
|
||||
|
@ -104,7 +105,9 @@
|
|||
-type message_matcher(Payload) :: #message_matcher{payload :: Payload}.
|
||||
|
||||
%% A batch of storage operations.
|
||||
-type batch() :: [operation()] | #dsbatch{}.
|
||||
-type batch() :: [operation()] | dsbatch().
|
||||
|
||||
-type dsbatch() :: #dsbatch{}.
|
||||
|
||||
-type operation() ::
|
||||
%% Store a message.
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_gateway_coap, [
|
||||
{description, "CoAP Gateway"},
|
||||
{vsn, "0.1.10"},
|
||||
{vsn, "0.1.9"},
|
||||
{registered, []},
|
||||
{applications, [kernel, stdlib, emqx, emqx_gateway]},
|
||||
{env, []},
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_gateway_exproto, [
|
||||
{description, "ExProto Gateway"},
|
||||
{vsn, "0.1.13"},
|
||||
{vsn, "0.1.12"},
|
||||
{registered, []},
|
||||
{applications, [kernel, stdlib, grpc, emqx, emqx_gateway]},
|
||||
{env, []},
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_gateway_gbt32960, [
|
||||
{description, "GBT32960 Gateway"},
|
||||
{vsn, "0.1.5"},
|
||||
{vsn, "0.1.4"},
|
||||
{registered, []},
|
||||
{applications, [kernel, stdlib, emqx, emqx_gateway]},
|
||||
{env, []},
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_gateway_jt808, [
|
||||
{description, "JT/T 808 Gateway"},
|
||||
{vsn, "0.1.1"},
|
||||
{vsn, "0.1.0"},
|
||||
{registered, []},
|
||||
{applications, [kernel, stdlib, emqx, emqx_gateway]},
|
||||
{env, []},
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_gateway_mqttsn, [
|
||||
{description, "MQTT-SN Gateway"},
|
||||
{vsn, "0.2.3"},
|
||||
{vsn, "0.2.2"},
|
||||
{registered, []},
|
||||
{applications, [kernel, stdlib, emqx, emqx_gateway]},
|
||||
{env, []},
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
{id, "emqx_machine"},
|
||||
{description, "The EMQX Machine"},
|
||||
% strict semver, bump manually!
|
||||
{vsn, "0.3.4"},
|
||||
{vsn, "0.3.3"},
|
||||
{modules, []},
|
||||
{registered, []},
|
||||
{applications, [kernel, stdlib, emqx_ctl, redbug]},
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
{application, emqx_management, [
|
||||
{description, "EMQX Management API and CLI"},
|
||||
% strict semver, bump manually!
|
||||
{vsn, "5.2.4"},
|
||||
{vsn, "5.2.3"},
|
||||
{modules, []},
|
||||
{registered, [emqx_management_sup]},
|
||||
{applications, [
|
||||
|
|
|
@ -29,9 +29,13 @@
|
|||
|
||||
start(_Type, _Args) ->
|
||||
ok = mria:wait_for_tables(emqx_mgmt_auth:create_tables()),
|
||||
emqx_mgmt_auth:try_init_bootstrap_file(),
|
||||
emqx_conf:add_handler([api_key], emqx_mgmt_auth),
|
||||
emqx_mgmt_sup:start_link().
|
||||
case emqx_mgmt_auth:init_bootstrap_file() of
|
||||
ok ->
|
||||
emqx_conf:add_handler([api_key], emqx_mgmt_auth),
|
||||
emqx_mgmt_sup:start_link();
|
||||
{error, Reason} ->
|
||||
{error, Reason}
|
||||
end.
|
||||
|
||||
stop(_State) ->
|
||||
emqx_conf:remove_handler([api_key]),
|
||||
|
|
|
@ -32,7 +32,7 @@
|
|||
update/5,
|
||||
delete/1,
|
||||
list/0,
|
||||
try_init_bootstrap_file/0,
|
||||
init_bootstrap_file/0,
|
||||
format/1
|
||||
]).
|
||||
|
||||
|
@ -52,7 +52,6 @@
|
|||
-ifdef(TEST).
|
||||
-export([create/7]).
|
||||
-export([trans/2, force_create_app/1]).
|
||||
-export([init_bootstrap_file/1]).
|
||||
-endif.
|
||||
|
||||
-define(APP, emqx_app).
|
||||
|
@ -115,12 +114,11 @@ post_config_update([api_key], _Req, NewConf, _OldConf, _AppEnvs) ->
|
|||
end,
|
||||
ok.
|
||||
|
||||
-spec try_init_bootstrap_file() -> ok | {error, _}.
|
||||
try_init_bootstrap_file() ->
|
||||
-spec init_bootstrap_file() -> ok | {error, _}.
|
||||
init_bootstrap_file() ->
|
||||
File = bootstrap_file(),
|
||||
?SLOG(debug, #{msg => "init_bootstrap_api_keys_from_file", file => File}),
|
||||
_ = init_bootstrap_file(File),
|
||||
ok.
|
||||
init_bootstrap_file(File).
|
||||
|
||||
create(Name, Enable, ExpiredAt, Desc, Role) ->
|
||||
ApiKey = generate_unique_api_key(Name),
|
||||
|
@ -359,6 +357,10 @@ init_bootstrap_file(File) ->
|
|||
init_bootstrap_file(File, Dev, MP);
|
||||
{error, Reason0} ->
|
||||
Reason = emqx_utils:explain_posix(Reason0),
|
||||
FmtReason = emqx_utils:format(
|
||||
"load API bootstrap file failed, file:~ts, reason:~ts",
|
||||
[File, Reason]
|
||||
),
|
||||
|
||||
?SLOG(
|
||||
error,
|
||||
|
@ -369,7 +371,7 @@ init_bootstrap_file(File) ->
|
|||
}
|
||||
),
|
||||
|
||||
{error, Reason}
|
||||
{error, FmtReason}
|
||||
end.
|
||||
|
||||
init_bootstrap_file(File, Dev, MP) ->
|
||||
|
|
|
@ -97,7 +97,7 @@ broker(_) ->
|
|||
%% @doc Cluster with other nodes
|
||||
|
||||
cluster(["join", SNode]) ->
|
||||
case mria:join(ekka_node:parse_name(SNode)) of
|
||||
case ekka:join(ekka_node:parse_name(SNode)) of
|
||||
ok ->
|
||||
emqx_ctl:print("Join the cluster successfully.~n"),
|
||||
%% FIXME: running status on the replicant immediately
|
||||
|
@ -112,7 +112,7 @@ cluster(["join", SNode]) ->
|
|||
end;
|
||||
cluster(["leave"]) ->
|
||||
_ = maybe_disable_autocluster(),
|
||||
case mria:leave() of
|
||||
case ekka:leave() of
|
||||
ok ->
|
||||
emqx_ctl:print("Leave the cluster successfully.~n"),
|
||||
cluster(["status"]);
|
||||
|
@ -121,7 +121,7 @@ cluster(["leave"]) ->
|
|||
end;
|
||||
cluster(["force-leave", SNode]) ->
|
||||
Node = ekka_node:parse_name(SNode),
|
||||
case mria:force_leave(Node) of
|
||||
case ekka:force_leave(Node) of
|
||||
ok ->
|
||||
case emqx_cluster_rpc:force_leave_clean(Node) of
|
||||
ok ->
|
||||
|
|
|
@ -100,7 +100,7 @@ t_bootstrap_file(_) ->
|
|||
BadBin = <<"test-1:secret-11\ntest-2 secret-12">>,
|
||||
ok = file:write_file(File, BadBin),
|
||||
update_file(File),
|
||||
?assertMatch({error, #{reason := "invalid_format"}}, emqx_mgmt_auth:init_bootstrap_file(File)),
|
||||
?assertMatch({error, #{reason := "invalid_format"}}, emqx_mgmt_auth:init_bootstrap_file()),
|
||||
?assertEqual(ok, auth_authorize(TestPath, <<"test-1">>, <<"secret-11">>)),
|
||||
?assertMatch({error, _}, auth_authorize(TestPath, <<"test-2">>, <<"secret-12">>)),
|
||||
update_file(<<>>),
|
||||
|
@ -123,7 +123,7 @@ t_bootstrap_file_override(_) ->
|
|||
ok = file:write_file(File, Bin),
|
||||
update_file(File),
|
||||
|
||||
?assertEqual(ok, emqx_mgmt_auth:init_bootstrap_file(File)),
|
||||
?assertEqual(ok, emqx_mgmt_auth:init_bootstrap_file()),
|
||||
|
||||
MatchFun = fun(ApiKey) -> mnesia:match_object(#?APP{api_key = ApiKey, _ = '_'}) end,
|
||||
?assertMatch(
|
||||
|
@ -156,7 +156,7 @@ t_bootstrap_file_dup_override(_) ->
|
|||
File = "./bootstrap_api_keys.txt",
|
||||
ok = file:write_file(File, Bin),
|
||||
update_file(File),
|
||||
?assertEqual(ok, emqx_mgmt_auth:init_bootstrap_file(File)),
|
||||
?assertEqual(ok, emqx_mgmt_auth:init_bootstrap_file()),
|
||||
|
||||
SameAppWithDiffName = #?APP{
|
||||
name = <<"name-1">>,
|
||||
|
@ -190,7 +190,7 @@ t_bootstrap_file_dup_override(_) ->
|
|||
|
||||
%% Similar to loading bootstrap file at node startup
|
||||
%% the duplicated apikey in mnesia will be cleaned up
|
||||
?assertEqual(ok, emqx_mgmt_auth:init_bootstrap_file(File)),
|
||||
?assertEqual(ok, emqx_mgmt_auth:init_bootstrap_file()),
|
||||
?assertMatch(
|
||||
{ok, [
|
||||
#?APP{
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_modules, [
|
||||
{description, "EMQX Modules"},
|
||||
{vsn, "5.0.28"},
|
||||
{vsn, "5.0.27"},
|
||||
{modules, []},
|
||||
{applications, [kernel, stdlib, emqx, emqx_ctl, observer_cli]},
|
||||
{mod, {emqx_modules_app, []}},
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{application, emqx_node_rebalance, [
|
||||
{description, "EMQX Node Rebalance"},
|
||||
{vsn, "5.0.10"},
|
||||
{vsn, "5.0.9"},
|
||||
{registered, [
|
||||
emqx_node_rebalance_sup,
|
||||
emqx_node_rebalance,
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_plugins, [
|
||||
{description, "EMQX Plugin Management"},
|
||||
{vsn, "0.2.3"},
|
||||
{vsn, "0.2.2"},
|
||||
{modules, []},
|
||||
{mod, {emqx_plugins_app, []}},
|
||||
{applications, [kernel, stdlib, emqx, erlavro]},
|
||||
|
|
|
@ -1049,22 +1049,19 @@ do_load_plugin_app(AppName, Ebin) ->
|
|||
end.
|
||||
|
||||
start_app(App) ->
|
||||
case run_with_timeout(application, ensure_all_started, [App], 10_000) of
|
||||
{ok, {ok, Started}} ->
|
||||
case application:ensure_all_started(App) of
|
||||
{ok, Started} ->
|
||||
case Started =/= [] of
|
||||
true -> ?SLOG(debug, #{msg => "started_plugin_apps", apps => Started});
|
||||
false -> ok
|
||||
end;
|
||||
{ok, {error, Reason}} ->
|
||||
throw(#{
|
||||
msg => "failed_to_start_app",
|
||||
app => App,
|
||||
reason => Reason
|
||||
});
|
||||
{error, Reason} ->
|
||||
end,
|
||||
?SLOG(debug, #{msg => "started_plugin_app", app => App}),
|
||||
ok;
|
||||
{error, {ErrApp, Reason}} ->
|
||||
throw(#{
|
||||
msg => "failed_to_start_plugin_app",
|
||||
app => App,
|
||||
err_app => ErrApp,
|
||||
reason => Reason
|
||||
})
|
||||
end.
|
||||
|
@ -1589,20 +1586,3 @@ bin(B) when is_binary(B) -> B.
|
|||
|
||||
wrap_to_list(Path) ->
|
||||
binary_to_list(iolist_to_binary(Path)).
|
||||
|
||||
run_with_timeout(Module, Function, Args, Timeout) ->
|
||||
Self = self(),
|
||||
Fun = fun() ->
|
||||
Result = apply(Module, Function, Args),
|
||||
Self ! {self(), Result}
|
||||
end,
|
||||
Pid = spawn(Fun),
|
||||
TimerRef = erlang:send_after(Timeout, self(), {timeout, Pid}),
|
||||
receive
|
||||
{Pid, Result} ->
|
||||
_ = erlang:cancel_timer(TimerRef),
|
||||
{ok, Result};
|
||||
{timeout, Pid} ->
|
||||
exit(Pid, kill),
|
||||
{error, timeout}
|
||||
end.
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
{application, emqx_prometheus, [
|
||||
{description, "Prometheus for EMQX"},
|
||||
% strict semver, bump manually!
|
||||
{vsn, "5.2.4"},
|
||||
{vsn, "5.2.3"},
|
||||
{modules, []},
|
||||
{registered, [emqx_prometheus_sup]},
|
||||
{applications, [kernel, stdlib, prometheus, emqx, emqx_auth, emqx_resource, emqx_management]},
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
%% -*- mode: erlang -*-
|
||||
{application, emqx_resource, [
|
||||
{description, "Manager for all external resources"},
|
||||
{vsn, "0.1.33"},
|
||||
{vsn, "0.1.32"},
|
||||
{registered, []},
|
||||
{mod, {emqx_resource_app, []}},
|
||||
{applications, [
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue