From c1f77da3b91caf92c592df727918ae323a32dc7b Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 15 Feb 2022 10:11:34 +0800 Subject: [PATCH 01/47] ci(build_packages): add target event --- .github/workflows/build_packages.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 8573d84ac..13a463da9 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -1,6 +1,9 @@ name: Cross build packages on: + push: + branches: + - 'main-v4.**' schedule: - cron: '0 */6 * * *' release: From 88769bc23bd10a5f290164524583bec23a750da3 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 15 Feb 2022 10:15:25 +0800 Subject: [PATCH 02/47] ci(build_packages): cancel repeating actions --- .github/workflows/build_packages.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 13a463da9..b3e63d58f 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -1,5 +1,9 @@ name: Cross build packages +concurrency: + group: build-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + on: push: branches: From 091398d863beac5475e4742f8cdcc9adf584bb05 Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Tue, 15 Feb 2022 11:54:40 +0800 Subject: [PATCH 03/47] build(relup): add new script for download relup base packages --- .github/workflows/build_packages.yaml | 34 ++------------- .github/workflows/run_fvt_tests.yaml | 10 +---- Makefile | 12 +++++- build | 7 ++- deploy/docker/Dockerfile | 2 + scripts/relup-base-packages.sh | 62 +++++++++++++++++++++++++++ scripts/relup-base-vsns.sh | 3 ++ 7 files changed, 87 insertions(+), 43 deletions(-) create mode 100755 scripts/relup-base-packages.sh diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 8573d84ac..08d92e981 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -15,7 +15,6 @@ jobs: outputs: profiles: ${{ steps.set_profile.outputs.profiles}} - old_vsns: ${{ steps.set_profile.outputs.old_vsns}} steps: - uses: actions/checkout@v2 @@ -28,12 +27,8 @@ jobs: run: | cd source if make emqx-ee --dry-run > /dev/null 2>&1; then - old_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)" - echo "::set-output name=old_vsns::$old_vsns" echo "::set-output name=profiles::[\"emqx-ee\"]" else - old_vsns="$(./scripts/relup-base-vsns.sh community | xargs)" - echo "::set-output name=old_vsns::$old_vsns" echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]" fi - name: get_all_deps @@ -216,6 +211,7 @@ jobs: needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} arch: @@ -269,32 +265,6 @@ jobs: path: . - name: unzip source code run: unzip -q source.zip - - name: downloads old emqx zip packages - env: - PROFILE: ${{ matrix.profile }} - ARCH: ${{ matrix.arch }} - SYSTEM: ${{ matrix.os }} - OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }} - run: | - set -e -x -u - broker=$PROFILE - if [ $PROFILE = "emqx" ];then - broker="emqx-ce" - fi - if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then - export ARCH="arm" - fi - - mkdir -p source/_upgrade_base - cd source/_upgrade_base - old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) - for tag in ${old_vsns[@]}; do - if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 - echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1 - fi - done - name: build emqx packages env: ERL_OTP: erl23.2.7.2-emqx-3 @@ -342,6 +312,7 @@ jobs: needs: prepare strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} registry: @@ -439,6 +410,7 @@ jobs: needs: [prepare, mac, linux, docker] strategy: + fail-fast: false matrix: profile: ${{fromJSON(needs.prepare.outputs.profiles)}} diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 6d4d94461..f8f03aabb 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -239,6 +239,7 @@ jobs: name: Checkout with: path: emqx + fetch-depth: 0 - name: Prepare credentials run: | if [ "$PROFILE" = "emqx-ee" ]; then @@ -246,15 +247,6 @@ jobs: git config --global credential.helper store echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token fi - - name: Download bases - run: | - set -e -x -u - mkdir -p emqx/_upgrade_base - cd emqx/_upgrade_base - old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) - for old_vsn in ${old_vsns[@]}; do - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip - done - name: Build emqx run: make -C emqx ${PROFILE}-zip - uses: actions/upload-artifact@v2 diff --git a/Makefile b/Makefile index ed7c5b6a3..6b3282272 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ REBAR_VERSION = 3.14.3-emqx-8 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts +export EMQX_RELUP ?= true export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine export EMQX_DEFAULT_RUNNER = alpine:3.12 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) @@ -125,10 +126,19 @@ COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS) $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS) @$(BUILD) $(subst -rel,,$(@)) rel +## download relup base packages +.PHONY: $(REL_PROFILES:%=%-relup-downloads) +define download-relup-packages +$1-relup-downloads: + @if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-base-packages.sh $1; fi +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt)))) + ## relup target is to create relup instructions .PHONY: $(REL_PROFILES:%=%-relup) define gen-relup-target -$1-relup: $(COMMON_DEPS) +$1-relup: $1-relup-downloads $(COMMON_DEPS) @$(BUILD) $1 relup endef ALL_ZIPS = $(REL_PROFILES) diff --git a/build b/build index 684057bf1..96fc9db10 100755 --- a/build +++ b/build @@ -4,6 +4,9 @@ # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg # arg2: artifact, e.g. rel | relup | zip | pkg +if [[ -n "$DEBUG" ]]; then + set -x +fi set -euo pipefail PROFILE="$1" @@ -71,8 +74,8 @@ make_relup() { tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir" unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir" - cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" - cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" + cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" || true + cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" || true rm -rf "$tmp_dir" fi releases+=( "$base_vsn" ) diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index c79508ec1..b11ad1ee8 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -23,6 +23,8 @@ COPY . /emqx ARG PKG_VSN ARG EMQX_NAME=emqx +ENV EMQX_RELUP=false + RUN cd /emqx \ && rm -rf _build/$EMQX_NAME/lib \ && make $EMQX_NAME diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh new file mode 100755 index 000000000..90aefe04f --- /dev/null +++ b/scripts/relup-base-packages.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +if [[ -n "$DEBUG" ]]; then + set -x +fi +set -euo pipefail + +PROFILE="${1}" +if [ "$PROFILE" = "" ]; then + $PROFILE="emqx" +fi + +case $PROFILE in + "emqx") + BROKER="broker" + ;; + "emqx-ee") + BROKER="enterprise" + ;; + "emqx-edge") + BROKER="edge" + ;; +esac + +SYSTEM="$(./scripts/get-distro.sh)" + +ARCH="$(uname -m)" +case "$ARCH" in + x86_64) + ARCH='amd64' + ;; + aarch64) + ARCH='arm64' + ;; + arm*) + ARCH=arm + ;; +esac + +SHASUM="sha256sum" +if [ "$SYSTEM" = "macos" ]; then + SHASUM="shasum -a 256" +fi + +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + +mkdir -p _upgrade_base +pushd _upgrade_base + +for tag in $(../scripts/relup-base-vsns.sh community | xargs echo -n); do + if [ ! -f "$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" ] \ + && [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://www.emqx.com/downloads/$BROKER/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then + wget --no-verbose https://www.emqx.com/downloads/$BROKER/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip + if [ "$SYSTEM" != "centos6" ]; then + wget --no-verbose https://www.emqx.com/downloads/$BROKER/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 + ## https://askubuntu.com/questions/1202208/checking-sha256-checksum + echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | $SHASUM -c || exit 1 + fi + fi +done + +popd diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 65d755994..96e43bb65 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -euo pipefail +# ensure dir +cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." + ## This script prints the relup upgrade base versions ## for the given EMQ X edition (specified as first arg) ## From e09e5673eb9d4793a2fc940826730cf6b039375a Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 15 Feb 2022 16:09:42 +0100 Subject: [PATCH 04/47] docs(README): EMQ X -> EMQX --- README-CN.md | 26 ++++++------ README-JP.md | 26 ++++++------ README-RU.md | 18 ++++----- README.md | 22 +++++----- apps/emqx_auth_http/README.md | 4 +- apps/emqx_auth_jwt/README.md | 4 +- apps/emqx_auth_ldap/README.md | 4 +- apps/emqx_auth_mongo/README.md | 4 +- apps/emqx_auth_mysql/README.md | 2 +- apps/emqx_auth_pgsql/README.md | 2 +- apps/emqx_auth_redis/README.md | 4 +- apps/emqx_bridge_mqtt/README.md | 26 ++++++------ apps/emqx_bridge_mqtt/docs/guide.rst | 26 ++++++------ apps/emqx_coap/README.md | 4 +- apps/emqx_exhook/README.md | 4 +- apps/emqx_exproto/README.md | 4 +- apps/emqx_lua_hook/README.md | 2 +- apps/emqx_lwm2m/README.md | 4 +- apps/emqx_management/README.md | 2 +- apps/emqx_prometheus/README.md | 6 +-- apps/emqx_recon/README.md | 4 +- apps/emqx_retainer/README.md | 4 +- apps/emqx_rule_engine/README.md | 2 +- apps/emqx_sn/README.md | 4 +- apps/emqx_stomp/README.md | 4 +- apps/emqx_web_hook/README.md | 4 +- deploy/charts/emqx/README.md | 60 ++++++++++++++-------------- deploy/docker/README.md | 20 +++++----- lib-ce/emqx_dashboard/README.md | 2 +- lib-extra/README.md | 4 +- 30 files changed, 151 insertions(+), 151 deletions(-) diff --git a/README-CN.md b/README-CN.md index 7d2888327..ae03a7280 100644 --- a/README-CN.md +++ b/README-CN.md @@ -1,4 +1,4 @@ -# EMQ X Broker +# EMQX Broker [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases) [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) @@ -13,20 +13,20 @@ [English](./README.md) | 简体中文 | [日本語](./README-JP.md) | [русский](./README-RU.md) -*EMQ X* 是一款完全开源，高度可伸缩，高可用的分布式 MQTT 消息服务器，适用于 IoT、M2M 和移动应用程序，可处理千万级别的并发客户端。 +*EMQX* 是一款完全开源，高度可伸缩，高可用的分布式 MQTT 消息服务器，适用于 IoT、M2M 和移动应用程序，可处理千万级别的并发客户端。 -从 3.0 版本开始，*EMQ X* 完整支持 MQTT V5.0 协议规范，向下兼容 MQTT V3.1 和 V3.1.1，并支持 MQTT-SN、CoAP、LwM2M、WebSocket 和 STOMP 等通信协议。EMQ X 3.0 单集群可支持千万级别的 MQTT 并发连接。 +从 3.0 版本开始，*EMQX* 完整支持 MQTT V5.0 协议规范，向下兼容 MQTT V3.1 和 V3.1.1，并支持 MQTT-SN、CoAP、LwM2M、WebSocket 和 STOMP 等通信协议。EMQX 3.0 单集群可支持千万级别的 MQTT 并发连接。 -- 新功能的完整列表，请参阅 [EMQ X Release Notes](https://github.com/emqx/emqx/releases)。 -- 获取更多信息，请访问 [EMQ X 官网](https://www.emqx.cn/)。 +- 新功能的完整列表，请参阅 [EMQX Release Notes](https://github.com/emqx/emqx/releases)。 +- 获取更多信息，请访问 [EMQX 官网](https://www.emqx.cn/)。 ## 安装 -*EMQ X* 是跨平台的，支持 Linux、Unix、macOS 以及 Windows。这意味着 *EMQ X* 可以部署在 x86_64 架构的服务器上，也可以部署在 Raspberry Pi 这样的 ARM 设备上。 +*EMQX* 是跨平台的，支持 Linux、Unix、macOS 以及 Windows。这意味着 *EMQX* 可以部署在 x86_64 架构的服务器上，也可以部署在 Raspberry Pi 这样的 ARM 设备上。 -Windows 上编译和运行 *EMQ X* 的详情参考：[Windows.md](./Windows.md) +Windows 上编译和运行 *EMQX* 的详情参考：[Windows.md](./Windows.md) -#### EMQ X Docker 镜像安装 +#### EMQX Docker 镜像安装 ``` docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx @@ -34,14 +34,14 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p #### 二进制软件包安装 -需从 [EMQ X 下载](https://www.emqx.cn/downloads) 页面获取相应操作系统的二进制软件包。 +需从 [EMQX 下载](https://www.emqx.cn/downloads) 页面获取相应操作系统的二进制软件包。 - [单节点安装文档](https://docs.emqx.cn/broker/latest/getting-started/install.html) - [集群配置文档](https://docs.emqx.cn/broker/latest/advanced/cluster.html) ## 从源码构建 -3.0 版本开始，构建 *EMQ X* 需要 Erlang/OTP R21+。 +3.0 版本开始，构建 *EMQX* 需要 Erlang/OTP R21+。 4.3 及以后的版本： @@ -77,7 +77,7 @@ _build/emqx/rel/emqx/bin/emqx console ./bin/emqx stop ``` -*EMQ X* 启动，可以使用浏览器访问 http://localhost:18083 来查看 Dashboard。 +*EMQX* 启动，可以使用浏览器访问 http://localhost:18083 来查看 Dashboard。 ## 测试 @@ -108,7 +108,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer ### FAQ -访问 [EMQ X FAQ](https://docs.emqx.cn/broker/latest/faq/faq.html) 以获取常见问题的帮助。 +访问 [EMQX FAQ](https://docs.emqx.cn/broker/latest/faq/faq.html) 以获取常见问题的帮助。 ### 问答 @@ -117,7 +117,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer ### 参与设计 -如果对 EMQ X 有改进建议，可以向[EIP](https://github.com/emqx/eip) 提交 PR 和 ISSUE +如果对 EMQX 有改进建议，可以向[EIP](https://github.com/emqx/eip) 提交 PR 和 ISSUE ### 插件开发 diff --git a/README-JP.md b/README-JP.md index b7afe8195..4a8ac5d6d 100644 --- a/README-JP.md +++ b/README-JP.md @@ -1,4 +1,4 @@ -# EMQ X Broker +# EMQX Broker [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases) [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) @@ -12,22 +12,22 @@ [English](./README.md) | [简体中文](./README-CN.md) | 日本語 | [русский](./README-RU.md) -*EMQ X* は、高い拡張性と可用性をもつ、分散型のMQTTブローカーです。数千万のクライアントを同時に処理するIoT、M2M、モバイルアプリケーション向けです。 +*EMQX* は、高い拡張性と可用性をもつ、分散型のMQTTブローカーです。数千万のクライアントを同時に処理するIoT、M2M、モバイルアプリケーション向けです。 -version 3.0 以降、*EMQ X* は MQTT V5.0 の仕様を完全にサポートしており、MQTT V3.1およびV3.1.1とも下位互換性があります。 +version 3.0 以降、*EMQX* は MQTT V5.0 の仕様を完全にサポートしており、MQTT V3.1およびV3.1.1とも下位互換性があります。 MQTT-SN、CoAP、LwM2M、WebSocket、STOMPなどの通信プロトコルをサポートしています。 MQTTの同時接続数は1つのクラスター上で1,000万以上にまでスケールできます。 -- 新機能の一覧については、[EMQ Xリリースノート](https://github.com/emqx/emqx/releases)を参照してください。 -- 詳細はこちら[EMQ X公式ウェブサイト](https://www.emqx.io/)をご覧ください。 +- 新機能の一覧については、[EMQXリリースノート](https://github.com/emqx/emqx/releases)を参照してください。 +- 詳細はこちら[EMQX公式ウェブサイト](https://www.emqx.io/)をご覧ください。 ## インストール -*EMQ X* はクロスプラットフォームで、Linux、Unix、macOS、Windowsをサポートしています。 -そのため、x86_64アーキテクチャサーバー、またはRaspberryPiなどのARMデバイスに *EMQ X* をデプロイすることもできます。 +*EMQX* はクロスプラットフォームで、Linux、Unix、macOS、Windowsをサポートしています。 +そのため、x86_64アーキテクチャサーバー、またはRaspberryPiなどのARMデバイスに *EMQX* をデプロイすることもできます。 -Windows上における *EMQ X* のビルドと実行については、[Windows.md](./Windows.md)をご参照ください。 +Windows上における *EMQX* のビルドと実行については、[Windows.md](./Windows.md)をご参照ください。 -#### Docker イメージによる EMQ X のインストール +#### Docker イメージによる EMQX のインストール ``` docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx @@ -35,14 +35,14 @@ docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p #### バイナリパッケージによるインストール -それぞれのOSに対応したバイナリソフトウェアパッケージは、[EMQ Xのダウンロード](https://www.emqx.io/downloads)ページから取得できます。 +それぞれのOSに対応したバイナリソフトウェアパッケージは、[EMQXのダウンロード](https://www.emqx.io/downloads)ページから取得できます。 - [シングルノードインストール](https://docs.emqx.io/broker/latest/en/getting-started/installation.html) - [マルチノードインストール](https://docs.emqx.io/broker/latest/en/advanced/cluster.html) ## ソースからビルド -version 3.0 以降の *EMQ X* をビルドするには Erlang/OTP R21+ が必要です。 +version 3.0 以降の *EMQX* をビルドするには Erlang/OTP R21+ が必要です。 version 4.3 以降の場合： @@ -71,7 +71,7 @@ emqx をソースコードからビルドした場合は、 ./bin/emqx stop ``` -*EMQ X* の起動後、ブラウザで http://localhost:18083 にアクセスするとダッシュボードが表示されます。 +*EMQX* の起動後、ブラウザで http://localhost:18083 にアクセスするとダッシュボードが表示されます。 ## テスト @@ -102,7 +102,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer ### FAQ -よくある質問については、[EMQ X FAQ](https://docs.emqx.io/broker/latest/en/faq/faq.html)をご確認ください。 +よくある質問については、[EMQX FAQ](https://docs.emqx.io/broker/latest/en/faq/faq.html)をご確認ください。 ### 質問する diff --git a/README-RU.md b/README-RU.md index cddaba4a5..d7bbb263e 100644 --- a/README-RU.md +++ b/README-RU.md @@ -1,4 +1,4 @@ -# Брокер EMQ X +# Брокер EMQX [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases) [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) @@ -13,20 +13,20 @@ [English](./README.md) | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | русский -*EMQ X* — это масштабируемый, высоко доступный, распределённый MQTT брокер с полностью открытым кодом для интернета вещей, межмашинного взаимодействия и мобильных приложений, который поддерживает миллионы одновременных подключений. +*EMQX* — это масштабируемый, высоко доступный, распределённый MQTT брокер с полностью открытым кодом для интернета вещей, межмашинного взаимодействия и мобильных приложений, который поддерживает миллионы одновременных подключений. -Начиная с релиза 3.0, брокер *EMQ X* полностью поддерживает протокол MQTT версии 5.0, и обратно совместим с версиями 3.1 и 3.1.1, а также протоколами MQTT-SN, CoAP, LwM2M, WebSocket и STOMP. Начиная с релиза 3.0, брокер *EMQ X* может масштабироваться до более чем 10 миллионов одновременных MQTT соединений на один кластер. +Начиная с релиза 3.0, брокер *EMQX* полностью поддерживает протокол MQTT версии 5.0, и обратно совместим с версиями 3.1 и 3.1.1, а также протоколами MQTT-SN, CoAP, LwM2M, WebSocket и STOMP. Начиная с релиза 3.0, брокер *EMQX* может масштабироваться до более чем 10 миллионов одновременных MQTT соединений на один кластер. -- Полный список возможностей доступен по ссылке: [EMQ X Release Notes](https://github.com/emqx/emqx/releases). -- Более подробная информация доступна на нашем сайте: [EMQ X homepage](https://www.emqx.io). +- Полный список возможностей доступен по ссылке: [EMQX Release Notes](https://github.com/emqx/emqx/releases). +- Более подробная информация доступна на нашем сайте: [EMQX homepage](https://www.emqx.io). ## Установка -Брокер *EMQ X* кросплатформенный, и поддерживает Linux, Unix, macOS и Windows. Он может работать на серверах с архитектурой x86_64 и устройствах на архитектуре ARM, таких как Raspberry Pi. +Брокер *EMQX* кросплатформенный, и поддерживает Linux, Unix, macOS и Windows. Он может работать на серверах с архитектурой x86_64 и устройствах на архитектуре ARM, таких как Raspberry Pi. Более подробная информация о запуске на Windows по ссылке: [Windows.md](./Windows.md) -#### Установка EMQ X с помощью Docker-образа +#### Установка EMQX с помощью Docker-образа ``` docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx @@ -34,7 +34,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p #### Установка бинарного пакета -Сборки для различных операционных систем: [Загрузить EMQ X](https://www.emqx.io/downloads). +Сборки для различных операционных систем: [Загрузить EMQX](https://www.emqx.io/downloads). - [Установка на одном сервере](https://docs.emqx.io/en/broker/latest/getting-started/install.html) - [Установка на кластере](https://docs.emqx.io/en/broker/latest/advanced/cluster.html) @@ -111,7 +111,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer ### FAQ -Наиболее частые проблемы разобраны в [EMQ X FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html). +Наиболее частые проблемы разобраны в [EMQX FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html). ### Вопросы diff --git a/README.md b/README.md index 8d8ed8731..32d505b34 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# EMQ X Broker +# EMQX Broker [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases) [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) @@ -12,20 +12,20 @@ English | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | [русский](./README-RU.md) -*EMQ X* broker is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients. +*EMQX* broker is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients. -Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scaled to 10+ million concurrent MQTT connections on one cluster. +Starting from 3.0 release, *EMQX* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQX* broker can scaled to 10+ million concurrent MQTT connections on one cluster. -- For full list of new features, please read [EMQ X Release Notes](https://github.com/emqx/emqx/releases). -- For more information, please visit [EMQ X homepage](https://www.emqx.io). +- For full list of new features, please read [EMQX Release Notes](https://github.com/emqx/emqx/releases). +- For more information, please visit [EMQX homepage](https://www.emqx.io). ## Installation -The *EMQ X* broker is cross-platform, which supports Linux, Unix, macOS and Windows. It means *EMQ X* can be deployed on x86_64 architecture servers and ARM devices like Raspberry Pi. +The *EMQX* broker is cross-platform, which supports Linux, Unix, macOS and Windows. It means *EMQX* can be deployed on x86_64 architecture servers and ARM devices like Raspberry Pi. -See more details for building and running *EMQ X* on Windows in [Windows.md](./Windows.md) +See more details for building and running *EMQX* on Windows in [Windows.md](./Windows.md) -#### Installing via EMQ X Docker Image +#### Installing via EMQX Docker Image ``` docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx @@ -33,7 +33,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p #### Installing via Binary Package -Get the binary package of the corresponding OS from [EMQ X Download](https://www.emqx.io/downloads) page. +Get the binary package of the corresponding OS from [EMQX Download](https://www.emqx.io/downloads) page. - [Single Node Install](https://docs.emqx.io/en/broker/latest/getting-started/install.html) - [Multi Node Install](https://docs.emqx.io/en/broker/latest/advanced/cluster.html) @@ -41,7 +41,7 @@ Get the binary package of the corresponding OS from [EMQ X Download](https://www ## Build From Source -The *EMQ X* broker requires Erlang/OTP R21+ to build since 3.0 release. +The *EMQX* broker requires Erlang/OTP R21+ to build since 3.0 release. For 4.3 and later versions. @@ -110,7 +110,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer ### FAQ -Visiting [EMQ X FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html) to get help of common problems. +Visiting [EMQX FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html) to get help of common problems. ### Questions diff --git a/apps/emqx_auth_http/README.md b/apps/emqx_auth_http/README.md index ed743334a..8d6819a46 100644 --- a/apps/emqx_auth_http/README.md +++ b/apps/emqx_auth_http/README.md @@ -1,7 +1,7 @@ emqx_auth_http ============== -EMQ X HTTP Auth/ACL Plugin +EMQX HTTP Auth/ACL Plugin Build ----- @@ -96,5 +96,5 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_auth_jwt/README.md b/apps/emqx_auth_jwt/README.md index 9675ae87c..7af41f490 100644 --- a/apps/emqx_auth_jwt/README.md +++ b/apps/emqx_auth_jwt/README.md @@ -1,7 +1,7 @@ # emqx-auth-jwt -EMQ X JWT Authentication Plugin +EMQX JWT Authentication Plugin Build ----- @@ -87,4 +87,4 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_auth_ldap/README.md b/apps/emqx_auth_ldap/README.md index c4d56c839..39b23b72a 100644 --- a/apps/emqx_auth_ldap/README.md +++ b/apps/emqx_auth_ldap/README.md @@ -1,7 +1,7 @@ emqx_auth_ldap ============== -EMQ X LDAP Authentication Plugin +EMQX LDAP Authentication Plugin Build ----- @@ -92,5 +92,5 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_auth_mongo/README.md b/apps/emqx_auth_mongo/README.md index 3bacfca5b..68db9c4e6 100644 --- a/apps/emqx_auth_mongo/README.md +++ b/apps/emqx_auth_mongo/README.md @@ -1,7 +1,7 @@ emqx_auth_mongo =============== -EMQ X Authentication/ACL with MongoDB +EMQX Authentication/ACL with MongoDB Build the Plugin ---------------- @@ -188,5 +188,5 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_auth_mysql/README.md b/apps/emqx_auth_mysql/README.md index e55a2103f..cde1f8639 100644 --- a/apps/emqx_auth_mysql/README.md +++ b/apps/emqx_auth_mysql/README.md @@ -164,4 +164,4 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_auth_pgsql/README.md b/apps/emqx_auth_pgsql/README.md index 2dccd6f53..5ef4345d2 100644 --- a/apps/emqx_auth_pgsql/README.md +++ b/apps/emqx_auth_pgsql/README.md @@ -179,5 +179,5 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_auth_redis/README.md b/apps/emqx_auth_redis/README.md index 9aa851f88..f83787d14 100644 --- a/apps/emqx_auth_redis/README.md +++ b/apps/emqx_auth_redis/README.md @@ -1,7 +1,7 @@ emqx_auth_redis =============== -EMQ X Redis Authentication/ACL Plugin +EMQX Redis Authentication/ACL Plugin Features --------- @@ -167,5 +167,5 @@ Load Plugin Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_bridge_mqtt/README.md b/apps/emqx_bridge_mqtt/README.md index 456fae584..fea8f46b3 100644 --- a/apps/emqx_bridge_mqtt/README.md +++ b/apps/emqx_bridge_mqtt/README.md @@ -1,13 +1,13 @@ # EMQ Bridge MQTT -The concept of **Bridge** means that EMQ X supports forwarding messages +The concept of **Bridge** means that EMQX supports forwarding messages of one of its own topics to another MQTT Broker in some way. **Bridge** differs from **Cluster** in that the bridge does not replicate the topic trie and routing tables and only forwards MQTT messages based on bridging rules. -At present, the bridging methods supported by EMQ X are as follows: +At present, the bridging methods supported by EMQX are as follows: - RPC bridge: RPC Bridge only supports message forwarding and does not support subscribing to the topic of remote nodes to synchronize @@ -19,7 +19,7 @@ These concepts are shown below: ![bridge](docs/images/bridge.png) -In addition, the EMQ X message broker supports multi-node bridge mode interconnection +In addition, the EMQX message broker supports multi-node bridge mode interconnection ``` --------- --------- --------- @@ -27,7 +27,7 @@ Publisher --> | Node1 | --Bridge Forward--> | Node2 | --Bridge Forward--> | Node --------- --------- --------- ``` -In EMQ X, bridge is configured by modifying `etc/emqx.conf`. EMQ X distinguishes between different bridges based on different names. E.g +In EMQX, bridge is configured by modifying `etc/emqx.conf`. EMQX distinguishes between different bridges based on different names. E.g ``` ## Bridge address: node name for local bridge, host:port for remote. @@ -38,7 +38,7 @@ This configuration declares a bridge named `aws` and specifies that it is bridge In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary (such as bridge.$name.address, where $name refers to the name of bridge) -The next two sections describe how to create a bridge in RPC and MQTT mode respectively and create a forwarding rule that forwards the messages from sensors. Assuming that two EMQ X nodes are running on two hosts: +The next two sections describe how to create a bridge in RPC and MQTT mode respectively and create a forwarding rule that forwards the messages from sensors. Assuming that two EMQX nodes are running on two hosts: | Name | Node | MQTT Port | @@ -47,7 +47,7 @@ The next two sections describe how to create a bridge in RPC and MQTT mode respe | emqx2| emqx2@192.168.1.2 | 1883 | -## EMQ X RPC Bridge Configuration +## EMQX RPC Bridge Configuration The following is the basic configuration of RPC bridging. A simplest RPC bridging only requires the following three items @@ -72,13 +72,13 @@ Limitations of RPC bridging: 1. The RPC bridge of emqx can only forward local messages to the remote node, and cannot synchronize the messages of the remote node to the local node; -2. RPC bridge can only bridge two EMQ X broker together and cannot bridge EMQ X broker to other MQTT brokers. +2. RPC bridge can only bridge two EMQX broker together and cannot bridge EMQX broker to other MQTT brokers. -## EMQ X MQTT Bridge Configuration +## EMQX MQTT Bridge Configuration -EMQ X 3.0 officially introduced MQTT bridge, so that EMQ X can bridge any MQTT broker. Because of the characteristics of the MQTT protocol, EMQ X can subscribe to the remote mqtt broker's topic through MQTT bridge, and then synchronize the remote MQTT broker's message to the local. +EMQX 3.0 officially introduced MQTT bridge, so that EMQX can bridge any MQTT broker. Because of the characteristics of the MQTT protocol, EMQX can subscribe to the remote mqtt broker's topic through MQTT bridge, and then synchronize the remote MQTT broker's message to the local. -EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and connect this MQTT client to the remote MQTT broker. Therefore, in the MQTT bridge configuration, following fields may be set for the EMQ X to connect to the remote broker as an mqtt client +EMQX MQTT bridging principle: Create an MQTT client on the EMQX broker, and connect this MQTT client to the remote MQTT broker. Therefore, in the MQTT bridge configuration, following fields may be set for the EMQX to connect to the remote broker as an mqtt client ``` ## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection @@ -159,7 +159,7 @@ bridge.mqtt.emqx2.max_inflight_batches = 32 ## Bridge Cache Configuration -The bridge of EMQ X has a message caching mechanism. The caching mechanism is applicable to both RPC bridging and MQTT bridging. When the bridge is disconnected (such as when the network connection is unstable), the messages with a topic specified in `forwards` can be cached to the local message queue. Until the bridge is restored, these messages are re-forwarded to the remote node. The configuration of the cache queue is as follows +The bridge of EMQX has a message caching mechanism. The caching mechanism is applicable to both RPC bridging and MQTT bridging. When the bridge is disconnected (such as when the network connection is unstable), the messages with a topic specified in `forwards` can be cached to the local message queue. Until the bridge is restored, these messages are re-forwarded to the remote node. The configuration of the cache queue is as follows ``` ## emqx_bridge internal number of messages used for batch @@ -179,9 +179,9 @@ bridge.mqtt.emqx2.queue.replayq_seg_bytes = 10MB `bridge.mqtt.emqx2.queue.replayq_seg_bytes` is used to specify the size of the largest single file of the message queue that is cached on disk. If the message queue size exceeds the specified value, a new file is created to store the message queue. -## CLI for EMQ X Bridge MQTT +## CLI for EMQX Bridge MQTT -CLI for EMQ X Bridge MQTT: +CLI for EMQX Bridge MQTT: ``` bash $ cd emqx1/ && ./bin/emqx_ctl bridges diff --git a/apps/emqx_bridge_mqtt/docs/guide.rst b/apps/emqx_bridge_mqtt/docs/guide.rst index 47235aa7d..bb0808367 100644 --- a/apps/emqx_bridge_mqtt/docs/guide.rst +++ b/apps/emqx_bridge_mqtt/docs/guide.rst @@ -2,14 +2,14 @@ EMQ Bridge MQTT =============== -The concept of **Bridge** means that EMQ X supports forwarding messages +The concept of **Bridge** means that EMQX supports forwarding messages of one of its own topics to another MQTT Broker in some way. **Bridge** differs from **Cluster** in that the bridge does not replicate the topic trie and routing tables and only forwards MQTT messages based on bridging rules. -At present, the bridging methods supported by EMQ X are as follows: +At present, the bridging methods supported by EMQX are as follows: * RPC bridge: RPC Bridge only supports message forwarding and does not @@ -26,7 +26,7 @@ These concepts are shown below: :alt: bridge -In addition, the EMQ X message broker supports multi-node bridge mode interconnection +In addition, the EMQX message broker supports multi-node bridge mode interconnection .. code-block:: @@ -34,7 +34,7 @@ In addition, the EMQ X message broker supports multi-node bridge mode interconne Publisher --> | Node1 | --Bridge Forward--> | Node2 | --Bridge Forward--> | Node3 | --> Subscriber --------- --------- --------- -In EMQ X, bridge is configured by modifying ``etc/emqx.conf``. EMQ X distinguishes between different bridges based on different names. E.g +In EMQX, bridge is configured by modifying ``etc/emqx.conf``. EMQX distinguishes between different bridges based on different names. E.g .. code-block:: @@ -45,7 +45,7 @@ This configuration declares a bridge named ``aws`` and specifies that it is brid In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary (such as bridge.$name.address, where $name refers to the name of bridge) -The next two sections describe how to create a bridge in RPC and MQTT mode respectively and create a forwarding rule that forwards the messages from sensors. Assuming that two EMQ X nodes are running on two hosts: +The next two sections describe how to create a bridge in RPC and MQTT mode respectively and create a forwarding rule that forwards the messages from sensors. Assuming that two EMQX nodes are running on two hosts: .. list-table:: :header-rows: 1 @@ -61,7 +61,7 @@ The next two sections describe how to create a bridge in RPC and MQTT mode respe - 1883 -EMQ X RPC Bridge Configuration +EMQX RPC Bridge Configuration ------------------------------ The following is the basic configuration of RPC bridging. A simplest RPC bridging only requires the following three items @@ -90,14 +90,14 @@ Limitations of RPC bridging: The RPC bridge of emqx can only forward local messages to the remote node, and cannot synchronize the messages of the remote node to the local node; #. - RPC bridge can only bridge two EMQ X broker together and cannot bridge EMQ X broker to other MQTT brokers. + RPC bridge can only bridge two EMQX broker together and cannot bridge EMQX broker to other MQTT brokers. -EMQ X MQTT Bridge Configuration +EMQX MQTT Bridge Configuration ------------------------------- -EMQ X 3.0 officially introduced MQTT bridge, so that EMQ X can bridge any MQTT broker. Because of the characteristics of the MQTT protocol, EMQ X can subscribe to the remote mqtt broker's topic through MQTT bridge, and then synchronize the remote MQTT broker's message to the local. +EMQX 3.0 officially introduced MQTT bridge, so that EMQX can bridge any MQTT broker. Because of the characteristics of the MQTT protocol, EMQX can subscribe to the remote mqtt broker's topic through MQTT bridge, and then synchronize the remote MQTT broker's message to the local. -EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and connect this MQTT client to the remote MQTT broker. Therefore, in the MQTT bridge configuration, following fields may be set for the EMQ X to connect to the remote broker as an mqtt client +EMQX MQTT bridging principle: Create an MQTT client on the EMQX broker, and connect this MQTT client to the remote MQTT broker. Therefore, in the MQTT bridge configuration, following fields may be set for the EMQX to connect to the remote broker as an mqtt client .. code-block:: @@ -179,7 +179,7 @@ EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and co Bridge Cache Configuration -------------------------- -The bridge of EMQ X has a message caching mechanism. The caching mechanism is applicable to both RPC bridging and MQTT bridging. When the bridge is disconnected (such as when the network connection is unstable), the messages with a topic specified in ``forwards`` can be cached to the local message queue. Until the bridge is restored, these messages are re-forwarded to the remote node. The configuration of the cache queue is as follows +The bridge of EMQX has a message caching mechanism. The caching mechanism is applicable to both RPC bridging and MQTT bridging. When the bridge is disconnected (such as when the network connection is unstable), the messages with a topic specified in ``forwards`` can be cached to the local message queue. Until the bridge is restored, these messages are re-forwarded to the remote node. The configuration of the cache queue is as follows .. code-block:: @@ -199,10 +199,10 @@ The bridge of EMQ X has a message caching mechanism. The caching mechanism is ap ``bridge.mqtt.emqx2.queue.replayq_seg_bytes`` is used to specify the size of the largest single file of the message queue that is cached on disk. If the message queue size exceeds the specified value, a new file is created to store the message queue. -CLI for EMQ X Bridge MQTT +CLI for EMQX Bridge MQTT ------------------------- -CLI for EMQ X Bridge MQTT: +CLI for EMQX Bridge MQTT: .. code-block:: bash diff --git a/apps/emqx_coap/README.md b/apps/emqx_coap/README.md index 1a9ee802c..3e28c2704 100644 --- a/apps/emqx_coap/README.md +++ b/apps/emqx_coap/README.md @@ -1,7 +1,7 @@ # emqx-coap -emqx-coap is a CoAP Gateway for EMQ X Broker. It translates CoAP messages into MQTT messages and make it possible to communiate between CoAP clients and MQTT clients. +emqx-coap is a CoAP Gateway for EMQX Broker. It translates CoAP messages into MQTT messages and make it possible to communiate between CoAP clients and MQTT clients. ### Client Usage Example libcoap is an excellent coap library which has a simple client tool. It is recommended to use libcoap as a coap client. @@ -252,5 +252,5 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_exhook/README.md b/apps/emqx_exhook/README.md index 216c39275..f43cbb334 100644 --- a/apps/emqx_exhook/README.md +++ b/apps/emqx_exhook/README.md @@ -1,6 +1,6 @@ # emqx_exhook -The `emqx_exhook` extremly enhance the extensibility for EMQ X. It allow using an others programming language to mount the hooks intead of erlang. +The `emqx_exhook` extremly enhance the extensibility for EMQX. It allow using an others programming language to mount the hooks intead of erlang. ## Feature @@ -10,7 +10,7 @@ The `emqx_exhook` extremly enhance the extensibility for EMQ X. It allow using a ## Architecture ``` -EMQ X Third-party Runtime +EMQX Third-party Runtime +========================+ +========+==========+ | ExHook | | | | | +----------------+ | gRPC | gRPC | User's | diff --git a/apps/emqx_exproto/README.md b/apps/emqx_exproto/README.md index a9375e5d3..792822863 100644 --- a/apps/emqx_exproto/README.md +++ b/apps/emqx_exproto/README.md @@ -1,6 +1,6 @@ # emqx-exproto -The `emqx_exproto` extremly enhance the extensibility for EMQ X. It allow using an others programming language to **replace the protocol handling layer in EMQ X Broker**. +The `emqx_exproto` extremly enhance the extensibility for EMQX. It allow using an others programming language to **replace the protocol handling layer in EMQX Broker**. ## Feature @@ -9,7 +9,7 @@ The `emqx_exproto` extremly enhance the extensibility for EMQ X. It allow using ## Architecture -![EMQ X ExProto Arch](./docs/images/exproto-arch.jpg) +![EMQX ExProto Arch](./docs/images/exproto-arch.jpg) ## Usage diff --git a/apps/emqx_lua_hook/README.md b/apps/emqx_lua_hook/README.md index a3f65a094..dc311410c 100644 --- a/apps/emqx_lua_hook/README.md +++ b/apps/emqx_lua_hook/README.md @@ -334,5 +334,5 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_lwm2m/README.md b/apps/emqx_lwm2m/README.md index f97feb12c..c2fb63c08 100644 --- a/apps/emqx_lwm2m/README.md +++ b/apps/emqx_lwm2m/README.md @@ -1,5 +1,5 @@ -# LwM2M Gateway for the EMQ X Broker. +# LwM2M Gateway for the EMQX Broker. [The LwM2M Specifications](http://www.openmobilealliance.org/release/LightweightM2M) is a Lightweight Machine to Machine protocol. @@ -354,4 +354,4 @@ Apache License Version 2.0 ## Author -EMQ X-Men Team. +EMQX-Men Team. diff --git a/apps/emqx_management/README.md b/apps/emqx_management/README.md index c71a47628..1c23add00 100644 --- a/apps/emqx_management/README.md +++ b/apps/emqx_management/README.md @@ -1,7 +1,7 @@ # emqx-management -EMQ X Management API +EMQX Management API ## How to Design RESTful API? diff --git a/apps/emqx_prometheus/README.md b/apps/emqx_prometheus/README.md index 04bf32cbd..6c4461118 100644 --- a/apps/emqx_prometheus/README.md +++ b/apps/emqx_prometheus/README.md @@ -1,6 +1,6 @@ # emqx-prometheus -EMQ X Prometheus Agent +EMQX Prometheus Agent ## push emqx stats/metrics to prometheus PushGateway @@ -48,7 +48,7 @@ json data ``` -## Before EMQ X v4.0.0 +## Before EMQX v4.0.0 The prometheus data simple is: @@ -279,5 +279,5 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_recon/README.md b/apps/emqx_recon/README.md index ca163be45..7ba6b283c 100644 --- a/apps/emqx_recon/README.md +++ b/apps/emqx_recon/README.md @@ -2,7 +2,7 @@ emqx-recon ========== -EMQ X Recon Debug/Optimize Plugin +EMQX Recon Debug/Optimize Plugin emqx_recon.conf ----=---------- @@ -57,5 +57,5 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_retainer/README.md b/apps/emqx_retainer/README.md index 84d777bbc..4e5256107 100644 --- a/apps/emqx_retainer/README.md +++ b/apps/emqx_retainer/README.md @@ -1,5 +1,5 @@ -EMQ X Retainer +EMQX Retainer ============== The retainer plugin is responsible for storing retained MQTT messages. @@ -55,5 +55,5 @@ Apache License Version 2.0 Author ------ -EMQ X Team +EMQX Team diff --git a/apps/emqx_rule_engine/README.md b/apps/emqx_rule_engine/README.md index 0b1ddd848..d9104ffc8 100644 --- a/apps/emqx_rule_engine/README.md +++ b/apps/emqx_rule_engine/README.md @@ -1,7 +1,7 @@ # emqx-rule-engine -IoT Rule Engine for EMQ X Broker. +IoT Rule Engine for EMQX Broker. ## Concept diff --git a/apps/emqx_sn/README.md b/apps/emqx_sn/README.md index d7251c49c..1285463f4 100644 --- a/apps/emqx_sn/README.md +++ b/apps/emqx_sn/README.md @@ -1,7 +1,7 @@ emqx-sn ======= -EMQ X MQTT-SN Gateway. +EMQX MQTT-SN Gateway. Configure Plugin ---------------- @@ -113,5 +113,5 @@ Apache License Version 2.0 Author ------ -EMQ X-Men Team. +EMQX-Men Team. diff --git a/apps/emqx_stomp/README.md b/apps/emqx_stomp/README.md index ec841b1e6..fcade7336 100644 --- a/apps/emqx_stomp/README.md +++ b/apps/emqx_stomp/README.md @@ -2,7 +2,7 @@ emqx-stomp ========== -The plugin adds STOMP 1.0/1.1/1.2 protocol supports to the EMQ X broker. +The plugin adds STOMP 1.0/1.1/1.2 protocol supports to the EMQX broker. The STOMP clients could PubSub to the MQTT clients. @@ -73,5 +73,5 @@ Apache License Version 2.0 Author ------ -EMQ X Team. +EMQX Team. diff --git a/apps/emqx_web_hook/README.md b/apps/emqx_web_hook/README.md index c76c2936d..45a2f139e 100644 --- a/apps/emqx_web_hook/README.md +++ b/apps/emqx_web_hook/README.md @@ -1,9 +1,9 @@ # emqx-web-hook -EMQ X WebHook plugin. +EMQX WebHook plugin. -Please see: [EMQ X - WebHook](https://docs.emqx.io/broker/latest/en/advanced/webhook.html) +Please see: [EMQX - WebHook](https://docs.emqx.io/broker/latest/en/advanced/webhook.html) ## emqx_web_hook.conf diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 4837ffced..54d267415 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -1,5 +1,5 @@ # Introduction -This chart bootstraps an [EMQ X](https://www.emqx.io/) deployment on a [Kubernetes](https://kubernetes.io/) (K8s) cluster using the [Helm](https://helm.sh/) package manager. +This chart bootstraps an [EMQX](https://www.emqx.io/) deployment on a [Kubernetes](https://kubernetes.io/) (K8s) cluster using the [Helm](https://helm.sh/) package manager. # Prerequisites + [Kubernetes](https://kubernetes.io/) 1.6+ @@ -31,21 +31,21 @@ $ helm del my-emqx # Configuration The following sections describe the configurable parameters of the chart and their default values. ## [K8s]((https://kubernetes.io/)) specific settings -The following table lists the configurable K8s parameters of the [EMQ X](https://www.emqx.io/) chart and their default values. +The following table lists the configurable K8s parameters of the [EMQX](https://www.emqx.io/) chart and their default values. Parameter | Description | Default Value --- | --- | --- `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | `3` -`image.tag` | EMQ X Image tag (defaults to `.Chart.AppVersion`) | `nil` -`image.repository` | EMQ X Image repository | `emqx/emqx` +`image.tag` | EMQX Image tag (defaults to `.Chart.AppVersion`) | `nil` +`image.repository` | EMQX Image repository | `emqx/emqx` `image.pullPolicy` | The image pull policy | `IfNotPresent` `image.pullSecrets ` | The image pull secrets (does not add image pull secrets to deployed pods) |``[]`` `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | `false` -`persistence.enabled` | Enable EMQ X persistence using PVC | `false` +`persistence.enabled` | Enable EMQX persistence using PVC | `false` `persistence.storageClass` | Storage class of backing PVC (uses alpha storage class annotation) | `nil` -`persistence.existingClaim` | EMQ X data Persistent Volume existing claim name, evaluated as a template | `""` -`persistence.accessMode` | PVC Access Mode for EMQ X volume | `ReadWriteOnce` -`persistence.size` | PVC Storage Request for EMQ X volume | `20Mi` -`initContainers` | Containers that run before the creation of EMQ X containers. They can contain utilities or setup scripts. |`{}` +`persistence.existingClaim` | EMQX data Persistent Volume existing claim name, evaluated as a template | `""` +`persistence.accessMode` | PVC Access Mode for EMQX volume | `ReadWriteOnce` +`persistence.size` | PVC Storage Request for EMQX volume | `20Mi` +`initContainers` | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. |`{}` `resources` | CPU/Memory resource requests/limits |`{}` `nodeSelector` | Node labels for pod assignment |`{}` `tolerations` | Toleration labels for pod assignment |``[]`` @@ -67,32 +67,32 @@ Parameter | Description | Default Value `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | `[]` `service.externalIPs` | ExternalIPs for the service | `[]` `service.annotations` | Service annotations (evaluated as a template) | `{}` -`ingress.dashboard.enabled` | Enable ingress for EMQ X Dashboard | false -`ingress.dashboard.ingressClassName` | Set the ingress class for EMQ X Dashboard -`ingress.dashboard.path` | Ingress path for EMQ X Dashboard | `/` -`ingress.dashboard.hosts` | Ingress hosts for EMQ X Mgmt API | dashboard.emqx.local -`ingress.dashboard.tls` | Ingress tls for EMQ X Mgmt API | `[]` -`ingress.dashboard.annotations` | Ingress annotations for EMQ X Mgmt API | `{}` -`ingress.mgmt.enabled` | Enable ingress for EMQ X Mgmt API | `false` -`ingress.mqtt.ingressClassName` | Set the ingress class for EMQ X Mgmt API | `nil` -`ingress.mgmt.path` | Ingress path for EMQ X Mgmt API | `/` -`ingress.mgmt.hosts` | Ingress hosts for EMQ X Mgmt API | `api.emqx.local` -`ingress.mgmt.tls` | Ingress tls for EMQ X Mgmt API | `[]` -`ingress.mgmt.annotations` | Ingress annotations for EMQ X Mgmt API | `{}` -`ingress.wss.enabled` | Enable ingress for EMQ X Mgmt API | `false` -`ingress.wss.ingressClassName` | Set the ingress class for EMQ X Mgmt API | `nil` -`ingress.wss.path` | Ingress path for EMQ X WSS | `/` -`ingress.wss.hosts` | Ingress hosts for EMQ X WSS | `wss.emqx.local` -`ingress.wss.tls` | Ingress tls for EMQ X WSS | `[]` -`ingress.wss.annotations` | Ingress annotations for EMQ X WSS | `{}` +`ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false +`ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard +`ingress.dashboard.path` | Ingress path for EMQX Dashboard | `/` +`ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local +`ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | `[]` +`ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | `{}` +`ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | `false` +`ingress.mqtt.ingressClassName` | Set the ingress class for EMQX Mgmt API | `nil` +`ingress.mgmt.path` | Ingress path for EMQX Mgmt API | `/` +`ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | `api.emqx.local` +`ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | `[]` +`ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | `{}` +`ingress.wss.enabled` | Enable ingress for EMQX Mgmt API | `false` +`ingress.wss.ingressClassName` | Set the ingress class for EMQX Mgmt API | `nil` +`ingress.wss.path` | Ingress path for EMQX WSS | `/` +`ingress.wss.hosts` | Ingress hosts for EMQX WSS | `wss.emqx.local` +`ingress.wss.tls` | Ingress tls for EMQX WSS | `[]` +`ingress.wss.annotations` | Ingress annotations for EMQX WSS | `{}` `extraEnv` | Aditional container env vars | `[]` `extraEnvFrom` | Aditional container env from vars (eg. [config map](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/), [secrets](https://kubernetes.io/docs/concepts/configuration/secret/) | `[]` `extraArgs` | Additional container executable arguments | `[]` `extraVolumes` | Additional container volumes (eg. for mounting certs from secrets) | `[]` `extraVolumeMounts` | Additional container volume mounts (eg. for mounting certs from secrets) | `[]` -## EMQ X specific settings -The following table lists the configurable [EMQ X](https://www.emqx.io/)-specific parameters of the chart and their default values. +## EMQX specific settings +The following table lists the configurable [EMQX](https://www.emqx.io/)-specific parameters of the chart and their default values. Parameter | Description | Default Value --- | --- | --- `emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix can be omitted) or using the configuration files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil` @@ -104,7 +104,7 @@ Parameter | Description | Default Value # Examples This section provides some examples for the configuration of common scenarios. ## Enable Websockets SSL via [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/) -The following settings describe a working scenario for acessing [EMQ X](https://www.emqx.io/) Websockets with SSL termination at the [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/). +The following settings describe a working scenario for acessing [EMQX](https://www.emqx.io/) Websockets with SSL termination at the [nginx-ingress community controller](https://kubernetes.github.io/ingress-nginx/). ```yaml ingress: wss: diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 51692bc07..35eb68e82 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -17,11 +17,11 @@ [the latest release](https://github.com/docker/docker-ce/releases/latest) -# What is EMQ X +# What is EMQX -[EMQ X MQTT broker](https://emqx.io/products/broker) is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients. +[EMQX MQTT broker](https://emqx.io/products/broker) is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients. -Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scaled to 10+ million concurrent MQTT connections on one cluster. +Starting from 3.0 release, *EMQX* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQX* broker can scaled to 10+ million concurrent MQTT connections on one cluster. # How to use this image @@ -39,7 +39,7 @@ The emqx broker runs as linux user `emqx` in the docker container. ### Configuration -Use the environment variable to configure the EMQ X docker container. +Use the environment variable to configure the EMQX docker container. By default, the environment variables with ``EMQX_`` prefix are mapped to key-value pairs in configuration files. @@ -72,9 +72,9 @@ EMQX_HOST These environment variables will ignore for configuration file. -#### EMQ X Configuration +#### EMQX Configuration -> NOTE: All EMQ X Configuration in [etc/emqx.conf](https://github.com/emqx/emqx/blob/main-v4.3/etc/emqx.conf) could config by environment. The following list is just an example, not a complete configuration. +> NOTE: All EMQX Configuration in [etc/emqx.conf](https://github.com/emqx/emqx/blob/main-v4.3/etc/emqx.conf) could config by environment. The following list is just an example, not a complete configuration. | Options | Default | Mapped | Description | | ---------------------------| ------------------ | ------------------------- | ------------------------------------- | @@ -148,7 +148,7 @@ EMQX_LOADED_PLUGINS="emqx_auth_redis emqx_auth_mysql" EMQX_LOADED_PLUGINS="emqx_auth_redis | emqx_auth_mysql" ``` -#### EMQ X Plugins Configuration +#### EMQX Plugins Configuration The environment variables which with ``EMQX_`` prefix are mapped to all emqx plugins' configuration file, ``.`` get replaced by ``__``. @@ -200,7 +200,7 @@ docker run -d --name emqx -p 18083:18083 -p 1883:1883 -p 4369:4369 \ ### Cluster -EMQ X supports a variety of clustering methods, see our [documentation](https://docs.emqx.io/broker/latest/en/advanced/cluster.html#emqx-service-discovery) for details. +EMQX supports a variety of clustering methods, see our [documentation](https://docs.emqx.io/broker/latest/en/advanced/cluster.html#emqx-service-discovery) for details. Let's create a static node list cluster from docker-compose. @@ -256,7 +256,7 @@ Let's create a static node list cluster from docker-compose. ### Persistence -If you want to persist the EMQ X docker container, you need to keep the following directories: +If you want to persist the EMQX docker container, you need to keep the following directories: + `/opt/emqx/data` + `/opt/emqx/etc` @@ -316,7 +316,7 @@ docker run -d --name emqx -p 18083:18083 -p 1883:1883 -p 4369:4369 \ ``` -> REMEMBER: DO NOT RUN EMQ X DOCKER PRIVILEGED OR MOUNT SYSTEM PROC IN CONTAINER TO TUNE LINUX KERNEL, IT IS UNSAFE. +> REMEMBER: DO NOT RUN EMQX DOCKER PRIVILEGED OR MOUNT SYSTEM PROC IN CONTAINER TO TUNE LINUX KERNEL, IT IS UNSAFE. ### Thanks diff --git a/lib-ce/emqx_dashboard/README.md b/lib-ce/emqx_dashboard/README.md index e9e50a7c9..16d279602 100644 --- a/lib-ce/emqx_dashboard/README.md +++ b/lib-ce/emqx_dashboard/README.md @@ -2,7 +2,7 @@ emqx-dashboard ============== -Dashboard for the EMQ X Broker. +Dashboard for the EMQX Broker. REST API -------- diff --git a/lib-extra/README.md b/lib-extra/README.md index 4127b53bc..56a60cffb 100644 --- a/lib-extra/README.md +++ b/lib-extra/README.md @@ -1,4 +1,4 @@ -# EMQ X Extra plugin apps +# EMQX Extra plugin apps This directory keeps a `plugins` file which defines all the approved external plugins from open-source community. @@ -20,7 +20,7 @@ e.g. For an Erlang plugin named `plugin_foo`: ``` Note: The `-emqx_plugin(?MODULE)` attribute should be added to -`_app.erl` file to indicate that this is an EMQ X Broker plugin. +`_app.erl` file to indicate that this is an EMQX Broker plugin. For example: ```erlang From a5f6e944671172aac2843fe4b45e2a9e9178cb97 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 15 Feb 2022 22:12:46 +0100 Subject: [PATCH 05/47] docs: EMQ X -> EMQX --- CHANGES-4.3.md | 10 +++++----- Windows.md | 16 ++++++++-------- apps/emqx_exhook/docs/design-cn.md | 4 ++-- apps/emqx_exproto/docs/design-cn.md | 8 ++++---- apps/emqx_rule_engine/docs/design.md | 6 +++--- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 983180251..ed3050058 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -1,6 +1,6 @@ -# EMQ X 4.3 Changes +# EMQX 4.3 Changes -Started tracking changes in CHANGE.md since EMQ X v4.3.11 +Started tracking changes in CHANGE.md since EMQX v4.3.11 NOTE: Keep prepending to the head of the file instead of the tail @@ -53,9 +53,9 @@ Important notes: ### Important changes -* Debian/Ubuntu package (deb) installed EMQ X now runs on systemd [#6389]
+* Debian/Ubuntu package (deb) installed EMQX now runs on systemd [#6389]
This is to take advantage of systemd's supervision functionality to ensure - EMQ X service is restarted after crashes. + EMQX service is restarted after crashes. ### Minor changes @@ -71,7 +71,7 @@ Important notes: * Fix session takeover race condition which may lead to message loss [#6396] -* EMQ X docker images are pushed to aws public ecr in an automated CI job [#6271]
+* EMQX docker images are pushed to aws public ecr in an automated CI job [#6271]
`docker pull public.ecr.aws/emqx/emqx:4.3.10` * Fix webhook URL path to allow rule-engine variable substitution [#6399] diff --git a/Windows.md b/Windows.md index f02b40920..6b6eff60a 100644 --- a/Windows.md +++ b/Windows.md @@ -1,4 +1,4 @@ -# Build and run EMQ X on Windows +# Build and run EMQX on Windows NOTE: The instructions and examples are based on Windows 10. @@ -6,7 +6,7 @@ NOTE: The instructions and examples are based on Windows 10. ### Visual studio for C/C++ compile and link -EMQ X includes Erlang NIF (Native Implmented Function) components, implemented +EMQX includes Erlang NIF (Native Implmented Function) components, implemented in C/C++. To compile and link C/C++ libraries, the easiest way is perhaps to install Visual Studio. @@ -51,7 +51,7 @@ Eshell V11.1.4 (abort with ^G) ### bash -All EMQ X build/run scripts are either in `bash` or `escript`. +All EMQX build/run scripts are either in `bash` or `escript`. `escript` is installed as a part of Erlang. To install a `bash` environment in Windows, there are quite a few options. @@ -68,7 +68,7 @@ Cygwin is what we tested with. ### Other tools -Some of the unix world tools are required to build EMQ X. Including: +Some of the unix world tools are required to build EMQX. Including: * git * curl @@ -84,7 +84,7 @@ When using scoop: scoop install git curl make jq zip unzip ``` -## Build EMQ X source code +## Build EMQX source code * Clone the repo: `git clone https://github.com/emqx/emqx.git` @@ -112,11 +112,11 @@ scoop install git curl make jq zip unzip To fix it, Visual Studio's bin paths should be ordered prior to Cygwin's (or similar installation's) bin paths in `Path` environment variable. -## Run EMQ X +## Run EMQX -To start EMQ X broker. +To start EMQX broker. -Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQ X. +Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQX. Then execute `_build\emqx\rel\emqx>.\bin\emqx_ctl status` to check status. If everything works fine, it should print out diff --git a/apps/emqx_exhook/docs/design-cn.md b/apps/emqx_exhook/docs/design-cn.md index 6686e96e3..21ee333eb 100644 --- a/apps/emqx_exhook/docs/design-cn.md +++ b/apps/emqx_exhook/docs/design-cn.md @@ -2,7 +2,7 @@ ## 动机 -在 EMQ X Broker v4.1-v4.2 中，我们发布了 2 个插件来扩展 emqx 的编程能力： +在 EMQX Broker v4.1-v4.2 中，我们发布了 2 个插件来扩展 emqx 的编程能力： 1. `emqx-extension-hook` 提供了使用 Java, Python 向 Broker 挂载钩子的功能 2. `emqx-exproto` 提供了使用 Java，Python 编写用户自定义协议接入插件的功能 @@ -26,7 +26,7 @@ 架构如下： ``` - EMQ X + EMQX +========================+ +========+==========+ | ExHook | | | | | +----------------+ | gRPC | gRPC | User's | diff --git a/apps/emqx_exproto/docs/design-cn.md b/apps/emqx_exproto/docs/design-cn.md index 7af7dbdb3..1e1ba9e31 100644 --- a/apps/emqx_exproto/docs/design-cn.md +++ b/apps/emqx_exproto/docs/design-cn.md @@ -2,7 +2,7 @@ `emqx-exproto` 插件用于协议解析的多语言支持。它能够允许其他编程语言（例如：Python，Java 等）直接处理数据流实现协议的解析，并提供 Pub/Sub 接口以实现与系统其它组件的通信。 -该插件给 EMQ X 带来的扩展性十分的强大，它能以你熟悉语言处理任何的私有协议，并享受由 EMQ X 系统带来的高连接，和高并发的优点。 +该插件给 EMQX 带来的扩展性十分的强大，它能以你熟悉语言处理任何的私有协议，并享受由 EMQX 系统带来的高连接，和高并发的优点。 ## 特性 @@ -25,12 +25,12 @@ - 提供 `Send` 接口。供外部模块调用，**用于发送数据包**。 - 提供 `Close` 接口。供外部模块调用，**用于主动关闭连接**。 -2. **协议/会话层：**该部分主要**提供 PUB/SUB 接口**，以实现与 EMQ X Broker 系统的消息互通。包括： +2. **协议/会话层：**该部分主要**提供 PUB/SUB 接口**，以实现与 EMQX Broker 系统的消息互通。包括： - 提供 `Authenticate` 接口。供外部模块调用，用于向集群注册客户端。 - 提供 `StartTimer` 接口。供外部模块调用，用于为该连接进程启动心跳等定时器。 - - 提供 `Publish` 接口。供外部模块调用，用于发布消息 EMQ X Broker 中。 - - 提供 `Subscribe` 接口。供外部模块调用，用于订阅某主题，以实现从 EMQ X Broker 中接收某些下行消息。 + - 提供 `Publish` 接口。供外部模块调用，用于发布消息 EMQX Broker 中。 + - 提供 `Subscribe` 接口。供外部模块调用，用于订阅某主题，以实现从 EMQX Broker 中接收某些下行消息。 - 提供 `Unsubscribe` 接口。供外部模块调用，用于取消订阅某主题。 - 调用 `OnTimerTimeout` 回调。用于处理定时器超时的事件。 - 调用 `OnReceivedMessages` 回调。用于接收下行消息（在订阅主题成功后，如果主题上有消息，便会回调该方法） diff --git a/apps/emqx_rule_engine/docs/design.md b/apps/emqx_rule_engine/docs/design.md index 3e2c60c41..f8cac0127 100644 --- a/apps/emqx_rule_engine/docs/design.md +++ b/apps/emqx_rule_engine/docs/design.md @@ -1,7 +1,7 @@ -# EMQ X Rule Engine +# EMQX Rule Engine -This is the design guide of message routing rule engine for the EMQ X Broker. +This is the design guide of message routing rule engine for the EMQX Broker. ## Concept @@ -149,7 +149,7 @@ The properties and behaviors of resources is defined by resource types. A resour ### Resource Type Provider -Provider of resource type is a EMQ X Plugin. +Provider of resource type is a EMQX Plugin. ### Resource Manager From d2840a8aa1170364a89cbd158f7e5946a3254563 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 15 Feb 2022 22:18:25 +0100 Subject: [PATCH 06/47] docs: change EMQ X to EMQX in more descriptive info --- NOTICE | 2 +- bin/emqx | 2 +- bin/node_dump | 2 +- deploy/charts/emqx/Chart.yaml | 2 +- deploy/charts/emqx/values.yaml | 12 ++++++------ deploy/docker/Dockerfile | 2 +- deploy/docker/docker-entrypoint.sh | 2 +- etc/emqx_cloud/vm.args | 2 +- etc/emqx_edge/vm.args | 2 +- scripts/relup-base-vsns.sh | 2 +- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/NOTICE b/NOTICE index e891b5745..2f70ed114 100644 --- a/NOTICE +++ b/NOTICE @@ -1,4 +1,4 @@ -EMQ X, a highly scalable, highly available distributed MQTT messaging broker for IoT. +EMQX, a highly scalable, highly available distributed MQTT messaging broker for IoT. Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. This product contains code developed at EMQ Technologies Co., Ltd. diff --git a/bin/emqx b/bin/emqx index 40cb7701d..a2a488474 100755 --- a/bin/emqx +++ b/bin/emqx @@ -44,7 +44,7 @@ if ! check_eralng_start >/dev/null 2>&1; then echoerr "FATAL: Unable to start Erlang." echoerr "Please make sure openssl-1.1.1 (libcrypto) and libncurses are installed." echoerr "Also ensure it's running on the correct platform," - echoerr "this EMQ X release is built for $BUILT_ON" + echoerr "this EMQX release is built for $BUILT_ON" exit 1 fi echoerr "WARNING: There seem to be missing dynamic libs from the OS. Using libs from ${DYNLIBS_DIR}" diff --git a/bin/node_dump b/bin/node_dump index c8f76841b..83b41332b 100755 --- a/bin/node_dump +++ b/bin/node_dump @@ -24,7 +24,7 @@ collect() { } show_help() { - echo "Collect information about the EMQ X node + echo "Collect information about the EMQX node USAGE: diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 0bafd6780..3f24cc699 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: emqx -description: A Helm chart for EMQ X +description: A Helm chart for EMQX # A chart can be either an 'application' or a 'library' chart. # # Application charts are a collection of templates that can be packaged into versioned archives diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index f630f3d25..d78f19a8e 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -62,13 +62,13 @@ resources: {} # cpu: 500m # memory: 512Mi -# Containers that run before the creation of EMQ X containers. They can contain utilities or setup scripts. +# Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts. initContainers: {} # - name: mysql-probe # image: alpine # command: ["sh", "-c", "for i in $(seq 1 300); do nc -zvw1 mysql 3306 && exit 0 || sleep 3; done; exit 1"] -## EMQ X configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) +## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) emqxConfig: EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443" ## The address type is used to extract host from k8s service. @@ -119,7 +119,7 @@ emqxLoadedModules: > {emqx_mod_subscription, false}. {emqx_mod_topic_metrics, false}. -## EMQ X Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicenseSecretName" +## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicenseSecretName" ## Example: ## kubectl create secret generic emqx-license-secret-name --from-file=/path/to/emqx.lic emqxLicenseSecretName: @@ -188,7 +188,7 @@ ingress: ## Ingress shared annotations annotations: {} - ## ingress for EMQ X Dashboard + ## ingress for EMQX Dashboard dashboard: enabled: false annotations: {} @@ -199,7 +199,7 @@ ingress: - dashboard.emqx.local tls: [] - ## ingress for EMQ X Mgmt API + ## ingress for EMQX Mgmt API mgmt: enabled: false annotations: {} @@ -210,7 +210,7 @@ ingress: - api.emqx.local tls: [] - ## ingress for EMQ X Mgmt API + ## ingress for EMQX Mgmt API wss: enabled: false # ingressClassName: nginx diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index c79508ec1..8e6e7c84a 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -38,7 +38,7 @@ LABEL org.label-schema.docker.dockerfile="Dockerfile" \ org.label-schema.url="https://emqx.io" \ org.label-schema.vcs-type="Git" \ org.label-schema.vcs-url="https://github.com/emqx/emqx" \ - maintainer="EMQ X Team " + maintainer="EMQX Team " ARG EMQX_NAME=emqx diff --git a/deploy/docker/docker-entrypoint.sh b/deploy/docker/docker-entrypoint.sh index 0776f957b..9c7fab4dc 100755 --- a/deploy/docker/docker-entrypoint.sh +++ b/deploy/docker/docker-entrypoint.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash ## EMQ docker image start script # Huang Rui -# EMQ X Team +# EMQX Team ## Shell setting if [[ -n "$DEBUG" ]]; then diff --git a/etc/emqx_cloud/vm.args b/etc/emqx_cloud/vm.args index 1e6b0b4cb..a199091c7 100644 --- a/etc/emqx_cloud/vm.args +++ b/etc/emqx_cloud/vm.args @@ -1,5 +1,5 @@ ###################################################################### -## Erlang VM Args for EMQ X Broker +## Erlang VM Args ###################################################################### ## NOTE: diff --git a/etc/emqx_edge/vm.args b/etc/emqx_edge/vm.args index ef9749738..2f61faa2d 100644 --- a/etc/emqx_edge/vm.args +++ b/etc/emqx_edge/vm.args @@ -1,5 +1,5 @@ ###################################################################### -## Erlang VM Args for EMQ X Edge +## Erlang VM Args ###################################################################### ## NOTE: diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 65d755994..0a52a1b61 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -2,7 +2,7 @@ set -euo pipefail ## This script prints the relup upgrade base versions -## for the given EMQ X edition (specified as first arg) +## for the given EMQX edition (specified as first arg) ## ## The second argument is the current release version ## if not provided, it's taken from pkg-vsn.sh From 2acdc64b9d1b802a9caf5cd80f216956b0dbfd1a Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Wed, 16 Feb 2022 14:52:31 +0800 Subject: [PATCH 07/47] style(script): rename variable name --- scripts/relup-base-packages.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 90aefe04f..2999b8e0d 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -11,13 +11,13 @@ fi case $PROFILE in "emqx") - BROKER="broker" + DIR="broker" ;; "emqx-ee") - BROKER="enterprise" + DIR="enterprise" ;; "emqx-edge") - BROKER="edge" + DIR="edge" ;; esac @@ -49,10 +49,10 @@ pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh community | xargs echo -n); do if [ ! -f "$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" ] \ - && [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://www.emqx.com/downloads/$BROKER/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then - wget --no-verbose https://www.emqx.com/downloads/$BROKER/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip + && [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://www.emqx.com/downloads/$DIR/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then + wget --no-verbose https://www.emqx.com/downloads/$DIR/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip if [ "$SYSTEM" != "centos6" ]; then - wget --no-verbose https://www.emqx.com/downloads/$BROKER/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 + wget --no-verbose https://www.emqx.com/downloads/$DIR/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 ## https://askubuntu.com/questions/1202208/checking-sha256-checksum echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | $SHASUM -c || exit 1 fi From f7135433be1f53e57ce276c04204c93820c2a378 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 16 Feb 2022 14:10:43 +0100 Subject: [PATCH 08/47] chore(relup-base-vsns.sh): rename PROFILE to EDITION to avoid confusion with rebar profile --- scripts/relup-base-vsns.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/relup-base-vsns.sh b/scripts/relup-base-vsns.sh index 3a2062387..8f391b01b 100755 --- a/scripts/relup-base-vsns.sh +++ b/scripts/relup-base-vsns.sh @@ -20,8 +20,8 @@ parse_semver() { echo "$1" | tr '.|-' ' ' } -PROFILE="${1:-}" -[ -z "${PROFILE}" ] && usage +EDITION="${1:-}" +[ -z "${EDITION}" ] && usage ## Get the current release version ## e.g. @@ -49,7 +49,7 @@ else IS_RELEASE=false fi -case "${PROFILE}" in +case "${EDITION}" in *enterprise*) GIT_TAG_PREFIX="e" ;; From 538f843f149bd8c6ad26cb1f99f4fe3a6ec4be8d Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 16 Feb 2022 14:13:57 +0100 Subject: [PATCH 09/47] fix(relup-base-packages.sh): download per release edition --- scripts/relup-base-packages.sh | 34 +++++++++++++++++++++------------- scripts/shellcheck.sh | 6 +++++- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 2999b8e0d..17f0d6ab4 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash + +## This script helps to download relup base version packages + if [[ -n "$DEBUG" ]]; then set -x fi @@ -6,24 +9,27 @@ set -euo pipefail PROFILE="${1}" if [ "$PROFILE" = "" ]; then - $PROFILE="emqx" + PROFILE="emqx" fi case $PROFILE in "emqx") - DIR="broker" + DIR='broker' + EDITION='community' ;; "emqx-ee") - DIR="enterprise" + DIR='enterprise' + EDITION='enterprise' ;; "emqx-edge") - DIR="edge" + DIR='edge' + EDITION='edge' ;; esac -SYSTEM="$(./scripts/get-distro.sh)" +SYSTEM="${SYSTEM:-$(./scripts/get-distro.sh)}" -ARCH="$(uname -m)" +ARCH="${ARCH:-$(uname -m)}" case "$ARCH" in x86_64) ARCH='amd64' @@ -47,16 +53,18 @@ cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.." mkdir -p _upgrade_base pushd _upgrade_base -for tag in $(../scripts/relup-base-vsns.sh community | xargs echo -n); do - if [ ! -f "$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" ] \ - && [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://www.emqx.com/downloads/$DIR/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then - wget --no-verbose https://www.emqx.com/downloads/$DIR/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip +for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do + filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" + url="https://www.emqx.com/downloads/$DIR/$tag/$filename" + echo "downloading ${filename} ..." + if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then + curl -L -o "${filename}" "${url}" if [ "$SYSTEM" != "centos6" ]; then - wget --no-verbose https://www.emqx.com/downloads/$DIR/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 + curl -L -o "${filename}.sha256" "${url}.sha256" ## https://askubuntu.com/questions/1202208/checking-sha256-checksum - echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | $SHASUM -c || exit 1 + echo "$(cat "${filename}.sha256") ${filename}" | $SHASUM -c || exit 1 fi fi -done +done popd diff --git a/scripts/shellcheck.sh b/scripts/shellcheck.sh index 5f8cdfd51..97445829f 100755 --- a/scripts/shellcheck.sh +++ b/scripts/shellcheck.sh @@ -3,7 +3,11 @@ set -euo pipefail target_files=() -while IFS='' read -r line; do target_files+=("$line"); done < <(grep -r -l --exclude-dir=.git --exclude-dir=_build "#!/bin/" .) +while IFS='' read -r line; +do + target_files+=("$line"); +done < <(git grep -r -l '^#!/\(bin/\|usr/bin/env bash\)' .) + return_code=0 for i in "${target_files[@]}"; do echo checking "$i" ... From 62dd815807ecf1772442bed58f7107ffd573ef3b Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Wed, 16 Feb 2022 14:35:23 +0100 Subject: [PATCH 10/47] chore(scripts): make shellcheck happy --- pkg-vsn.sh | 2 +- scripts/find-props.sh | 1 + scripts/find-suites.sh | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg-vsn.sh b/pkg-vsn.sh index f3bdb1884..10780c467 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -18,7 +18,7 @@ RELEASE="$(grep -E "define.+EMQX_RELEASE.+${EDITION}" include/emqx_release.hrl | git_exact_vsn() { local tag tag="$(git describe --tags --match "[e|v]*" --exact 2>/dev/null)" - echo "$tag" | sed 's/^[v|e]//g' + echo "${tag#[e|v]}" } GIT_EXACT_VSN="$(git_exact_vsn)" diff --git a/scripts/find-props.sh b/scripts/find-props.sh index 582ef26ae..e2d9f7c31 100755 --- a/scripts/find-props.sh +++ b/scripts/find-props.sh @@ -13,4 +13,5 @@ if [ "$1" != "emqx" ]; then BASEDIR="$1" fi +# shellcheck disable=SC2038 find "${BASEDIR}/test/props" -name "prop_*.erl" 2>/dev/null | xargs -I{} basename {} .erl | xargs | tr ' ' ',' diff --git a/scripts/find-suites.sh b/scripts/find-suites.sh index 97939d931..7294a98e6 100755 --- a/scripts/find-suites.sh +++ b/scripts/find-suites.sh @@ -12,4 +12,5 @@ TESTDIR="test" if [ "$1" != "emqx" ]; then TESTDIR="$1/test" fi +# shellcheck disable=SC2038 find "${TESTDIR}" -name "*_SUITE.erl" 2>/dev/null | xargs | tr ' ' ',' From 42c70cd2a50023a75c4c0a9cb8e8494b863d2708 Mon Sep 17 00:00:00 2001 From: DDDHuang <44492639+DDDHuang@users.noreply.github.com> Date: Thu, 17 Feb 2022 14:06:17 +0800 Subject: [PATCH 11/47] fix(doc): code spelling --- apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src | 2 +- apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src | 12 +++++++++++- apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index e6d1c55c4..339c69eba 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mqtt, [{description, "EMQ X Bridge to MQTT Broker"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.3.4"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,replayq,emqtt]}, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src index 3b50949cd..52b4e24fc 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.appup.src @@ -1,21 +1,31 @@ %% -*-: erlang -*- -{"4.3.3", +{"4.3.4", [ + {<<"4.3.3">>, [ + {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []} + ]}, {<<"4.3.[1-2]">>, [ + {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}, {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ + {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}, {load_module, emqx_bridge_worker, brutal_purge, soft_purge, []}, {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ + {<<"4.3.3">>, [ + {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []} + ]}, {<<"4.3.[1-2]">>, [ + {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}, {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} ]}, {"4.3.0", [ + {load_module, emqx_bridge_mqtt, brutal_purge, soft_purge, []}, {load_module, emqx_bridge_worker, brutal_purge, soft_purge, []}, {load_module, emqx_bridge_mqtt_actions, brutal_purge, soft_purge, []} ]}, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl index d612af668..3bd12564c 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl @@ -144,7 +144,7 @@ send(#{client_pid := ClientPid} = Conn, [Msg | Rest], PktIds) -> {ok, PktId} -> send(Conn, Rest, [PktId | PktIds]); {error, Reason} -> - %% NOTE: There is no partial sucess of a batch and recover from the middle + %% NOTE: There is no partial success of a batch and recover from the middle %% only to retry all messages in one batch {error, Reason} end. @@ -154,7 +154,7 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent) RC =:= ?RC_NO_MATCHING_SUBSCRIBERS -> Parent ! {batch_ack, PktId}, ok; handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) -> - ?LOG(warning, "Publish ~p to remote node falied, reason_code: ~p", [PktId, RC]). + ?LOG(warning, "Publish ~p to remote node failed, reason_code: ~p", [PktId, RC]). handle_publish(Msg, Mountpoint) -> emqx_broker:publish(emqx_bridge_msg:to_broker_msg(Msg, Mountpoint)). From 3690d1951ee5ccf79445f37f25a47805af24ff65 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 18 Feb 2022 22:41:03 +0800 Subject: [PATCH 12/47] chore: add more debug info for relup-base-packages.sh --- scripts/relup-base-packages.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index 17f0d6ab4..bd3e99780 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -56,13 +56,15 @@ pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" url="https://www.emqx.com/downloads/$DIR/$tag/$filename" - echo "downloading ${filename} ..." + echo "downloading base package from ${url} ..." if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then curl -L -o "${filename}" "${url}" if [ "$SYSTEM" != "centos6" ]; then curl -L -o "${filename}.sha256" "${url}.sha256" + SUMSTR=$(cat "${filename}.sha256") + echo "got sha265sum: ${SUMSTR}" ## https://askubuntu.com/questions/1202208/checking-sha256-checksum - echo "$(cat "${filename}.sha256") ${filename}" | $SHASUM -c || exit 1 + echo "${SUMSTR} ${filename}" | $SHASUM -c || exit 1 fi fi done From 3bd961ca732a2164b92b5c367c8c0e5183b3c1ed Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Fri, 18 Feb 2022 23:05:35 +0800 Subject: [PATCH 13/47] fix(CI): remove e|v from downloading urls of emqx packages --- scripts/relup-base-packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/relup-base-packages.sh b/scripts/relup-base-packages.sh index bd3e99780..bedc050c7 100755 --- a/scripts/relup-base-packages.sh +++ b/scripts/relup-base-packages.sh @@ -55,7 +55,7 @@ pushd _upgrade_base for tag in $(../scripts/relup-base-vsns.sh $EDITION | xargs echo -n); do filename="$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" - url="https://www.emqx.com/downloads/$DIR/$tag/$filename" + url="https://www.emqx.com/downloads/$DIR/${tag#[e|v]}/$filename" echo "downloading base package from ${url} ..." if [ ! -f "$filename" ] && curl -I -m 10 -o /dev/null -s -w "%{http_code}" "${url}" | grep -q -oE "^[23]+" ; then curl -L -o "${filename}" "${url}" From 557c52dd8a97dcfd7a083d4cbe440d98ca343a3f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 18 Feb 2022 20:35:11 +0300 Subject: [PATCH 14/47] chore(codestyle): remove Elvis --- .github/workflows/code_style_check.yaml | 24 ----- .../src/emqx_management.app.src | 2 +- apps/emqx_management/src/emqx_mgmt.erl | 2 - apps/emqx_stomp/src/emqx_stomp.app.src | 2 +- apps/emqx_stomp/src/emqx_stomp.appup.src | 25 +++-- apps/emqx_stomp/src/emqx_stomp_connection.erl | 2 - apps/emqx_stomp/src/emqx_stomp_protocol.erl | 2 - elvis.config | 47 ---------- scripts/elvis-check.sh | 84 ----------------- src/emqx.appup.src | 92 +++++++++++++------ src/emqx_banned.erl | 2 - src/emqx_connection.erl | 2 - src/emqx_ctl.erl | 2 - src/emqx_logger_jsonfmt.erl | 2 - src/emqx_message.erl | 2 - 15 files changed, 85 insertions(+), 207 deletions(-) delete mode 100644 .github/workflows/code_style_check.yaml delete mode 100644 elvis.config delete mode 100755 scripts/elvis-check.sh diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml deleted file mode 100644 index 959512c20..000000000 --- a/.github/workflows/code_style_check.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: Code style check - -on: [pull_request] - -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 1000 - - name: Set git token - if: endsWith(github.repository, 'enterprise') - run: | - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - - name: Run elvis check - run: | - set -e - if [ -f EMQX_ENTERPRISE ]; then - ./scripts/elvis-check.sh $GITHUB_BASE_REF emqx-enterprise - else - ./scripts/elvis-check.sh $GITHUB_BASE_REF emqx - fi diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index bee65781a..64f44637a 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -1,6 +1,6 @@ {application, emqx_management, [{description, "EMQ X Management API and CLI"}, - {vsn, "4.3.11"}, % strict semver, bump manually! + {vsn, "4.3.12"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel,stdlib,minirest]}, diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 2a9636b6b..bc271ee20 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -123,8 +123,6 @@ -define(MAX_ROW_LIMIT, 10000). -define(APP, emqx_management). - --elvis([{elvis_style, god_modules, disable}]). %%-------------------------------------------------------------------- %% Node Info %%-------------------------------------------------------------------- diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src index a70826e34..5883cbca5 100644 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ b/apps/emqx_stomp/src/emqx_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_stomp, [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.3.4"}, % strict semver, bump manually! + {vsn, "4.3.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_stomp_sup]}, {applications, [kernel,stdlib]}, diff --git a/apps/emqx_stomp/src/emqx_stomp.appup.src b/apps/emqx_stomp/src/emqx_stomp.appup.src index 8d61073d0..3493ade03 100644 --- a/apps/emqx_stomp/src/emqx_stomp.appup.src +++ b/apps/emqx_stomp/src/emqx_stomp.appup.src @@ -1,8 +1,15 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.3",[{load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]}, - {"4.3.2", + [{"4.3.4", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + {"4.3.3", + [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]}, + {"4.3.2", + [{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]}, {"4.3.1", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, @@ -12,14 +19,20 @@ [{restart_application,emqx_stomp}, {apply,{emqx_stomp,force_clear_after_app_stoped,[]}}]}, {<<".*">>,[]}], - [{"4.3.3",[{load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]}, - {"4.3.2", + [{"4.3.4", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, + {"4.3.3", + [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]}, + {"4.3.2", + [{load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}]}, {"4.3.1", [{load_module,emqx_stomp_protocol,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_stomp_connection,brutal_purge,soft_purge,[]}]}, - {"4.3.0", - [{restart_application,emqx_stomp}]}, + {"4.3.0",[{restart_application,emqx_stomp}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_stomp/src/emqx_stomp_connection.erl b/apps/emqx_stomp/src/emqx_stomp_connection.erl index 19ae6bc26..4e5793dad 100644 --- a/apps/emqx_stomp/src/emqx_stomp_connection.erl +++ b/apps/emqx_stomp/src/emqx_stomp_connection.erl @@ -91,8 +91,6 @@ -define(ENABLED(X), (X =/= undefined)). --elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_stomp_connection]}}]). - -dialyzer({nowarn_function, [ ensure_stats_timer/2 ]}). diff --git a/apps/emqx_stomp/src/emqx_stomp_protocol.erl b/apps/emqx_stomp/src/emqx_stomp_protocol.erl index 4e371d9b0..6049b88f3 100644 --- a/apps/emqx_stomp/src/emqx_stomp_protocol.erl +++ b/apps/emqx_stomp/src/emqx_stomp_protocol.erl @@ -108,8 +108,6 @@ , init/2 ]}). --elvis([{elvis_style, dont_repeat_yourself, disable}]). - -type(pstate() :: #pstate{}). %% @doc Init protocol diff --git a/elvis.config b/elvis.config deleted file mode 100644 index ae47fc8f9..000000000 --- a/elvis.config +++ /dev/null @@ -1,47 +0,0 @@ -%% -*-: erlang -*- -[ - { - elvis, - [ - {config, - [ - #{dirs => ["src", "apps/**/src", "lib-ce/**/src", "lib-ee/**/src"], - filter => "*.erl", - ruleset => erl_files, - rules => [ - {elvis_style, state_record_and_type, disable}, - {elvis_style, no_common_caveats_call, #{}}, - {elvis_style, no_debug_call, #{ debug_functions => [ {ct, pal} - , {ct, print} - ]}}, - {elvis_style, operator_spaces, #{rules => [{right, "|"}, - {left, "|"}, - {right, "||"}, - {left, "||"}]}} - ] - }, - #{dirs => ["test", "apps/**/test", "lib-ce/**/src"], - filter => "*.erl", - rules => [ - {elvis_text_style, line_length, #{ limit => 100 - , skip_comments => false }}, - {elvis_style, dont_repeat_yourself, #{ min_complexity => 100 }} - ] - }, - #{dirs => ["."], - filter => "Makefile", - ruleset => makefiles - }, - #{dirs => ["."], - filter => "rebar.config", - ruleset => rebar_config - }, - #{dirs => ["."], - filter => "elvis.config", - ruleset => elvis_config - } - ] - } - ] - } -]. diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh deleted file mode 100755 index 264200d6b..000000000 --- a/scripts/elvis-check.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -## This script checks style of changed files. -## Expect argument 1 to be the git compare base. - -set -euo pipefail - -elvis_version='1.0.0-emqx-2' - -base="${1:-}" -repo="${2:-emqx/emqx}" -REPO="${GITHUB_REPOSITORY:-${repo}}" -if [ "${base}" = "" ]; then - echo "Usage $0 " - exit 1 -fi - -echo "elvis -v: $elvis_version" -echo "git diff base: $base" - -if [ ! -f ./elvis ] || [ "$(./elvis -v | grep -oE '[1-9]+\.[0-9]+\.[0-9]+\-emqx-[0-9]+')" != "$elvis_version" ]; then - curl --silent --show-error -fLO "https://github.com/emqx/elvis/releases/download/$elvis_version/elvis" - chmod +x ./elvis -fi - -if [[ "$base" =~ [0-9a-f]{8,40} ]]; then - # base is a commit sha1 - compare_base="$base" -else - remote="$(git remote -v | grep -E "github\.com(:|/)$REPO((\.git)|(\s))" | grep fetch | awk '{print $1}')" - git fetch "$remote" "$base" - compare_base="$remote/$base" -fi - -git_diff() { - git diff --name-only --diff-filter=ACMRTUXB "$compare_base"...HEAD -} - -bad_file_count=0 -for file in $(git_diff); do - if [ ! -f "$file" ]; then - # file is deleted, skip - continue - fi - if [[ $file != *.erl ]]; then - # not .erl file - continue - fi - if ! ./elvis rock "$file" -c elvis.config; then - bad_file_count=$(( bad_file_count + 1)) - fi -done -if [ $bad_file_count -gt 0 ]; then - echo "elvis: $bad_file_count errors" - exit 1 -fi - -### now check new-line at EOF for changed files - -nl_at_eof() { - local file="$1" - if ! [ -f "$file" ]; then - return - fi - case "$file" in - *.png|*rebar3) - return - ;; - esac - local lastbyte - lastbyte="$(tail -c 1 "$file" 2>&1)" - if [ "$lastbyte" != '' ]; then - echo "$file" - return 1 - fi -} - -for file in $(git_diff); do - if ! nl_at_eof "$file"; then - bad_file_count=$(( bad_file_count + 1 )) - fi -done - -exit $bad_file_count diff --git a/src/emqx.appup.src b/src/emqx.appup.src index cd5b02954..a133d37a6 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,8 +1,14 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.13",[{load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + [{"4.3.13", + [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -16,7 +22,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -35,7 +42,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -54,7 +62,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -78,7 +87,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -102,7 +112,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -128,7 +139,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -154,7 +166,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -181,7 +194,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -209,7 +223,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -238,7 +253,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -267,7 +283,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -300,7 +317,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, @@ -336,9 +354,15 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.13",[{load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, + [{"4.3.13", + [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -351,7 +375,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -369,7 +394,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, @@ -387,7 +413,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -410,7 +437,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -433,7 +461,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -458,7 +487,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, @@ -483,7 +513,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, @@ -509,7 +540,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, @@ -536,7 +568,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, @@ -564,7 +597,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, @@ -592,7 +626,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pqueue,brutal_purge,soft_purge,[]}, @@ -624,7 +659,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, + [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_banned.erl b/src/emqx_banned.erl index 762a2b61b..af4603078 100644 --- a/src/emqx_banned.erl +++ b/src/emqx_banned.erl @@ -47,8 +47,6 @@ , code_change/3 ]). --elvis([{elvis_style, state_record_and_type, disable}]). - -define(BANNED_TAB, ?MODULE). %%-------------------------------------------------------------------- diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl index ef12f6bcf..18816ccd9 100644 --- a/src/emqx_connection.erl +++ b/src/emqx_connection.erl @@ -30,8 +30,6 @@ -compile(nowarn_export_all). -endif. --elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_connection]}}]). - %% API -export([ start_link/3 , stop/1 diff --git a/src/emqx_ctl.erl b/src/emqx_ctl.erl index 7b73c50fe..400821537 100644 --- a/src/emqx_ctl.erl +++ b/src/emqx_ctl.erl @@ -57,8 +57,6 @@ , code_change/3 ]). --elvis([{elvis_style, invalid_dynamic_call, #{ignore => [emqx_ctl]}}]). - -record(state, {seq = 0}). -type(cmd() :: atom()). diff --git a/src/emqx_logger_jsonfmt.erl b/src/emqx_logger_jsonfmt.erl index 31e4c2fda..b2311935d 100644 --- a/src/emqx_logger_jsonfmt.erl +++ b/src/emqx_logger_jsonfmt.erl @@ -37,8 +37,6 @@ -export_type([config/0]). --elvis([{elvis_style, no_nested_try_catch, #{ ignore => [emqx_logger_jsonfmt]}}]). - -type config() :: #{depth => pos_integer() | unlimited, report_cb => logger:report_cb(), single_line => boolean()}. diff --git a/src/emqx_message.erl b/src/emqx_message.erl index a0ee98434..07ae593c1 100644 --- a/src/emqx_message.erl +++ b/src/emqx_message.erl @@ -84,8 +84,6 @@ -export([format/1]). --elvis([{elvis_style, god_modules, disable}]). - -spec(make(emqx_topic:topic(), emqx_types:payload()) -> emqx_types:message()). make(Topic, Payload) -> make(undefined, Topic, Payload). From 2325e8cfb3965e7cb50f97c692ca33fd85b02203 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 20 Feb 2022 19:57:56 +0100 Subject: [PATCH 15/47] chore: ensure remsh node name prefix for upgrader for remsh prefixed nodes, ekka pick a random port to listen on instead of following the conventional port mapping rule: ekka listens on 4370 for emqx or emqx0 and 4371 for emqx1 and so on --- bin/install_upgrade.escript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 97548cba8..f88af106d 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -365,7 +365,7 @@ start_distribution(TargetNode, NameTypeArg, Cookie) -> make_script_node(Node) -> [Name, Host] = string:tokens(atom_to_list(Node), "@"), - list_to_atom(lists:concat([Name, "_upgrader_", os:getpid(), "@", Host])). + list_to_atom(lists:concat(["remsh_", Name, "_upgrader_", os:getpid(), "@", Host])). %% get name type from arg get_name_type(NameTypeArg) -> From 107e447de46d77910ec1f067fec339b272c66fef Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Sun, 20 Feb 2022 09:19:56 +0100 Subject: [PATCH 16/47] build: fix find command for windows windows has its own find command --- Makefile | 7 +++++-- build | 11 +++++++++-- scripts/find-apps.sh | 11 +++++++++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index ed7c5b6a3..b57543c52 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,9 @@ export EMQX_CE_DASHBOARD_VERSION ?= v4.3.5 export DOCKERFILE := deploy/docker/Dockerfile ifeq ($(OS),Windows_NT) export REBAR_COLOR=none + FIND=/usr/bin/find +else + FIND=find endif PROFILE ?= emqx @@ -90,8 +93,8 @@ $(PROFILES:%=clean-%): @if [ -d _build/$(@:clean-%=%) ]; then \ rm rebar.lock \ rm -rf _build/$(@:clean-%=%)/rel; \ - find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ - find _build/$(@:clean-%=%) -type l -delete; \ + $(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ + $(FIND) _build/$(@:clean-%=%) -type l -delete; \ fi .PHONY: clean-all diff --git a/build b/build index 684057bf1..517efc132 100755 --- a/build +++ b/build @@ -45,6 +45,13 @@ if [ "$(uname -s)" = 'Linux' ]; then esac fi +if [ "${SYSTEM}" = 'windows' ]; then + # windows does not like the find + FIND="/usr/bin/find" +else + FIND='find' +fi + log() { local msg="$1" # rebar3 prints ===>, so we print ===< @@ -76,7 +83,7 @@ make_relup() { rm -rf "$tmp_dir" fi releases+=( "$base_vsn" ) - done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f) fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -96,7 +103,7 @@ cp_dyn_libs() { mkdir -p "$target_dir" while read -r so_file; do cp -L "$so_file" "$target_dir/" - done < <(find "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ + done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \ | xargs -0 ldd \ | grep -E '(libcrypto)|(libtinfo)' \ | awk '{print $3}' \ diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index fabec239e..c4a0eec62 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -5,9 +5,16 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")/.." +if [ "$(./scripts/get-distro.sh)" = 'windows' ]; then + # Otherwise windows may resolve to find.exe + FIND="/usr/bin/find" +else + FIND='find' +fi + find_app() { local appdir="$1" - find "${appdir}" -mindepth 1 -maxdepth 1 -type d + "$FIND" "${appdir}" -mindepth 1 -maxdepth 1 -type d } # append emqx application first @@ -23,4 +30,4 @@ fi ## find directories in lib-extra find_app 'lib-extra' ## find symlinks in lib-extra -find 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print +"$FIND" 'lib-extra' -mindepth 1 -maxdepth 1 -type l -exec test -e {} \; -print From 31b006a0d12f7990547015eb506ae967311c72a7 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 21 Feb 2022 18:40:12 +0800 Subject: [PATCH 17/47] fix(frame): `server_keepalive` only for MQTT v5.0 --- src/emqx_channel.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index dc0c52598..32f0062ea 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1515,11 +1515,13 @@ enrich_connack_caps(AckProps, _Channel) -> AckProps. %%-------------------------------------------------------------------- %% Enrich server keepalive -enrich_server_keepalive(AckProps, #channel{clientinfo = #{zone := Zone}}) -> +enrich_server_keepalive(AckProps, ?IS_MQTT_V5 = #channel{clientinfo = #{zone := Zone}}) -> case emqx_zone:server_keepalive(Zone) of undefined -> AckProps; Keepalive -> AckProps#{'Server-Keep-Alive' => Keepalive} - end. + end; + +enrich_server_keepalive(AckProps, _Channel) -> AckProps. %%-------------------------------------------------------------------- %% Enrich response information @@ -1565,7 +1567,7 @@ init_alias_maximum(#mqtt_packet_connect{proto_ver = ?MQTT_PROTO_V5, init_alias_maximum(_ConnPkt, _ClientInfo) -> undefined. %%-------------------------------------------------------------------- -%% Enrich Keepalive +%% Ensure Keepalive %% MQTT 5 ensure_keepalive(#{'Server-Keep-Alive' := Interval}, Channel = #channel{conninfo = ConnInfo}) -> From b802bcb6fc6ff1772e580c4872683ce7a08eee35 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 28 Jan 2022 12:47:57 +0800 Subject: [PATCH 18/47] refactor(exproto): pre-register the anonymous client --- .../emqx_exproto/src/emqx_exproto_channel.erl | 55 ++++++++++++------- apps/emqx_exproto/src/emqx_exproto_conn.erl | 3 +- .../src/emqx_mgmt_api_clients.erl | 1 + src/emqx_pmon.erl | 6 +- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 44b8b64d3..130ad155a 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -94,9 +94,6 @@ awaiting_rel_max ]). --define(CHANMOCK(P), {exproto_anonymous_client, P}). --define(CHAN_CONN_TAB, emqx_channel_conn). - %%-------------------------------------------------------------------- %% Info, Attrs and Caps %%-------------------------------------------------------------------- @@ -155,15 +152,14 @@ init(ConnInfo = #{socktype := Socktype, Channel = #channel{gcli = #{channel => GRpcChann}, conninfo = NConnInfo, clientinfo = ClientInfo, - conn_state = connecting, + conn_state = accepted, timers = #{} }, case emqx_hooks:run_fold('client.connect', [NConnInfo], #{}) of {error, _Reason} -> throw(nopermission); _ -> - ConnMod = maps:get(conn_mod, NConnInfo), - true = ets:insert(?CHAN_CONN_TAB, {?CHANMOCK(self()), ConnMod}), + ok = register_the_anonymous_client(ClientInfo, NConnInfo), Req = #{conninfo => peercert(Peercert, #{socktype => socktype(Socktype), @@ -172,6 +168,22 @@ init(ConnInfo = #{socktype := Socktype, try_dispatch(on_socket_created, wrap(Req), Channel) end. +register_the_anonymous_client(ClientInfo, ConnInfo) -> + ClientId = maps:get(clientid, ClientInfo), + case emqx_cm:open_session(true, ClientInfo, ConnInfo) of + {ok, _} -> + ?LOG(debug, "Registered an anonymous connection, " + "temporary clientid: ~s", [ClientId]), + emqx_logger:set_metadata_clientid(ClientId), + _ = self() ! {event, accepted}, + ok; + {error, Reason} -> + throw({register_anonymous_error, Reason}) + end. + +unregister_the_anonymous_client(ClientId) -> + emqx_cm:unregister_channel(ClientId). + %% @private peercert(NoSsl, ConnInfo) when NoSsl == nossl; NoSsl == undefined -> @@ -274,15 +286,14 @@ handle_call(close, Channel) -> handle_call({auth, ClientInfo, _Password}, Channel = #channel{conn_state = connected}) -> ?LOG(warning, "Duplicated authorized command, dropped ~p", [ClientInfo]), {reply, {error, ?RESP_PERMISSION_DENY, <<"Duplicated authenticate command">>}, Channel}; -handle_call({auth, ClientInfo0, Password}, +handle_call({auth, RequestedClientInfo, Password}, Channel = #channel{conninfo = ConnInfo, - clientinfo = ClientInfo}) -> - ClientInfo1 = enrich_clientinfo(ClientInfo0, ClientInfo), - NConnInfo = enrich_conninfo(ClientInfo0, ConnInfo), + clientinfo = ClientInfo0}) -> + ClientInfo1 = enrich_clientinfo(RequestedClientInfo, ClientInfo0), + NConnInfo = enrich_conninfo(RequestedClientInfo, ConnInfo), Channel1 = Channel#channel{conninfo = NConnInfo, clientinfo = ClientInfo1}, - #{clientid := ClientId, username := Username} = ClientInfo1, case emqx_access_control:authenticate(ClientInfo1#{password => Password}) of @@ -292,9 +303,10 @@ handle_call({auth, ClientInfo0, Password}, emqx_metrics:inc('client.auth.anonymous'), NClientInfo = maps:merge(ClientInfo1, AuthResult), NChannel = Channel1#channel{clientinfo = NClientInfo}, - clean_anonymous_clients(), case emqx_cm:open_session(true, NClientInfo, NConnInfo) of {ok, _Session} -> + AnonymousClientId = maps:get(clientid, ClientInfo0), + unregister_the_anonymous_client(AnonymousClientId), ?LOG(debug, "Client ~s (Username: '~s') authorized successfully!", [ClientId, Username]), {reply, ok, [{event, connected}], ensure_connected(NChannel)}; @@ -406,16 +418,12 @@ handle_info(Info, Channel) -> -spec(terminate(any(), channel()) -> channel()). terminate(Reason, Channel) -> - clean_anonymous_clients(), Req = #{reason => stringfy(Reason)}, try_dispatch(on_socket_closed, wrap(Req), Channel). is_anonymous(#{anonymous := true}) -> true; is_anonymous(_AuthResult) -> false. -clean_anonymous_clients() -> - ets:delete(?CHAN_CONN_TAB, ?CHANMOCK(self())). - packet_to_message(Topic, Qos, Payload, #channel{ conninfo = #{proto_ver := ProtoVer}, @@ -608,23 +616,32 @@ default_conninfo(ConnInfo) -> username => undefined, conn_props => #{}, connected => true, + proto_name => <<"exproto">>, + proto_ver => <<"1.0">>, connected_at => erlang:system_time(millisecond), keepalive => 0, receive_maximum => 0, expiry_interval => 0}. -default_clientinfo(#{peername := {PeerHost, _}, +default_clientinfo(#{peername := {PeerHost, PeerPort}, sockname := {_, SockPort}}) -> #{zone => external, - protocol => undefined, + protocol => exproto, peerhost => PeerHost, sockport => SockPort, - clientid => undefined, + clientid => anonymous_clientid(PeerHost, PeerPort), username => undefined, is_bridge => false, is_superuser => false, mountpoint => undefined}. +anonymous_clientid(PeerHost, PeerPort) -> + iolist_to_binary( + ["exproto-anonymous-", + inet:ntoa(PeerHost), "-", integer_to_list(PeerPort), + "-", emqx_rule_id:gen() + ]). + stringfy(Reason) -> unicode:characters_to_binary((io_lib:format("~0p", [Reason]))). diff --git a/apps/emqx_exproto/src/emqx_exproto_conn.erl b/apps/emqx_exproto/src/emqx_exproto_conn.erl index 02c0b31d6..f0cca7bec 100644 --- a/apps/emqx_exproto/src/emqx_exproto_conn.erl +++ b/apps/emqx_exproto/src/emqx_exproto_conn.erl @@ -439,7 +439,8 @@ handle_msg({close, Reason}, State) -> ?LOG(debug, "Force to close the socket due to ~p", [Reason]), handle_info({sock_closed, Reason}, close_socket(State)); -handle_msg({event, connected}, State = #state{channel = Channel}) -> +handle_msg({event, Event}, State = #state{channel = Channel}) + when Event == connected; Event == accepted -> ClientId = emqx_exproto_channel:info(clientid, Channel), emqx_cm:insert_channel_info(ClientId, info(State), stats(State)); diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 7d3dbddc8..f47fea6eb 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -271,6 +271,7 @@ format_channel_info({_Key, Info, Stats0}) -> SessCreated = maps:get(created_at, Session, maps:get(connected_at, ConnInfo)), Connected = case maps:get(conn_state, Info, connected) of connected -> true; + accepted -> true; %% for exproto anonymous clients _ -> false end, NStats = Stats#{max_subscriptions => maps:get(subscriptions_max, Stats, 0), diff --git a/src/emqx_pmon.erl b/src/emqx_pmon.erl index 1f9ba7af6..cfa366027 100644 --- a/src/emqx_pmon.erl +++ b/src/emqx_pmon.erl @@ -50,9 +50,11 @@ monitor(Pid, PMon) -> ?MODULE:monitor(Pid, undefined, PMon). -spec(monitor(pid(), term(), pmon()) -> pmon()). -monitor(Pid, Val, PMon = ?PMON(Map)) -> +monitor(Pid, Val, ?PMON(Map)) -> case maps:is_key(Pid, Map) of - true -> PMon; + true -> + {Ref, _Val} = maps:get(Pid, Map), + ?PMON(maps:put(Pid, {Ref, Val}, Map)); false -> Ref = erlang:monitor(process, Pid), ?PMON(maps:put(Pid, {Ref, Val}, Map)) From a2d8405f99ee304efb065aff95cbe16be3729131 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Fri, 11 Feb 2022 09:33:48 +0800 Subject: [PATCH 19/47] chore: update appup.src --- apps/emqx_exproto/src/emqx_exproto.app.src | 2 +- apps/emqx_exproto/src/emqx_exproto.appup.src | 14 ++------- .../src/emqx_management.appup.src | 4 +-- src/emqx.appup.src | 30 ++++++++++++++++++- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 0ed54e6fd..4ce137fc5 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.5"}, %% 4.3.3 is used by ee + {vsn, "4.3.6"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index 7da28e773..590a2106f 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,11 +1,6 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.4", - [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {"4.3.3", - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, - {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {"4.3.2", + [{<<"4\\.3\\.[2-5]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-1]">>, @@ -14,12 +9,7 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.4", - [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {"4.3.3", - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, - {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {"4.3.2", + [{<<"4\\.3\\.[2-5]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-1]">>, diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src index 1463334b4..a91868861 100644 --- a/apps/emqx_management/src/emqx_management.appup.src +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -1,13 +1,13 @@ %% -*- mode: erlang -*- {VSN, - [ {<<"4\\.3\\.[0-9]+">>, + [ {<<"4\\.3\\.([0-9]|1[0])">>, [ {apply,{minirest,stop_http,['http:management']}}, {apply,{minirest,stop_http,['https:management']}}, {restart_application, emqx_management} ]}, {<<".*">>, []} ], - [ {<<"4\\.3\\.[0-9]+">>, + [ {<<"4\\.3\\.([0-9]|1[0])">>, [ {apply,{minirest,stop_http,['http:management']}}, {apply,{minirest,stop_http,['https:management']}}, {restart_application, emqx_management} diff --git a/src/emqx.appup.src b/src/emqx.appup.src index a133d37a6..5300e6bc8 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -4,10 +4,12 @@ [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, @@ -23,6 +25,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, @@ -43,6 +46,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, @@ -63,6 +67,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, @@ -88,6 +93,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, @@ -113,6 +119,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, @@ -139,7 +146,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + [{load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -167,6 +175,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -195,6 +204,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -224,6 +234,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -254,6 +265,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -284,6 +296,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -318,6 +331,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, @@ -357,11 +371,13 @@ [{"4.3.13", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -376,6 +392,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -395,6 +412,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -414,6 +432,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -438,6 +457,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -462,6 +482,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -488,6 +509,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -514,6 +536,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -541,6 +564,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -569,6 +593,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -598,6 +623,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -627,6 +653,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -660,6 +687,7 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_banned,brutal_purge,soft_purge,[]}, + {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, From 9300d6636d6bdbe22117639e30c2bd6663736073 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 15 Feb 2022 10:14:58 +0800 Subject: [PATCH 20/47] docs: update CHANGES-4.3.md --- CHANGES-4.3.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index ed3050058..b14be45a7 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -16,11 +16,13 @@ File format: * CLI `emqx_ctl pem_cache clean` to force purge x509 certificate cache, to force an immediate reload of all certificates after the files are updated on disk. +* Refactor the ExProto so that anonymous clients can also be displayed on the dashboard [#6983] ### Bug fixes * Fix case where publishing to a non-existent topic alias would crash the connection [#6979] * Fix HTTP-API 500 error on querying the lwm2m client list on the another node [#7009] +* Fix the ExProto connection registry is not released after the client process abnormally exits [#6983] ## v4.3.12 ### Important changes From 3cc2f0e2d3bafb230505f1d0c8a46680de27dee0 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 22 Feb 2022 11:31:14 +0800 Subject: [PATCH 21/47] fix(stomp): `conn_props` should be map --- apps/emqx_stomp/src/emqx_stomp_protocol.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_stomp/src/emqx_stomp_protocol.erl b/apps/emqx_stomp/src/emqx_stomp_protocol.erl index 6049b88f3..96e1a3203 100644 --- a/apps/emqx_stomp/src/emqx_stomp_protocol.erl +++ b/apps/emqx_stomp/src/emqx_stomp_protocol.erl @@ -152,7 +152,7 @@ default_conninfo(ConnInfo) -> clean_start => true, clientid => undefined, username => undefined, - conn_props => [], + conn_props => #{}, connected => false, connected_at => undefined, keepalive => undefined, @@ -814,4 +814,3 @@ interval(outgoing_timer, #pstate{heart_beats = HrtBt}) -> emqx_stomp_heartbeat:interval(outgoing, HrtBt); interval(clean_trans_timer, _) -> ?TRANS_TIMEOUT. - From d98d7e77c8ccb8159107e101d7ac251a7dee225b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 22 Feb 2022 11:47:04 +0800 Subject: [PATCH 22/47] chore: update CHANGES.md --- CHANGES-4.3.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index ed3050058..15393dfea 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -21,6 +21,8 @@ File format: * Fix case where publishing to a non-existent topic alias would crash the connection [#6979] * Fix HTTP-API 500 error on querying the lwm2m client list on the another node [#7009] +* Fix Server-KeepAlive wrongly applied on MQTT v3.0/v3.1 [#7085] +* Fix Stomp client can not trigger `$event/client_connection` message [#7096] ## v4.3.12 ### Important changes From 43c900bb5ada9e054959201cfe27a7497176ce17 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 11:16:38 +0100 Subject: [PATCH 23/47] chore: rename more descriptive EQM X to EMQX --- .github/ISSUE_TEMPLATE/bug-report.md | 4 ++-- Makefile | 1 - apps/emqx_auth_http/etc/emqx_auth_http.conf | 2 +- apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf | 2 +- apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf | 2 +- apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf | 2 +- apps/emqx_auth_redis/etc/emqx_auth_redis.conf | 4 ++-- .../emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf | 2 +- apps/emqx_exhook/etc/emqx_exhook.conf | 2 +- apps/emqx_exproto/etc/emqx_exproto.conf | 6 +++--- apps/emqx_lua_hook/etc/emqx_lua_hook.conf | 2 +- apps/emqx_management/etc/emqx_management.conf | 2 +- .../test/etc/emqx_management.conf | 2 +- apps/emqx_prometheus/etc/emqx_prometheus.conf | 2 +- apps/emqx_retainer/etc/emqx_retainer.conf | 2 +- .../emqx_rule_engine/etc/emqx_rule_engine.conf | 2 +- apps/emqx_web_hook/etc/emqx_web_hook.conf | 2 +- apps/emqx_web_hook/priv/emqx_web_hook.schema | 2 +- etc/emqx.conf | 18 +++++++++--------- lib-ce/emqx_dashboard/etc/emqx_dashboard.conf | 2 +- priv/emqx.schema | 2 +- 21 files changed, 32 insertions(+), 33 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 0258866dd..d8a0e22cd 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -12,11 +12,11 @@ assignees: tigercl **Environment**: -- EMQ X version (e.g. `emqx_ctl status`): +- EMQX version (e.g. `emqx_ctl status`): - Hardware configuration (e.g. `lscpu`): - OS (e.g. `cat /etc/os-release`): - Kernel (e.g. `uname -a`): -- Erlang/OTP version (in case you build emqx from source code): +- Erlang/OTP version (in case you build emqx from source code): - Others: **What happened and what you expected to happen**: diff --git a/Makefile b/Makefile index 626a4a3f1..6bc37d2db 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,6 @@ export EMQX_RELUP ?= true export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine export EMQX_DEFAULT_RUNNER = alpine:3.12 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) -export EMQX_DESC ?= EMQ X export EMQX_CE_DASHBOARD_VERSION ?= v4.3.5 export DOCKERFILE := deploy/docker/Dockerfile ifeq ($(OS),Windows_NT) diff --git a/apps/emqx_auth_http/etc/emqx_auth_http.conf b/apps/emqx_auth_http/etc/emqx_auth_http.conf index 2dd131dec..584448046 100644 --- a/apps/emqx_auth_http/etc/emqx_auth_http.conf +++ b/apps/emqx_auth_http/etc/emqx_auth_http.conf @@ -168,7 +168,7 @@ auth.http.enable_pipelining = true ## If not specified, the server's names returned in server's certificate is validated against ## what's provided `auth.http.auth_req.url` config's host part. -## Setting to 'disable' will make EMQ X ignore unmatched server names. +## Setting to 'disable' will make EMQX ignore unmatched server names. ## If set with a host name, the server's names returned in server's certificate is validated ## against this value. ## diff --git a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf index 2a3d038f0..f72e9859b 100644 --- a/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf +++ b/apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf @@ -78,7 +78,7 @@ auth.mongo.database = mqtt ## If not specified, the server's names returned in server's certificate is validated against ## what's provided `auth.mongo.server` config's host part. -## Setting to 'disable' will make EMQ X ignore unmatched server names. +## Setting to 'disable' will make EMQX ignore unmatched server names. ## If set with a host name, the server's names returned in server's certificate is validated ## against this value. ## diff --git a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf index 6014329b3..e38357e07 100644 --- a/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf +++ b/apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf @@ -123,7 +123,7 @@ auth.mysql.acl_query = select allow, ipaddr, username, clientid, access, topic f ## If not specified, the server's names returned in server's certificate is validated against ## what's provided `auth.mysql.server` config's host part. -## Setting to 'disable' will make EMQ X ignore unmatched server names. +## Setting to 'disable' will make EMQX ignore unmatched server names. ## If set with a host name, the server's names returned in server's certificate is validated ## against this value. ## diff --git a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf index e39d0c78a..b58be120b 100644 --- a/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf +++ b/apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf @@ -70,7 +70,7 @@ auth.pgsql.ssl = off ## If not specified, the server's names returned in server's certificate is validated against ## what's provided `auth.pgsql.server` config's host part. -## Setting to 'disable' will make EMQ X ignore unmatched server names. +## Setting to 'disable' will make EMQX ignore unmatched server names. ## If set with a host name, the server's names returned in server's certificate is validated ## against this value. ## diff --git a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf index 5a56c5dce..44aa250f9 100644 --- a/apps/emqx_auth_redis/etc/emqx_auth_redis.conf +++ b/apps/emqx_auth_redis/etc/emqx_auth_redis.conf @@ -123,9 +123,9 @@ auth.redis.acl_cmd = HGETALL mqtt_acl:%u ## If not specified, the server's names returned in server's certificate is validated against ## what's provided `auth.redis.server` config's host part. -## Setting to 'disable' will make EMQ X ignore unmatched server names. +## Setting to 'disable' will make EMQX ignore unmatched server names. ## If set with a host name, the server's names returned in server's certificate is validated ## against this value. ## ## Value: String | disable -## auth.redis.ssl.server_name_indication = disable \ No newline at end of file +## auth.redis.ssl.server_name_indication = disable diff --git a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf index f92ef042b..1192863c0 100644 --- a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf +++ b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf @@ -1,5 +1,5 @@ ##==================================================================== -## Configuration for EMQ X MQTT Broker Bridge +## Configuration for EMQX MQTT Broker Bridge ##==================================================================== ##-------------------------------------------------------------------- diff --git a/apps/emqx_exhook/etc/emqx_exhook.conf b/apps/emqx_exhook/etc/emqx_exhook.conf index ffb71e43b..4d94def06 100644 --- a/apps/emqx_exhook/etc/emqx_exhook.conf +++ b/apps/emqx_exhook/etc/emqx_exhook.conf @@ -1,5 +1,5 @@ ##==================================================================== -## EMQ X Hooks +## EMQX Hooks ##==================================================================== ## The default value or action will be returned, while the request to diff --git a/apps/emqx_exproto/etc/emqx_exproto.conf b/apps/emqx_exproto/etc/emqx_exproto.conf index 713685734..ae79e1a42 100644 --- a/apps/emqx_exproto/etc/emqx_exproto.conf +++ b/apps/emqx_exproto/etc/emqx_exproto.conf @@ -1,5 +1,5 @@ ##==================================================================== -## EMQ X ExProto +## EMQX ExProto ##==================================================================== exproto.server.http.port = 9100 @@ -65,7 +65,7 @@ exproto.listener.protoname.idle_timeout = 30s ## Example: allow 192.168.0.0/24 exproto.listener.protoname.access.1 = allow all -## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed +## Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed ## behind HAProxy or Nginx. ## ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ @@ -73,7 +73,7 @@ exproto.listener.protoname.access.1 = allow all ## Value: on | off ## exproto.listener.protoname.proxy_protocol = on -## Sets the timeout for proxy protocol. EMQ X will close the TCP connection +## Sets the timeout for proxy protocol. EMQX will close the TCP connection ## if no proxy protocol packet recevied within the timeout. ## ## Value: Duration diff --git a/apps/emqx_lua_hook/etc/emqx_lua_hook.conf b/apps/emqx_lua_hook/etc/emqx_lua_hook.conf index f0256afae..c8d220677 100644 --- a/apps/emqx_lua_hook/etc/emqx_lua_hook.conf +++ b/apps/emqx_lua_hook/etc/emqx_lua_hook.conf @@ -1,4 +1,4 @@ ##-------------------------------------------------------------------- -## EMQ X Lua Hook +## EMQX Lua Hook ##-------------------------------------------------------------------- diff --git a/apps/emqx_management/etc/emqx_management.conf b/apps/emqx_management/etc/emqx_management.conf index aa737add9..0170059d7 100644 --- a/apps/emqx_management/etc/emqx_management.conf +++ b/apps/emqx_management/etc/emqx_management.conf @@ -1,5 +1,5 @@ ##-------------------------------------------------------------------- -## EMQ X Management Plugin +## EMQX Management Plugin ##-------------------------------------------------------------------- ## Max Row Limit diff --git a/apps/emqx_management/test/etc/emqx_management.conf b/apps/emqx_management/test/etc/emqx_management.conf index cec70cc8e..074e10cd1 100644 --- a/apps/emqx_management/test/etc/emqx_management.conf +++ b/apps/emqx_management/test/etc/emqx_management.conf @@ -1,5 +1,5 @@ ##-------------------------------------------------------------------- -## EMQ X Management Plugin +## EMQX Management Plugin ##-------------------------------------------------------------------- ## Max Row Limit diff --git a/apps/emqx_prometheus/etc/emqx_prometheus.conf b/apps/emqx_prometheus/etc/emqx_prometheus.conf index 7bfa22095..48765f207 100644 --- a/apps/emqx_prometheus/etc/emqx_prometheus.conf +++ b/apps/emqx_prometheus/etc/emqx_prometheus.conf @@ -1,5 +1,5 @@ ##-------------------------------------------------------------------- -## emqx_prometheus for EMQ X +## emqx_prometheus for EMQX ##-------------------------------------------------------------------- ## The Prometheus Push Gateway URL address diff --git a/apps/emqx_retainer/etc/emqx_retainer.conf b/apps/emqx_retainer/etc/emqx_retainer.conf index 8b5c251c6..1b6e1082e 100644 --- a/apps/emqx_retainer/etc/emqx_retainer.conf +++ b/apps/emqx_retainer/etc/emqx_retainer.conf @@ -1,5 +1,5 @@ ##-------------------------------------------------------------------- -## EMQ X Retainer +## EMQX Retainer ##-------------------------------------------------------------------- ## Where to store the retained messages. diff --git a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf index 2fe946779..c0cba59bd 100644 --- a/apps/emqx_rule_engine/etc/emqx_rule_engine.conf +++ b/apps/emqx_rule_engine/etc/emqx_rule_engine.conf @@ -1,5 +1,5 @@ ##==================================================================== -## Rule Engine for EMQ X R4.0 +## EMQX Rule Engine ##==================================================================== rule_engine.ignore_sys_message = on diff --git a/apps/emqx_web_hook/etc/emqx_web_hook.conf b/apps/emqx_web_hook/etc/emqx_web_hook.conf index 6bffbc7ef..12934bf8b 100644 --- a/apps/emqx_web_hook/etc/emqx_web_hook.conf +++ b/apps/emqx_web_hook/etc/emqx_web_hook.conf @@ -45,7 +45,7 @@ web.hook.body.encoding_of_payload_field = plain ## If not specified, the server's names returned in server's certificate is validated against ## what's provided `web.hook.url` config's host part. -## Setting to 'disable' will make EMQ X ignore unmatched server names. +## Setting to 'disable' will make EMQX ignore unmatched server names. ## If set with a host name, the server's names returned in server's certificate is validated ## against this value. ## diff --git a/apps/emqx_web_hook/priv/emqx_web_hook.schema b/apps/emqx_web_hook/priv/emqx_web_hook.schema index a80f460c9..2dfd0b081 100644 --- a/apps/emqx_web_hook/priv/emqx_web_hook.schema +++ b/apps/emqx_web_hook/priv/emqx_web_hook.schema @@ -1,5 +1,5 @@ %%-*- mode: erlang -*- -%% EMQ X R3.0 config mapping +%% EMQX config mapping {mapping, "web.hook.url", "emqx_web_hook.url", [ {datatype, string} diff --git a/etc/emqx.conf b/etc/emqx.conf index 94416783f..65d96cd33 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -1,4 +1,4 @@ -## EMQ X Configuration 4.3 +## EMQX Configuration 4.3 ## NOTE: Do not change format of CONFIG_SECTION_{BGN,END} comments! @@ -203,10 +203,10 @@ node.data_dir = {{ platform_data_dir }} ## the heartbeat pings. ## ## NOTE: When managed by systemd (or other supervision tools like systemd), -## heart will probably only cause EMQ X to stop, but restart or not will +## heart will probably only cause EMQX to stop, but restart or not will ## depend on systemd's restart strategy. ## NOTE: When running in docker, the container will die as soon as the the -## heart process kills EMQ X, but restart or not will depend on container +## heart process kills EMQX, but restart or not will depend on container ## supervision strategy, such as k8s restartPolicy. ## ## Value: on @@ -1118,7 +1118,7 @@ listener.tcp.external.zone = external ## Example: allow 192.168.0.0/24 listener.tcp.external.access.1 = allow all -## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed +## Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed ## behind HAProxy or Nginx. ## ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ @@ -1126,7 +1126,7 @@ listener.tcp.external.access.1 = allow all ## Value: on | off ## listener.tcp.external.proxy_protocol = on -## Sets the timeout for proxy protocol. EMQ X will close the TCP connection +## Sets the timeout for proxy protocol. EMQX will close the TCP connection ## if no proxy protocol packet recevied within the timeout. ## ## Value: Duration @@ -1629,13 +1629,13 @@ listener.ws.external.access.1 = allow all ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 ## listener.ws.external.supported_subprotocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## Specify which HTTP header for real source IP if the EMQ X cluster is +## Specify which HTTP header for real source IP if the EMQX cluster is ## deployed behind NGINX or HAProxy. ## ## Default: X-Forwarded-For ## listener.ws.external.proxy_address_header = X-Forwarded-For -## Specify which HTTP header for real source port if the EMQ X cluster is +## Specify which HTTP header for real source port if the EMQX cluster is ## deployed behind NGINX or HAProxy. ## ## Default: X-Forwarded-Port @@ -1898,13 +1898,13 @@ listener.wss.external.access.1 = allow all ## Default: mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 ## listener.wss.external.supported_subprotocols = mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5 -## Specify which HTTP header for real source IP if the EMQ X cluster is +## Specify which HTTP header for real source IP if the EMQX cluster is ## deployed behind NGINX or HAProxy. ## ## Default: X-Forwarded-For ## listener.wss.external.proxy_address_header = X-Forwarded-For -## Specify which HTTP header for real source port if the EMQ X cluster is +## Specify which HTTP header for real source port if the EMQX cluster is ## deployed behind NGINX or HAProxy. ## ## Default: X-Forwarded-Port diff --git a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf index 933b9885d..f67b88c17 100644 --- a/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf +++ b/lib-ce/emqx_dashboard/etc/emqx_dashboard.conf @@ -1,5 +1,5 @@ ##-------------------------------------------------------------------- -## EMQ X Dashboard +## EMQX Dashboard ##-------------------------------------------------------------------- ## Default user's login name. diff --git a/priv/emqx.schema b/priv/emqx.schema index 9623f9b86..b6a9ee41e 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -1,5 +1,5 @@ %%-*- mode: erlang -*- -%% EMQ X R4.0 config mapping +%% EMQX Config Mapping %%-------------------------------------------------------------------- %% Cluster From b4836a01fdd387ac7201002d764dd8d6d6541d03 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 12:42:13 +0100 Subject: [PATCH 24/47] fix(docker): allow config changes in docker rebuild --- CHANGES-4.3.md | 5 +++++ deploy/docker/Dockerfile | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 15393dfea..0dc98d2c9 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -12,6 +12,11 @@ File format: ## v4.3.13 +### Important changes + +* For docker image, /opt/emqx/etc has been removed from the VOLUME list, + this made easier for users to rebuild image on top with changed configs. + ### Enhancements * CLI `emqx_ctl pem_cache clean` to force purge x509 certificate cache, diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 4409f7d22..f24b4a348 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -60,7 +60,8 @@ RUN chgrp -Rf emqx /opt/emqx && chmod -Rf g+w /opt/emqx \ USER emqx -VOLUME ["/opt/emqx/log", "/opt/emqx/data", "/opt/emqx/etc"] +## NOTE: /opt/emqx/etc is removed from the VOLUME list since 4.3.13 +VOLUME ["/opt/emqx/log", "/opt/emqx/data"] # emqx will occupy these port: # - 1883 port for MQTT From 1fa27b28e4f4d9eb2f08ce0581f8ae7a79e7a7b0 Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Tue, 22 Feb 2022 16:05:50 +0100 Subject: [PATCH 25/47] fix: bump emqx_web_hook app version --- CHANGES-4.3.md | 2 +- apps/emqx_web_hook/src/emqx_web_hook.app.src | 2 +- apps/emqx_web_hook/src/emqx_web_hook.appup.src | 18 ++++++++++++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 0dc98d2c9..7542e0ef3 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -15,7 +15,7 @@ File format: ### Important changes * For docker image, /opt/emqx/etc has been removed from the VOLUME list, - this made easier for users to rebuild image on top with changed configs. + this made it easier for the users to rebuild image on top with changed configs. ### Enhancements diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index acabcd954..64cd28784 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.9"}, % strict semver, bump manually! + {vsn, "4.3.10"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index 531ad308b..d90663cca 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -1,24 +1,34 @@ %% -*- mode: erlang -*- {VSN, - [{<<"4\\.3\\.[0-2]">>, + [{<<"4\\.3\\.[0-2]$">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[3-8]">>, + {<<"4\\.3\\.[3-7]$">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.8", + [{load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, + {"4.3.9", + [ %% nothing so far + ]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[0-2]">>, + [{<<"4\\.3\\.[0-2]$">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[3-8]">>, + {<<"4\\.3\\.[3-7]$">>, [{apply,{application,stop,[emqx_web_hook]}}, {load_module,emqx_web_hook_app,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook,brutal_purge,soft_purge,[]}, {load_module,emqx_web_hook_actions,brutal_purge,soft_purge,[]}]}, + {"4.3.8", + [{load_module,emqx_web_hook,brutal_purge,soft_purge,[]}]}, + {"4.3.9", + [ %% nothing so far + ]}, {<<".*">>,[]}]}. From 841d6d5036073e830228487c66e462c9a5c53e87 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 23 Feb 2022 09:14:28 +0100 Subject: [PATCH 26/47] fix: system mem false alarm --- src/emqx_os_mon.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/emqx_os_mon.erl b/src/emqx_os_mon.erl index 55c324d57..e7565947c 100644 --- a/src/emqx_os_mon.erl +++ b/src/emqx_os_mon.erl @@ -190,7 +190,7 @@ ensure_system_memory_alarm(HW) -> case erlang:whereis(memsup) of undefined -> ok; _Pid -> - {Allocated, Total, _Worst} = memsup:get_memory_data(), + {Total, Allocated, _Worst} = memsup:get_memory_data(), case Total =/= 0 andalso Allocated/Total * 100 > HW of true -> emqx_alarm:activate(high_system_memory_usage, #{high_watermark => HW}); false -> ok From 372ea29d96b40413b5c89e41625cb8b7921821d7 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 23 Feb 2022 09:26:17 +0100 Subject: [PATCH 27/47] fix: upgrade code, system mem false alarm --- src/emqx.appup.src | 1 + 1 file changed, 1 insertion(+) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 5300e6bc8..610b40c75 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -5,6 +5,7 @@ {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, From 2c4fc6f6f7d7fa8917182dc09c52bfc439fafdf5 Mon Sep 17 00:00:00 2001 From: William Yang Date: Wed, 23 Feb 2022 09:28:48 +0100 Subject: [PATCH 28/47] docs(changelog): system mem false alarm --- CHANGES-4.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index e38cb1b0f..148627628 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -30,6 +30,7 @@ File format: * Fix the ExProto connection registry is not released after the client process abnormally exits [#6983] * Fix Server-KeepAlive wrongly applied on MQTT v3.0/v3.1 [#7085] * Fix Stomp client can not trigger `$event/client_connection` message [#7096] +* Fix system memory false alarm at boot ## v4.3.12 ### Important changes From 442a1290a8496f82652449c0979276147e2d97dc Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 23 Feb 2022 19:12:00 +0800 Subject: [PATCH 29/47] fix(relup): release upgrade failed on symlink already exists --- bin/install_upgrade.escript | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index f88af106d..0bd4bdee1 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -6,6 +6,8 @@ -define(TIMEOUT, 300000). -define(INFO(Fmt,Args), io:format(Fmt++"~n",Args)). +-mode(compile). + main([Command0, DistInfoStr | CommandArgs]) -> %% convert the distribution info arguments string to an erlang term {ok, Tokens, _} = erl_scan:string(DistInfoStr ++ "."), @@ -210,15 +212,24 @@ find_and_link_release_package(Version, RelName) -> ok = filelib:ensure_dir(filename:join([filename:dirname(ReleaseLink), "dummy"])), %% create the symlink pointing to the full path name of the %% release package we found - case file:make_symlink(filename:absname(Filename), ReleaseLink) of - ok -> - ok; - {error, eperm} -> % windows! - {ok,_} = file:copy(filename:absname(Filename), ReleaseLink) - end, + make_symlink_or_copy(filename:absname(Filename), ReleaseLink), {Filename, ReleaseHandlerPackageLink} end. +make_symlink_or_copy(Filename, ReleaseLink) -> + case file:make_symlink(Filename, ReleaseLink) of + ok -> ok; + {error, eexist} -> + ?INFO("symlink ~p already exists, recreate it", [ReleaseLink]), + ok = file:delete(ReleaseLink), + make_symlink_or_copy(Filename, ReleaseLink); + {error, Reason} when Reason =:= eperm; Reason =:= enotsup -> + {ok, _} = file:copy(Filename, ReleaseLink); + {error, Reason} -> + ?INFO("create symlink ~p failed", [ReleaseLink]), + error({Reason, ReleaseLink}) + end. + unpack_zipballs(RelNameStr, Version) -> {ok, Cwd} = file:get_cwd(), GzFile = filename:absname(filename:join(["releases", RelNameStr ++ "-" ++ Version ++ ".tar.gz"])), From 6dd0b49dd2ec81469271701ac95f9982fca1b450 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 16 Feb 2022 14:51:01 +0800 Subject: [PATCH 30/47] feat(cm): force shutdown of processe that cannot answer takeover event Related PR: #6030 --- src/emqx_cm.erl | 84 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 23f078568..4811313d8 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -226,18 +226,24 @@ open_session(true, ClientInfo = #{clientid := ClientId}, ConnInfo) -> open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> Self = self(), ResumeStart = fun(_) -> - case takeover_session(ClientId) of - {ok, ConnMod, ChanPid, Session} -> - ok = emqx_session:resume(ClientInfo, Session), - Pendings = ConnMod:call(ChanPid, {takeover, 'end'}, ?T_TAKEOVER), - register_channel(ClientId, Self, ConnInfo), - {ok, #{session => Session, - present => true, - pendings => Pendings}}; - {error, not_found} -> + CreateSess = + fun() -> Session = create_session(ClientInfo, ConnInfo), register_channel(ClientId, Self, ConnInfo), {ok, #{session => Session, present => false}} + end, + case takeover_session(ClientId) of + {ok, ConnMod, ChanPid, Session} -> + ok = emqx_session:resume(ClientInfo, Session), + case call_or_kill({takeover, 'end'}, ConnMod, ChanPid) of + {error, _} -> CreateSess(); + Pendings -> + register_channel(ClientId, Self, ConnInfo), + {ok, #{session => Session, + present => true, + pendings => Pendings}} + end; + {error, _Reason} -> CreateSess() end end, emqx_cm_locker:trans(ClientId, ResumeStart). @@ -271,9 +277,12 @@ takeover_session(ClientId, ChanPid) when node(ChanPid) == node() -> undefined -> {error, not_found}; ConnMod when is_atom(ConnMod) -> - %% TODO: if takeover times out, maybe kill the old? - Session = ConnMod:call(ChanPid, {takeover, 'begin'}, ?T_TAKEOVER), - {ok, ConnMod, ChanPid, Session} + case call_or_kill({takeover, 'begin'}, ConnMod, ChanPid) of + {error, Reason} -> + {error, Reason}; + Session -> + {ok, ConnMod, ChanPid, Session} + end end; takeover_session(ClientId, ChanPid) -> rpc_call(node(ChanPid), takeover_session, [ClientId, ChanPid], ?T_TAKEOVER). @@ -286,44 +295,63 @@ discard_session(ClientId) when is_binary(ClientId) -> ChanPids -> lists:foreach(fun(Pid) -> discard_session(ClientId, Pid) end, ChanPids) end. -%% @private Kick a local stale session to force it step down. -%% If failed to kick (e.g. timeout) force a kill. +%% @private call a local stale session to execute an Action. +%% If failed to response (e.g. timeout) force a kill. %% Keeping the stale pid around, or returning error or raise an exception %% benefits nobody. --spec kick_or_kill(kick | discard, module(), pid()) -> ok. -kick_or_kill(Action, ConnMod, Pid) -> +-spec call_or_kill(Action, module(), pid()) + -> ok %% returned by kick, discard + | emqx_session:session() %% returned by {takeover, 'begin'} + | list(emqx_type:deliver()) %% returned by {takeover, 'end'} + when Action :: kick | discard | {takeover, 'begin'} | {takeover, 'end'}. +call_or_kill(Action, ConnMod, Pid) -> try + Timeout = case Action of + {takeover, _} -> ?T_TAKEOVER; + _ -> ?T_KICK + end, %% this is essentailly a gen_server:call implemented in emqx_connection %% and emqx_ws_connection. %% the handle_call is implemented in emqx_channel - ok = apply(ConnMod, call, [Pid, Action, ?T_KICK]) + apply(ConnMod, call, [Pid, Action, Timeout]) catch _ : noproc -> % emqx_ws_connection: call - ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}); + ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}), + return_error_if_action_is_takeover(Action, noproc); _ : {noproc, _} -> % emqx_connection: gen_server:call - ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}); - _ : {shutdown, _} -> - ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}); - _ : {{shutdown, _}, _} -> - ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}); + ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}), + return_error_if_action_is_takeover(Action, noproc); + _ : Reason = {shutdown, _} -> + ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), + return_error_if_action_is_takeover(Action, Reason); + _ : Reason = {{shutdown, _}, _} -> + ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), + return_error_if_action_is_takeover(Action, Reason); _ : {timeout, {gen_server, call, _}} -> - ?tp(warning, "session_kick_timeout", + ?tp(warning, "call_session_timeout", #{pid => Pid, action => Action, stale_channel => stale_channel_info(Pid) }), - ok = force_kill(Pid); + ok = force_kill(Pid), + return_error_if_action_is_takeover(Action, timeout); _ : Error : St -> - ?tp(error, "session_kick_exception", + ?tp(error, "call_session_exception", #{pid => Pid, action => Action, reason => Error, stacktrace => St, stale_channel => stale_channel_info(Pid) }), - ok = force_kill(Pid) + ok = force_kill(Pid), + return_error_if_action_is_takeover(Action, Error) end. +return_error_if_action_is_takeover({takeover, _}, Reason) -> + {error, Reason}; +return_error_if_action_is_takeover(_, _) -> + ok. + force_kill(Pid) -> exit(Pid, kill), ok. @@ -344,7 +372,7 @@ kick_session(Action, ClientId, ChanPid) when node(ChanPid) == node() -> %% already deregistered ok; ConnMod when is_atom(ConnMod) -> - ok = kick_or_kill(Action, ConnMod, ChanPid) + ok = call_or_kill(Action, ConnMod, ChanPid) end; kick_session(Action, ClientId, ChanPid) -> %% call remote node on the old APIs because we do not know if they have upgraded From 66807f17dfb1258bead39807385fe1732de97438 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 16 Feb 2022 15:43:31 +0800 Subject: [PATCH 31/47] test(cm): cover {takeover, 'begin'/'end'} action --- test/emqx_cm_SUITE.erl | 115 ++++++++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 41 deletions(-) diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl index a46c046ca..c82e525f0 100644 --- a/test/emqx_cm_SUITE.erl +++ b/test/emqx_cm_SUITE.erl @@ -187,57 +187,89 @@ t_open_session_race_condition(_) -> ok = flush_emqx_pool(), ?assertEqual([], emqx_cm:lookup_channels(ClientId)). -t_kick_session_discard_normal(_) -> - test_kick_session(discard, normal). +t_call_session_discard_normal(_) -> + test_call_session(discard, normal). -t_kick_session_discard_shutdown(_) -> - test_kick_session(discard, shutdown). +t_call_session_discard_shutdown(_) -> + test_call_session(discard, shutdown). -t_kick_session_discard_shutdown_with_reason(_) -> - test_kick_session(discard, {shutdown, discard}). +t_call_session_discard_shutdown_with_reason(_) -> + test_call_session(discard, {shutdown, discard}). -t_kick_session_discard_timeout(_) -> - test_kick_session(discard, timeout). +t_call_session_discard_timeout(_) -> + test_call_session(discard, timeout). -t_kick_session_discard_noproc(_) -> - test_kick_session(discard, noproc). +t_call_session_discard_noproc(_) -> + test_call_session(discard, noproc). -t_kick_session_kick_normal(_) -> - test_kick_session(discard, normal). +t_call_session_kick_normal(_) -> + test_call_session(kick, normal). -t_kick_session_kick_shutdown(_) -> - test_kick_session(discard, shutdown). +t_call_session_kick_shutdown(_) -> + test_call_session(kick, shutdown). -t_kick_session_kick_shutdown_with_reason(_) -> - test_kick_session(discard, {shutdown, discard}). +t_call_session_kick_shutdown_with_reason(_) -> + test_call_session(kick, {shutdown, discard}). -t_kick_session_kick_timeout(_) -> - test_kick_session(discard, timeout). +t_call_session_kick_timeout(_) -> + test_call_session(kick, timeout). -t_kick_session_kick_noproc(_) -> - test_kick_session(discard, noproc). +t_call_session_kick_noproc(_) -> + test_call_session(discard, noproc). -test_kick_session(Action, Reason) -> +t_call_session_takeover_begin_normal(_) -> + test_call_session({takeover, 'begin'}, normal). + +t_call_session_takeover_begin_shutdown(_) -> + test_call_session({takeover, 'begin'}, shutdown). + +t_call_session_takeover_begin_shutdown_with_reason(_) -> + test_call_session({takeover, 'begin'}, {shutdown, discard}). + +t_call_session_takeover_begin_timeout(_) -> + test_call_session({takeover, 'begin'}, timeout). + +t_call_session_takeover_begin_noproc(_) -> + test_call_session({takeover, 'begin'}, noproc). + +t_call_session_takeover_end_normal(_) -> + test_call_session({takeover, 'end'}, normal). + +t_call_session_takeover_end_shutdown(_) -> + test_call_session({takeover, 'end'}, shutdown). + +t_call_session_takeover_end_shutdown_with_reason(_) -> + test_call_session({takeover, 'end'}, {shutdown, discard}). + +t_call_session_takeover_end_timeout(_) -> + test_call_session({takeover, 'end'}, timeout). + +t_call_session_takeover_end_noproc(_) -> + test_call_session({takeover, 'end'}, noproc). + +test_call_session(Action, Reason) -> ClientId = rand_client_id(), #{conninfo := ConnInfo} = ?ChanInfo, FakeSessionFun = fun Loop() -> - receive - {'$gen_call', From, A} when A =:= kick orelse - A =:= discard -> - case Reason of - normal -> - gen_server:reply(From, ok); - timeout -> - %% no response to the call - Loop(); - _ -> - exit(Reason) - end; - Msg -> - ct:pal("(~p) fake_session_discarded ~p", [Action, Msg]), - Loop() - end + receive + {'$gen_call', From, A} when A =:= kick orelse + A =:= discard orelse + A =:= {takeover, 'begin'} orelse + A =:= {takeover, 'end'} -> + case Reason of + normal when A =:= kick orelse A =:= discard -> + gen_server:reply(From, ok); + timeout -> + %% no response to the call + Loop(); + _ -> + exit(Reason) + end; + Msg -> + ct:pal("(~p) fake_session_discarded ~p", [Action, Msg]), + Loop() + end end, {Pid1, _} = spawn_monitor(FakeSessionFun), {Pid2, _} = spawn_monitor(FakeSessionFun), @@ -249,10 +281,11 @@ test_kick_session(Action, Reason) -> noproc -> exit(Pid1, kill), exit(Pid2, kill); _ -> ok end, - ok = case Action of - kick -> emqx_cm:kick_session(ClientId); - discard -> emqx_cm:discard_session(ClientId) - end, + _ = case Action of + kick -> emqx_cm:kick_session(ClientId); + discard -> emqx_cm:discard_session(ClientId); + {takeover, _} -> emqx_cm:takeover_session(ClientId) + end, case Reason =:= timeout orelse Reason =:= noproc of true -> ?assertEqual(killed, ?WAIT({'DOWN', _, process, Pid1, R}, 2_000, R)), From 96a86061c2155a1ff2e408957cd3dc68bb67eda7 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 24 Feb 2022 11:26:42 +0800 Subject: [PATCH 32/47] refactor(sn): pretty frame print formation --- apps/emqx_sn/src/emqx_sn_frame.erl | 89 ++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 24 deletions(-) diff --git a/apps/emqx_sn/src/emqx_sn_frame.erl b/apps/emqx_sn/src/emqx_sn_frame.erl index eed32803d..28a20956e 100644 --- a/apps/emqx_sn/src/emqx_sn_frame.erl +++ b/apps/emqx_sn/src/emqx_sn_frame.erl @@ -268,40 +268,81 @@ message_type(16#1d) -> message_type(Type) -> io_lib:format("Unknown Type ~p", [Type]). +format(?SN_CONNECT_MSG(Flags, ProtocolId, Duration, ClientId)) -> + #mqtt_sn_flags{ + will = Will, + clean_start = CleanStart} = Flags, + io_lib:format("SN_CONNECT(W~w, C~w, ProtocolId=~w, Duration=~w, " + "ClientId=~s)", + [bool(Will), bool(CleanStart), + ProtocolId, Duration, ClientId]); +format(?SN_CONNACK_MSG(ReturnCode)) -> + io_lib:format("SN_CONNACK(ReturnCode=~w)", [ReturnCode]); +format(?SN_WILLTOPICREQ_MSG()) -> + "SN_WILLTOPICREQ()"; +format(?SN_WILLTOPIC_MSG(Flags, Topic)) -> + #mqtt_sn_flags{ + qos = QoS, + retain = Retain} = Flags, + io_lib:format("SN_WILLTOPIC(Q~w, R~w, Topic=~s)", + [QoS, bool(Retain), Topic]); +format(?SN_WILLTOPIC_EMPTY_MSG) -> + "SN_WILLTOPIC(_)"; +format(?SN_WILLMSGREQ_MSG()) -> + "SN_WILLMSGREQ()"; +format(?SN_WILLMSG_MSG(Msg)) -> + io_lib:format("SN_WILLMSG_MSG(Msg=~p)", [Msg]); format(?SN_PUBLISH_MSG(Flags, TopicId, MsgId, Data)) -> - io_lib:format("mqtt_sn_message SN_PUBLISH, ~s, TopicId=~w, MsgId=~w, Payload=~w", - [format_flag(Flags), TopicId, MsgId, Data]); -format(?SN_PUBACK_MSG(Flags, MsgId, ReturnCode)) -> - io_lib:format("mqtt_sn_message SN_PUBACK, ~s, MsgId=~w, ReturnCode=~w", - [format_flag(Flags), MsgId, ReturnCode]); + #mqtt_sn_flags{ + dup = Dup, + qos = QoS, + retain = Retain, + topic_id_type = TopicIdType} = Flags, + io_lib:format("SN_PUBLISH(D~w, Q~w, R~w, TopicIdType=~w, TopicId=~w, " + "MsgId=~w, Payload=~p)", + [bool(Dup), QoS, bool(Retain), + TopicIdType, TopicId, MsgId, Data]); +format(?SN_PUBACK_MSG(TopicId, MsgId, ReturnCode)) -> + io_lib:format("SN_PUBACK(TopicId=~w, MsgId=~w, ReturnCode=~w)", + [TopicId, MsgId, ReturnCode]); format(?SN_PUBREC_MSG(?SN_PUBCOMP, MsgId)) -> - io_lib:format("mqtt_sn_message SN_PUBCOMP, MsgId=~w", [MsgId]); + io_lib:format("SN_PUBCOMP(MsgId=~w)", [MsgId]); format(?SN_PUBREC_MSG(?SN_PUBREC, MsgId)) -> - io_lib:format("mqtt_sn_message SN_PUBREC, MsgId=~w", [MsgId]); + io_lib:format("SN_PUBREC(MsgId=~w)", [MsgId]); format(?SN_PUBREC_MSG(?SN_PUBREL, MsgId)) -> - io_lib:format("mqtt_sn_message SN_PUBREL, MsgId=~w", [MsgId]); + io_lib:format("SN_PUBREL(MsgId=~w)", [MsgId]); format(?SN_SUBSCRIBE_MSG(Flags, Msgid, Topic)) -> - io_lib:format("mqtt_sn_message SN_SUBSCRIBE, ~s, MsgId=~w, TopicId=~w", - [format_flag(Flags), Msgid, Topic]); + #mqtt_sn_flags{ + dup = Dup, + qos = QoS, + topic_id_type = TopicIdType} = Flags, + io_lib:format("SN_SUBSCRIBE(D~w, Q~w, TopicIdType=~w, MsgId=~w, " + "TopicId=~w)", + [bool(Dup), QoS, TopicIdType, Msgid, Topic]); format(?SN_SUBACK_MSG(Flags, TopicId, MsgId, ReturnCode)) -> - io_lib:format("mqtt_sn_message SN_SUBACK, ~s, MsgId=~w, TopicId=~w, ReturnCode=~w", - [format_flag(Flags), MsgId, TopicId, ReturnCode]); + #mqtt_sn_flags{qos = QoS} = Flags, + io_lib:format("SN_SUBACK(GrantedQoS=~w, MsgId=~w, TopicId=~w, " + "ReturnCode=~w)", + [QoS, MsgId, TopicId, ReturnCode]); format(?SN_UNSUBSCRIBE_MSG(Flags, Msgid, Topic)) -> - io_lib:format("mqtt_sn_message SN_UNSUBSCRIBE, ~s, MsgId=~w, TopicId=~w", - [format_flag(Flags), Msgid, Topic]); + #mqtt_sn_flags{topic_id_type = TopicIdType} = Flags, + io_lib:format("SN_UNSUBSCRIBE(TopicIdType=~s, MsgId=~w, TopicId=~w)", + [TopicIdType, Msgid, Topic]); format(?SN_UNSUBACK_MSG(MsgId)) -> - io_lib:format("mqtt_sn_message SN_UNSUBACK, MsgId=~w", [MsgId]); + io_lib:format("SN_UNSUBACK(MsgId=~w)", [MsgId]); format(?SN_REGISTER_MSG(TopicId, MsgId, TopicName)) -> - io_lib:format("mqtt_sn_message SN_REGISTER, TopicId=~w, MsgId=~w, TopicName=~w", + io_lib:format("SN_REGISTER(TopicId=~w, MsgId=~w, TopicName=~s)", [TopicId, MsgId, TopicName]); format(?SN_REGACK_MSG(TopicId, MsgId, ReturnCode)) -> - io_lib:format("mqtt_sn_message SN_REGACK, TopicId=~w, MsgId=~w, ReturnCode=~w", + io_lib:format("SN_REGACK(TopicId=~w, MsgId=~w, ReturnCode=~w)", [TopicId, MsgId, ReturnCode]); +format(?SN_PINGREQ_MSG(ClientId)) -> + io_lib:format("SN_PINGREQ(ClientId=~s)", [ClientId]); +format(?SN_PINGRESP_MSG()) -> + "SN_PINGREQ()"; +format(?SN_DISCONNECT_MSG(Duration)) -> + io_lib:format("SN_DISCONNECT(Duration=~s)", [Duration]); + format(#mqtt_sn_message{type = Type, variable = Var}) -> - io_lib:format("mqtt_sn_message type=~s, Var=~w", [emqx_sn_frame:message_type(Type), Var]). - -format_flag(#mqtt_sn_flags{dup = Dup, qos = QoS, retain = Retain, will = Will, clean_start = CleanStart, topic_id_type = TopicType}) -> - io_lib:format("mqtt_sn_flags{dup=~p, qos=~p, retain=~p, will=~p, clean_session=~p, topic_id_type=~p}", - [Dup, QoS, Retain, Will, CleanStart, TopicType]); -format_flag(_Flag) -> "invalid flag". - + io_lib:format("mqtt_sn_message(type=~s, Var=~w)", + [emqx_sn_frame:message_type(Type), Var]). From e9031ea4c1573d304f4a15f0e8366a6a6cdac523 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 10 Feb 2022 14:25:01 +0800 Subject: [PATCH 33/47] fix(mqttsn): cache and replay the invaild-topic-id delivery --- apps/emqx_sn/src/emqx_sn_gateway.erl | 113 ++++++++++---- apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl | 155 ++++++++++++++++++- 2 files changed, 235 insertions(+), 33 deletions(-) diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index 442aa1db8..265607229 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -94,6 +94,8 @@ idle_timeout :: integer(), enable_qos3 = false :: boolean(), has_pending_pingresp = false :: boolean(), + %% Store all qos0 messages for waiting REGACK + %% Note: QoS1/QoS2 messages will kept inflight queue pending_topic_ids = #{} :: pending_msgs() }). @@ -490,7 +492,7 @@ handle_event({call, From}, Req, _StateName, State) -> {reply, Reply, NState} -> gen_server:reply(From, Reply), {keep_state, NState}; - {stop, Reason, Reply, NState} -> + {shutdown, Reason, Reply, NState} -> State0 = case NState#state.sockstate of running -> send_message(?SN_DISCONNECT_MSG(undefined), NState); @@ -518,10 +520,9 @@ handle_event(info, {datagram, SockPid, Data}, StateName, end; handle_event(info, {deliver, _Topic, Msg}, asleep, - State = #state{channel = Channel, pending_topic_ids = Pendings}) -> + State = #state{channel = Channel}) -> % section 6.14, Support of sleeping clients - ?LOG(debug, "enqueue downlink message in asleep state, msg: ~0p, pending_topic_ids: ~0p", - [Msg, Pendings]), + ?LOG(debug, "enqueue downlink message in asleep state, msg: ~0p", [Msg]), Session = emqx_session:enqueue(emqx_channel:info(clientinfo, Channel), Msg, emqx_channel:get_session(Channel)), {keep_state, State#state{channel = emqx_channel:set_session(Session, Channel)}}; @@ -610,7 +611,7 @@ handle_call(_From, Req, State = #state{channel = Channel}) -> {reply, Reply, NChannel} -> {reply, Reply, State#state{channel = NChannel}}; {shutdown, Reason, Reply, NChannel} -> - stop(Reason, Reply, State#state{channel = NChannel}) + {shutdown, Reason, Reply, State#state{channel = NChannel}} end. handle_info({sock_closed, Reason} = Info, State = #state{channel = Channel}) -> @@ -723,11 +724,19 @@ mqtt2sn(?PUBCOMP_PACKET(MsgId), _State) -> mqtt2sn(?UNSUBACK_PACKET(MsgId), _State)-> ?SN_UNSUBACK_MSG(MsgId); -mqtt2sn(?PUBLISH_PACKET(QoS, Topic, PacketId, Payload), #state{channel = Channel}) -> - NewPacketId = if QoS =:= ?QOS_0 -> 0; +mqtt2sn( + #mqtt_packet{header = #mqtt_packet_header{ + type = ?PUBLISH, + qos = QoS, + %dup = Dup, + retain = Retain}, + variable = #mqtt_packet_publish{ + topic_name = Topic, + packet_id = PacketId}, + payload = Payload}, #state{clientid = ClientId}) -> + NPacketId = if QoS =:= ?QOS_0 -> 0; true -> PacketId end, - ClientId = emqx_channel:info(clientid, Channel), {TopicIdType, TopicContent} = case emqx_sn_registry:lookup_topic_id(ClientId, Topic) of {predef, PredefTopicId} -> {?SN_PREDEFINED_TOPIC, PredefTopicId}; @@ -737,8 +746,12 @@ mqtt2sn(?PUBLISH_PACKET(QoS, Topic, PacketId, Payload), #state{channel = Channe {?SN_SHORT_TOPIC, Topic} end, - Flags = #mqtt_sn_flags{qos = QoS, topic_id_type = TopicIdType}, - ?SN_PUBLISH_MSG(Flags, TopicContent, NewPacketId, Payload); + Flags = #mqtt_sn_flags{ + %dup = Dup, + qos = QoS, + retain = Retain, + topic_id_type = TopicIdType}, + ?SN_PUBLISH_MSG(Flags, TopicContent, NPacketId, Payload); mqtt2sn(?SUBACK_PACKET(MsgId, ReturnCodes), _State)-> % if success, suback is sent by handle_info({suback, MsgId, [GrantedQoS]}, ...) @@ -766,9 +779,10 @@ send_connack(State) -> send_message(Msg = #mqtt_sn_message{type = Type}, State = #state{sockpid = SockPid, peername = Peername}) -> - ?LOG(debug, "SEND ~s~n", [emqx_sn_frame:format(Msg)]), + ?LOG(info, "SEND ~s~n", [emqx_sn_frame:format(Msg)]), inc_outgoing_stats(Type), Data = emqx_sn_frame:serialize(Msg), + ?LOG(debug, "SEND ~0p", [Data]), ok = emqx_metrics:inc('bytes.sent', iolist_size(Data)), SockPid ! {datagram, Peername, Data}, State. @@ -793,13 +807,6 @@ stop(Reason, State) -> maybe_send_will_msg(Reason, State), {stop, {shutdown, Reason}, State}. -stop({shutdown, Reason}, Reply, State) -> - stop(Reason, Reply, State); -stop(Reason, Reply, State) -> - ?LOG(stop_log_level(Reason), "stop due to ~p", [Reason]), - maybe_send_will_msg(Reason, State), - {stop, {shutdown, Reason}, Reply, State}. - maybe_send_will_msg(normal, _State) -> ok; maybe_send_will_msg(_Reason, State) -> @@ -825,6 +832,9 @@ do_connect(ClientId, CleanStart, WillFlag, Duration, State) -> %% At any point in time a client may have only one QoS level 1 or 2 PUBLISH message %% outstanding, i.e. it has to wait for the termination of this PUBLISH message exchange %% before it could start a new level 1 or 2 transaction. + %% + %% FIXME: But we should have a re-try timer to re-send the inflight + %% qos1/qos2 message OnlyOneInflight = #{'Receive-Maximum' => 1}, ConnPkt = #mqtt_packet_connect{clientid = ClientId, clean_start = CleanStart, @@ -973,11 +983,16 @@ do_puback(TopicId, MsgId, ReturnCode, StateName, case emqx_sn_registry:lookup_topic(ClientId, TopicId) of undefined -> {keep_state, State}; TopicName -> - %%notice that this TopicName maybe normal or predefined, - %% involving the predefined topic name in register to enhance the gateway's robustness even inconsistent with MQTT-SN channels - {keep_state, send_register(TopicName, TopicId, MsgId, State)} + %% notice that this TopicName maybe normal or predefined, + %% involving the predefined topic name in register to + %% enhance the gateway's robustness even inconsistent + %% with MQTT-SN channel + {keep_state, + send_register(TopicName, TopicId, MsgId, State)} end; _ -> + %% XXX: We need to handle others error code + %% 'Rejection: congestion' ?LOG(error, "CAN NOT handle PUBACK ReturnCode=~p", [ReturnCode]), {keep_state, State} end. @@ -1050,7 +1065,7 @@ handle_incoming(Packet, _StName, State) -> channel_handle_in(Packet = ?PACKET(Type), #state{channel = Channel}) -> _ = inc_incoming_stats(Type), ok = emqx_metrics:inc_recv(Packet), - ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]), + ?LOG(debug, "Transed-RECV ~s", [emqx_packet:format(Packet)]), emqx_channel:handle_in(Packet, Channel). handle_outgoing(Packets, State) when is_list(Packets) -> @@ -1064,7 +1079,9 @@ handle_outgoing(PubPkt = ?PUBLISH_PACKET(_, TopicName, _, _), ClientId = emqx_channel:info(clientid, Channel), TopicId = emqx_sn_registry:lookup_topic_id(ClientId, TopicName), case (TopicId == undefined) andalso (byte_size(TopicName) =/= 2) of - true -> register_and_notify_client(PubPkt, State); + true -> + %% TODO: only one REGISTER inflight if qos=0?? + register_and_notify_client(PubPkt, State); false -> send_message(mqtt2sn(PubPkt, State), State) end; @@ -1077,13 +1094,40 @@ cache_no_reg_publish_message(Pendings, TopicId, PubPkt, State) -> Msgs = maps:get(pending_topic_ids, Pendings, []), Pendings#{TopicId => Msgs ++ [mqtt2sn(PubPkt, State)]}. -replay_no_reg_pending_publishes(TopicId, #state{pending_topic_ids = Pendings} = State0) -> - ?LOG(debug, "replay non-registered publish message for topic-id: ~p, pendings: ~0p", - [TopicId, Pendings]), +replay_no_reg_pending_publishes(TopicId, + State0 = #state{ + pending_topic_ids = Pendings}) -> + ?LOG(debug, "replay non-registered qos0 publish message for " + "topic-id: ~p, pendings: ~0p", [TopicId, Pendings]), State = lists:foldl(fun(Msg, State1) -> send_message(Msg, State1) end, State0, maps:get(TopicId, Pendings, [])), - State#state{pending_topic_ids = maps:remove(TopicId, Pendings)}. + + NState = State#state{pending_topic_ids = maps:remove(TopicId, Pendings)}, + case replay_inflight_messages(TopicId, State#state.channel) of + [] -> ok; + Outgoings -> + ?LOG(debug, "replay non-registered qos1/qos2 publish message " + "for topic-id: ~0p, messages: ~0p", + [TopicId, Outgoings]), + handle_outgoing(Outgoings, NState) + end. + +replay_inflight_messages(TopicId, Channel) -> + Inflight = emqx_session:info(inflight, emqx_channel:get_session(Channel)), + + case emqx_inflight:to_list(Inflight) of + [] -> []; + [{PktId, {Msg, _Ts}}] -> %% Fixed inflight size 1 + ClientId = emqx_channel:info(clientid, Channel), + ReplayTopic = emqx_sn_registry:lookup_topic(ClientId, TopicId), + case ReplayTopic =:= emqx_message:topic(Msg) of + false -> []; + true -> + NMsg = emqx_message:set_flag(dup, true, Msg), + [emqx_message:to_packet(PktId, NMsg)] + end + end. register_and_notify_client(?PUBLISH_PACKET(QoS, TopicName, PacketId, Payload) = PubPkt, State = #state{pending_topic_ids = Pendings, channel = Channel}) -> @@ -1091,10 +1135,17 @@ register_and_notify_client(?PUBLISH_PACKET(QoS, TopicName, PacketId, Payload) = #mqtt_packet{header = #mqtt_packet_header{dup = Dup, retain = Retain}} = PubPkt, ClientId = emqx_channel:info(clientid, Channel), TopicId = emqx_sn_registry:register_topic(ClientId, TopicName), - ?LOG(debug, "Register TopicId=~p, TopicName=~p, Payload=~p, Dup=~p, QoS=~p, " - "Retain=~p, MsgId=~p", [TopicId, TopicName, Payload, Dup, QoS, Retain, MsgId]), - NewPendings = cache_no_reg_publish_message(Pendings, TopicId, PubPkt, State), - send_register(TopicName, TopicId, MsgId, State#state{pending_topic_ids = NewPendings}). + ?LOG(debug, "Register TopicId=~p, TopicName=~p, Payload=~p, Dup=~p, " + "QoS=~p,Retain=~p, MsgId=~p", + [TopicId, TopicName, Payload, Dup, QoS, Retain, MsgId]), + NPendings = case QoS == ?QOS_0 of + true -> + cache_no_reg_publish_message( + Pendings, TopicId, PubPkt, State); + _ -> Pendings + end, + send_register(TopicName, TopicId, MsgId, + State#state{pending_topic_ids = NPendings}). message_id(undefined) -> rand:uniform(16#FFFF); diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index 1371c5123..9ff519fb5 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -819,6 +819,151 @@ t_publish_qos2_case03(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). +t_delivery_qos1_register_invalid_topic_id(_) -> + Dup = 0, + QoS = 1, + Retain = 0, + Will = 0, + CleanSession = 0, + MsgId = 1, + TopicId = ?MAX_PRED_TOPIC_ID + 1, + {ok, Socket} = gen_udp:open(0, [binary]), + send_connect_msg(Socket, <<"test">>), + ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), + + send_subscribe_msg_normal_topic(Socket, QoS, <<"ab">>, MsgId), + ?assertEqual(<<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, + ?SN_NORMAL_TOPIC:2, TopicId:16, MsgId:16, ?SN_RC_ACCEPTED>>, + receive_response(Socket)), + + Payload = <<"test-registration-inconsistent">>, + _ = emqx:publish(emqx_message:make(test, ?QOS_1, <<"ab">>, Payload)), + + ?assertEqual( + <<(7 + byte_size(Payload)), ?SN_PUBLISH, + Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, MsgId:16, Payload/binary>>, receive_response(Socket)), + %% acked with ?SN_RC_INVALID_TOPIC_ID to + send_puback_msg(Socket, TopicId, MsgId, ?SN_RC_INVALID_TOPIC_ID), + ?assertEqual( + {TopicId, MsgId}, + check_register_msg_on_udp(<<"ab">>, receive_response(Socket))), + send_regack_msg(Socket, TopicId, MsgId + 1), + + %% receive the replay message + ?assertEqual( + <<(7 + byte_size(Payload)), ?SN_PUBLISH, + Dup:1, QoS:2, Retain:1, + Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2, + TopicId:16, (MsgId):16, Payload/binary>>, receive_response(Socket)), + + send_disconnect_msg(Socket, undefined), + ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + gen_udp:close(Socket). + +t_delivery_takeover_and_re_register(_) -> + MsgId = 1, + {ok, Socket} = gen_udp:open(0, [binary]), + send_connect_msg(Socket, <<"test">>, 0), + ?assertMatch(<<_, ?SN_CONNACK, ?SN_RC_ACCEPTED>>, + receive_response(Socket)), + + send_subscribe_msg_normal_topic(Socket, ?QOS_1, <<"topic-a">>, MsgId+1), + <<_, ?SN_SUBACK, 2#00100000, + TopicIdA:16, _:16, ?SN_RC_ACCEPTED>> = receive_response(Socket), + + send_subscribe_msg_normal_topic(Socket, ?QOS_2, <<"topic-b">>, MsgId+2), + <<_, ?SN_SUBACK, 2#01000000, + TopicIdB:16, _:16, ?SN_RC_ACCEPTED>> = receive_response(Socket), + + _ = emqx:publish( + emqx_message:make(test, ?QOS_1, <<"topic-a">>, <<"test-a">>)), + _ = emqx:publish( + emqx_message:make(test, ?QOS_2, <<"topic-b">>, <<"test-b">>)), + + <<_, ?SN_PUBLISH, 2#00100000, + TopicIdA:16, MsgId1:16, "test-a">> = receive_response(Socket), + send_puback_msg(Socket, TopicIdA, MsgId1, ?SN_RC_ACCEPTED), + + <<_, ?SN_PUBLISH, 2#01000000, + TopicIdB:16, MsgId2:16, "test-b">> = receive_response(Socket), + send_puback_msg(Socket, TopicIdB, MsgId2, ?SN_RC_ACCEPTED), + + send_disconnect_msg(Socket, undefined), + ?assertMatch(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), + gen_udp:close(Socket), + + %% offline messages will be queued into the MQTT-SN session + _ = emqx:publish(emqx_message:make(test, ?QOS_1, <<"topic-a">>, <<"m1">>)), + _ = emqx:publish(emqx_message:make(test, ?QOS_1, <<"topic-a">>, <<"m2">>)), + _ = emqx:publish(emqx_message:make(test, ?QOS_1, <<"topic-a">>, <<"m3">>)), + _ = emqx:publish(emqx_message:make(test, ?QOS_2, <<"topic-b">>, <<"m1">>)), + _ = emqx:publish(emqx_message:make(test, ?QOS_2, <<"topic-b">>, <<"m2">>)), + _ = emqx:publish(emqx_message:make(test, ?QOS_2, <<"topic-b">>, <<"m3">>)), + + {ok, NSocket} = gen_udp:open(0, [binary]), + send_connect_msg(NSocket, <<"test">>, 0), + ?assertMatch(<<_, ?SN_CONNACK, ?SN_RC_ACCEPTED>>, + receive_response(NSocket)), + + %% qos1 + + %% received the resume messages + <<_, ?SN_PUBLISH, 2#00100000, + TopicIdA:16, MsgIdA0:16, "m1">> = receive_response(NSocket), + %% only one qos1/qos2 inflight + ?assertEqual(udp_receive_timeout, receive_response(NSocket)), + send_puback_msg(NSocket, TopicIdA, MsgIdA0, ?SN_RC_INVALID_TOPIC_ID), + %% recv register + <<_, ?SN_REGISTER, + TopicIdA:16, RegMsgIdA:16, "topic-a">> = receive_response(NSocket), + send_regack_msg(NSocket, TopicIdA, RegMsgIdA), + %% received the replay messages + <<_, ?SN_PUBLISH, 2#00100000, + TopicIdA:16, MsgIdA1:16, "m1">> = receive_response(NSocket), + send_puback_msg(NSocket, TopicIdA, MsgIdA1, ?SN_RC_ACCEPTED), + + <<_, ?SN_PUBLISH, 2#00100000, + TopicIdA:16, MsgIdA2:16, "m2">> = receive_response(NSocket), + send_puback_msg(NSocket, TopicIdA, MsgIdA2, ?SN_RC_ACCEPTED), + + <<_, ?SN_PUBLISH, 2#00100000, + TopicIdA:16, MsgIdA3:16, "m3">> = receive_response(NSocket), + send_puback_msg(NSocket, TopicIdA, MsgIdA3, ?SN_RC_ACCEPTED), + + %% qos2 + <<_, ?SN_PUBLISH, 2#01000000, + TopicIdB:16, MsgIdB0:16, "m1">> = receive_response(NSocket), + %% only one qos1/qos2 inflight + ?assertEqual(udp_receive_timeout, receive_response(NSocket)), + send_puback_msg(NSocket, TopicIdB, MsgIdB0, ?SN_RC_INVALID_TOPIC_ID), + %% recv register + <<_, ?SN_REGISTER, + TopicIdB:16, RegMsgIdB:16, "topic-b">> = receive_response(NSocket), + send_regack_msg(NSocket, TopicIdB, RegMsgIdB), + %% received the replay messages + <<_, ?SN_PUBLISH, 2#01000000, + TopicIdB:16, MsgIdB1:16, "m1">> = receive_response(NSocket), + send_pubrec_msg(NSocket, MsgIdB1), + <<_, ?SN_PUBREL, MsgIdB1:16>> = receive_response(NSocket), + send_pubcomp_msg(NSocket, MsgIdB1), + + <<_, ?SN_PUBLISH, 2#01000000, + TopicIdB:16, MsgIdB2:16, "m2">> = receive_response(NSocket), + send_puback_msg(NSocket, TopicIdB, MsgIdB2, ?SN_RC_ACCEPTED), + + <<_, ?SN_PUBLISH, 2#01000000, + TopicIdB:16, MsgIdB3:16, "m3">> = receive_response(NSocket), + send_puback_msg(NSocket, TopicIdB, MsgIdB3, ?SN_RC_ACCEPTED), + + %% no more messages + ?assertEqual(udp_receive_timeout, receive_response(NSocket)), + + send_disconnect_msg(NSocket, undefined), + ?assertMatch(<<2, ?SN_DISCONNECT>>, receive_response(NSocket)), + gen_udp:close(NSocket). + t_will_case01(_) -> QoS = 1, Duration = 1, @@ -1591,13 +1736,16 @@ send_searchgw_msg(Socket) -> ok = gen_udp:send(Socket, ?HOST, ?PORT, <>). send_connect_msg(Socket, ClientId) -> + send_connect_msg(Socket, ClientId, 1). + +send_connect_msg(Socket, ClientId, CleanSession) when CleanSession == 0; + CleanSession == 1 -> Length = 6 + byte_size(ClientId), MsgType = ?SN_CONNECT, Dup = 0, QoS = 0, Retain = 0, Will = 0, - CleanSession = 1, TopicIdType = 0, ProtocolId = 1, Duration = 10, @@ -1713,9 +1861,12 @@ send_publish_msg_short_topic(Socket, QoS, MsgId, TopicName, Data) -> ok = gen_udp:send(Socket, ?HOST, ?PORT, PublishPacket). send_puback_msg(Socket, TopicId, MsgId) -> + send_puback_msg(Socket, TopicId, MsgId, ?SN_RC_ACCEPTED). + +send_puback_msg(Socket, TopicId, MsgId, Rc) -> Length = 7, MsgType = ?SN_PUBACK, - PubAckPacket = <>, + PubAckPacket = <>, ?LOG("send_puback_msg TopicId=~p, MsgId=~p", [TopicId, MsgId]), ok = gen_udp:send(Socket, ?HOST, ?PORT, PubAckPacket). From 76d0835e4fdff866d4d18f63b4210b557896f6ab Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 10 Feb 2022 14:29:05 +0800 Subject: [PATCH 34/47] chore(mqttsn): update appup.src --- apps/emqx_sn/src/emqx_sn.app.src | 2 +- apps/emqx_sn/src/emqx_sn.appup.src | 20 ++++++++------------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/apps/emqx_sn/src/emqx_sn.app.src b/apps/emqx_sn/src/emqx_sn.app.src index ef0d24bf9..319137fed 100644 --- a/apps/emqx_sn/src/emqx_sn.app.src +++ b/apps/emqx_sn/src/emqx_sn.app.src @@ -1,6 +1,6 @@ {application, emqx_sn, [{description, "EMQ X MQTT-SN Plugin"}, - {vsn, "4.3.5"}, % strict semver, bump manually! + {vsn, "4.3.6"}, % strict semver, bump manually! {modules, []}, {registered, []}, {applications, [kernel,stdlib,esockd]}, diff --git a/apps/emqx_sn/src/emqx_sn.appup.src b/apps/emqx_sn/src/emqx_sn.appup.src index 749a72956..6a4eb66d1 100644 --- a/apps/emqx_sn/src/emqx_sn.appup.src +++ b/apps/emqx_sn/src/emqx_sn.appup.src @@ -1,29 +1,25 @@ %% -*- mode: erlang -*- {VSN, [ - {"4.3.4",[ + {<<"4\\.3\\.[4-5]">>,[ + {load_module,emqx_sn_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} ]}, - {"4.3.3",[ - {load_module,emqx_sn_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} - ]}, - {"4.3.2", [ + {<<"4.3.[2-3]">>,[ {load_module,emqx_sn_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_sn_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} ]}, {<<"4\\.3\\.[0-1]">>, [{restart_application,emqx_sn}]} ], [ - {"4.3.4",[ + {<<"4\\.3\\.[4-5]">>,[ + {load_module,emqx_sn_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} ]}, - {"4.3.3",[ - {load_module,emqx_sn_registry,brutal_purge,soft_purge,[]}, - {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} - ]}, - {"4.3.2", [ + {<<"4.3.[2-3]">>,[ {load_module,emqx_sn_registry,brutal_purge,soft_purge,[]}, + {load_module,emqx_sn_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_sn_gateway,brutal_purge,soft_purge,[]} ]}, {<<"4\\.3\\.[0-1]">>, [{restart_application,emqx_sn}]} From 2c512c20976ddc36555f090f453d17cb10a423c6 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 15 Feb 2022 10:01:57 +0800 Subject: [PATCH 35/47] docs: update CHANGES-4.3.md --- CHANGES-4.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 148627628..dbbd94a6b 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -31,6 +31,7 @@ File format: * Fix Server-KeepAlive wrongly applied on MQTT v3.0/v3.1 [#7085] * Fix Stomp client can not trigger `$event/client_connection` message [#7096] * Fix system memory false alarm at boot +* Fix the MQTT-SN message replay when the topic is not registered to the client [#6970] ## v4.3.12 ### Important changes From bfd0fd901932ee23d7aba0d7b4a986f208bafaf9 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Tue, 22 Feb 2022 10:16:29 +0800 Subject: [PATCH 36/47] refactor(cm): rename call_or_kill to takeover --- src/emqx_cm.erl | 117 +++++++++++++++++++++++++----------------------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 4811313d8..002b1a746 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -235,13 +235,14 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> case takeover_session(ClientId) of {ok, ConnMod, ChanPid, Session} -> ok = emqx_session:resume(ClientInfo, Session), - case call_or_kill({takeover, 'end'}, ConnMod, ChanPid) of - {error, _} -> CreateSess(); - Pendings -> + case takeover('end', ConnMod, ChanPid) of + {ok, Pendings} -> register_channel(ClientId, Self, ConnInfo), {ok, #{session => Session, present => true, - pendings => Pendings}} + pendings => Pendings}}; + {error, _} -> + CreateSess() end; {error, _Reason} -> CreateSess() end @@ -277,11 +278,11 @@ takeover_session(ClientId, ChanPid) when node(ChanPid) == node() -> undefined -> {error, not_found}; ConnMod when is_atom(ConnMod) -> - case call_or_kill({takeover, 'begin'}, ConnMod, ChanPid) of + case takeover('begin', ConnMod, ChanPid) of + {ok, Session} -> + {ok, ConnMod, ChanPid, Session}; {error, Reason} -> - {error, Reason}; - Session -> - {ok, ConnMod, ChanPid, Session} + {error, Reason} end end; takeover_session(ClientId, ChanPid) -> @@ -299,59 +300,61 @@ discard_session(ClientId) when is_binary(ClientId) -> %% If failed to response (e.g. timeout) force a kill. %% Keeping the stale pid around, or returning error or raise an exception %% benefits nobody. --spec call_or_kill(Action, module(), pid()) - -> ok %% returned by kick, discard - | emqx_session:session() %% returned by {takeover, 'begin'} - | list(emqx_type:deliver()) %% returned by {takeover, 'end'} - when Action :: kick | discard | {takeover, 'begin'} | {takeover, 'end'}. -call_or_kill(Action, ConnMod, Pid) -> - try - Timeout = case Action of - {takeover, _} -> ?T_TAKEOVER; - _ -> ?T_KICK - end, +-spec takeover(Action, module(), pid()) + -> ok + | {ok, emqx_session:session() | list(emqx_type:deliver())} + | {error, term()} + when Action :: kick | discard | 'begin' | 'end'. +takeover(Action, ConnMod, Pid) -> + {NAction, Timeout} = + case Action == kick orelse Action == discard of + true -> {Action, ?T_KICK}; + _ -> {{takeover, Action},?T_TAKEOVER} + end, + Return = %% this is essentailly a gen_server:call implemented in emqx_connection %% and emqx_ws_connection. %% the handle_call is implemented in emqx_channel - apply(ConnMod, call, [Pid, Action, Timeout]) - catch - _ : noproc -> % emqx_ws_connection: call - ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}), - return_error_if_action_is_takeover(Action, noproc); - _ : {noproc, _} -> % emqx_connection: gen_server:call - ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}), - return_error_if_action_is_takeover(Action, noproc); - _ : Reason = {shutdown, _} -> - ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), - return_error_if_action_is_takeover(Action, Reason); - _ : Reason = {{shutdown, _}, _} -> - ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), - return_error_if_action_is_takeover(Action, Reason); - _ : {timeout, {gen_server, call, _}} -> - ?tp(warning, "call_session_timeout", - #{pid => Pid, - action => Action, - stale_channel => stale_channel_info(Pid) - }), - ok = force_kill(Pid), - return_error_if_action_is_takeover(Action, timeout); - _ : Error : St -> - ?tp(error, "call_session_exception", - #{pid => Pid, - action => Action, - reason => Error, - stacktrace => St, - stale_channel => stale_channel_info(Pid) - }), - ok = force_kill(Pid), - return_error_if_action_is_takeover(Action, Error) + try apply(ConnMod, call, [Pid, NAction, Timeout]) of + ok -> ok; + Reply -> {ok, Reply} + catch + _ : noproc -> % emqx_ws_connection: call + ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}), + {error, noproc}; + _ : {noproc, _} -> % emqx_connection: gen_server:call + ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}), + {error, noproc}; + _ : Reason = {shutdown, _} -> + ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), + {error, Reason}; + _ : Reason = {{shutdown, _}, _} -> + ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), + {error, Reason}; + _ : {timeout, {gen_server, call, _}} -> + ?tp(warning, "takeover_session_timeout", + #{pid => Pid, + action => Action, + stale_channel => stale_channel_info(Pid) + }), + ok = force_kill(Pid), + {error, timeout}; + _ : Error : St -> + ?tp(error, "takeover_session_exception", + #{pid => Pid, + action => Action, + reason => Error, + stacktrace => St, + stale_channel => stale_channel_info(Pid) + }), + ok = force_kill(Pid), + {error, Error} + end, + case Action == kick orelse Action == discard of + true -> ok; + _ -> Return end. -return_error_if_action_is_takeover({takeover, _}, Reason) -> - {error, Reason}; -return_error_if_action_is_takeover(_, _) -> - ok. - force_kill(Pid) -> exit(Pid, kill), ok. @@ -372,7 +375,7 @@ kick_session(Action, ClientId, ChanPid) when node(ChanPid) == node() -> %% already deregistered ok; ConnMod when is_atom(ConnMod) -> - ok = call_or_kill(Action, ConnMod, ChanPid) + ok = takeover(Action, ConnMod, ChanPid) end; kick_session(Action, ClientId, ChanPid) -> %% call remote node on the old APIs because we do not know if they have upgraded From 4673ca43a0f7838a09fdfb62297aa141e47e4281 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 16 Feb 2022 15:49:59 +0800 Subject: [PATCH 37/47] chore(emqx): update appup.src --- src/emqx.appup.src | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 610b40c75..6e9c43d01 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -6,6 +6,7 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -17,6 +18,7 @@ {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, @@ -34,6 +36,7 @@ {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -54,6 +57,7 @@ {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, @@ -374,6 +378,8 @@ {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.12", [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -382,6 +388,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -398,6 +405,7 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, @@ -417,6 +425,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, From bf565efc99a5506ba68a79a8c07579cc1897edd5 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 16 Feb 2022 19:55:01 +0800 Subject: [PATCH 38/47] chore: update CHANGES-4.3.md --- CHANGES-4.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 148627628..af96c295a 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -22,6 +22,7 @@ File format: * CLI `emqx_ctl pem_cache clean` to force purge x509 certificate cache, to force an immediate reload of all certificates after the files are updated on disk. * Refactor the ExProto so that anonymous clients can also be displayed on the dashboard [#6983] +* Force shutdown of processe that cannot answer takeover event [#7026] ### Bug fixes From 72362063730cd655bd214d2443cf22fcc66cd0e2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 17 Feb 2022 09:35:30 +0800 Subject: [PATCH 39/47] fix(exproto): handle discard call --- apps/emqx_exproto/src/emqx_exproto_channel.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 130ad155a..527f2e9b7 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -366,6 +366,9 @@ handle_call({publish, Topic, Qos, Payload}, handle_call(kick, Channel) -> {shutdown, kicked, ok, Channel}; +handle_call(discard, Channel) -> + {shutdown, discarded, ok, Channel}; + handle_call(Req, Channel) -> ?LOG(warning, "Unexpected call: ~p", [Req]), {reply, {error, unexpected_call}, Channel}. From d943cc2f1cf940aab4cf4e803e5515dee6f1eb1e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 17 Feb 2022 09:57:55 +0800 Subject: [PATCH 40/47] chore(exproto): update appup.src --- apps/emqx_exproto/src/emqx_exproto.appup.src | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index 590a2106f..e66e8a58c 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,18 +1,22 @@ %% -*- mode: erlang -*- {VSN, - [{<<"4\\.3\\.[2-5]">>, + [{<<"4\\.3\\.[4-5]">>, + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[2-3]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4.3.[0-1]">>, + {<<"4\\.3\\.[0-1]">>, [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[2-5]">>, + [{<<"4\\.3\\.[4-5]">>, + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[2-3]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4.3.[0-1]">>, + {<<"4\\.3\\.[0-1]">>, [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, From 7dbb5b1032e203179950280b5f32df2aa966dad8 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 24 Feb 2022 12:52:57 +0800 Subject: [PATCH 41/47] fix(CI): don't guarantee not to lose a single message during relup --- .ci/fvt_tests/relup.lux | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 2940f5ce0..9a6ba45e9 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -136,17 +136,20 @@ !./bin/emqx_ctl broker metrics | grep "messages.publish" ???SH-PROMPT +## We don't guarantee not to lose a single message! +## So even if we received 290~300 messages, we consider it as success [shell bench] !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].metrics[] | select(.node==\"emqx@127.0.0.1\").matched" - ?300 + ?(29[0-9])|(300) ?SH-PROMPT !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].actions[0].metrics[] | select(.node==\"emqx@127.0.0.1\").success" - ?300 + ?(29[0-9])|(300) ?SH-PROMPT + ## The /counter API is provided by .ci/fvt_test/http_server !curl http://127.0.0.1:8080/counter - ???{"data":300,"code":0} + ?\{"data":(29[0-9])|(300),"code":0\} ?SH-PROMPT [shell emqx2] From d622eb848faa6dcf7d765896bded62dee92ce1ba Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 24 Feb 2022 14:43:46 +0800 Subject: [PATCH 42/47] refactor(cm): rename takeover -> request_stepdown see: https://github.com/emqx/emqx/pull/7026#pullrequestreview-891954135 --- apps/emqx_exproto/src/emqx_exproto.appup.src | 12 +++++----- src/emqx_cm.erl | 24 ++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index e66e8a58c..adeeeab3f 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,22 +1,22 @@ %% -*- mode: erlang -*- {VSN, - [{<<"4\\.3\\.[4-5]">>, + [{<<"4\\.3\\.[4-5]$">>, [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]">>, + {<<"4\\.3\\.[2-3]$">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-1]">>, + {<<"4\\.3\\.[0-1]$">>, [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[4-5]">>, + [{<<"4\\.3\\.[4-5]$">>, [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]">>, + {<<"4\\.3\\.[2-3]$">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-1]">>, + {<<"4\\.3\\.[0-1]$">>, [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl index 002b1a746..3a12b5a24 100644 --- a/src/emqx_cm.erl +++ b/src/emqx_cm.erl @@ -235,7 +235,7 @@ open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> case takeover_session(ClientId) of {ok, ConnMod, ChanPid, Session} -> ok = emqx_session:resume(ClientInfo, Session), - case takeover('end', ConnMod, ChanPid) of + case request_stepdown({takeover, 'end'}, ConnMod, ChanPid) of {ok, Pendings} -> register_channel(ClientId, Self, ConnInfo), {ok, #{session => Session, @@ -278,7 +278,7 @@ takeover_session(ClientId, ChanPid) when node(ChanPid) == node() -> undefined -> {error, not_found}; ConnMod when is_atom(ConnMod) -> - case takeover('begin', ConnMod, ChanPid) of + case request_stepdown({takeover, 'begin'}, ConnMod, ChanPid) of {ok, Session} -> {ok, ConnMod, ChanPid, Session}; {error, Reason} -> @@ -300,22 +300,22 @@ discard_session(ClientId) when is_binary(ClientId) -> %% If failed to response (e.g. timeout) force a kill. %% Keeping the stale pid around, or returning error or raise an exception %% benefits nobody. --spec takeover(Action, module(), pid()) +-spec request_stepdown(Action, module(), pid()) -> ok | {ok, emqx_session:session() | list(emqx_type:deliver())} | {error, term()} - when Action :: kick | discard | 'begin' | 'end'. -takeover(Action, ConnMod, Pid) -> - {NAction, Timeout} = + when Action :: kick | discard | {takeover, 'begin'} | {takeover, 'end'}. +request_stepdown(Action, ConnMod, Pid) -> + Timeout = case Action == kick orelse Action == discard of - true -> {Action, ?T_KICK}; - _ -> {{takeover, Action},?T_TAKEOVER} + true -> ?T_KICK; + _ -> ?T_TAKEOVER end, Return = %% this is essentailly a gen_server:call implemented in emqx_connection %% and emqx_ws_connection. %% the handle_call is implemented in emqx_channel - try apply(ConnMod, call, [Pid, NAction, Timeout]) of + try apply(ConnMod, call, [Pid, Action, Timeout]) of ok -> ok; Reply -> {ok, Reply} catch @@ -332,7 +332,7 @@ takeover(Action, ConnMod, Pid) -> ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}), {error, Reason}; _ : {timeout, {gen_server, call, _}} -> - ?tp(warning, "takeover_session_timeout", + ?tp(warning, "session_stepdown_request_timeout", #{pid => Pid, action => Action, stale_channel => stale_channel_info(Pid) @@ -340,7 +340,7 @@ takeover(Action, ConnMod, Pid) -> ok = force_kill(Pid), {error, timeout}; _ : Error : St -> - ?tp(error, "takeover_session_exception", + ?tp(error, "session_stepdown_request_exception", #{pid => Pid, action => Action, reason => Error, @@ -375,7 +375,7 @@ kick_session(Action, ClientId, ChanPid) when node(ChanPid) == node() -> %% already deregistered ok; ConnMod when is_atom(ConnMod) -> - ok = takeover(Action, ConnMod, ChanPid) + ok = request_stepdown(Action, ConnMod, ChanPid) end; kick_session(Action, ClientId, ChanPid) -> %% call remote node on the old APIs because we do not know if they have upgraded From 63fbf8d9f8159118ce213307d877a48c4759da93 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 24 Feb 2022 12:54:07 +0800 Subject: [PATCH 43/47] fix(CI): add cases for relup to downgrade to old versions --- .ci/fvt_tests/relup.lux | 46 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 9a6ba45e9..ce3b014fa 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -18,8 +18,9 @@ ?> [shell emqx] + !OLD_VSN=$(echo $OLD_VSN | sed -r 's/[v|e]//g') !cd $PACKAGE_PATH - !unzip -q -o $PROFILE-ubuntu20.04-$(echo $OLD_VSN | sed -r 's/[v|e]//g')-amd64.zip + !unzip -q -o $PROFILE-ubuntu20.04-$${OLD_VSN}-amd64.zip ?SH-PROMPT !cd emqx @@ -30,6 +31,7 @@ ?SH-PROMPT [shell emqx2] + !OLD_VSN=$(echo $OLD_VSN | sed -r 's/[v|e]//g') !cd $PACKAGE_PATH !cp -f $ONE_MORE_EMQX_PATH/one_more_$(echo $PROFILE | sed 's/-/_/g').sh . !./one_more_$(echo $PROFILE | sed 's/-/_/g').sh emqx2 @@ -82,6 +84,27 @@ !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + ## upgrade to the new version + !./bin/emqx install $VSN + ?Made release permanent: "$VSN" + ?SH-PROMPT + + !./bin/emqx versions |grep permanent + ?(.*)$VSN + ?SH-PROMPT + + ## downgrade to the old version + !./bin/emqx install $${OLD_VSN} + ?Made release permanent:.* + ?SH-PROMPT + + !./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}" + ?SH-PROMPT: + !echo ==$$?== + ?^==0== + ?SH-PROMPT: + + ## again, upgrade to the new version !./bin/emqx install $VSN ?Made release permanent: "$VSN" ?SH-PROMPT @@ -107,6 +130,27 @@ !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/ + ## upgrade to the new version + !./bin/emqx install $VSN + ?Made release permanent: "$VSN" + ?SH-PROMPT + + !./bin/emqx versions |grep permanent + ?(.*)$VSN + ?SH-PROMPT + + ## downgrade to the old version + !./bin/emqx install $${OLD_VSN} + ?Made release permanent:.* + ?SH-PROMPT + + !./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}" + ?SH-PROMPT: + !echo ==$$?== + ?^==0== + ?SH-PROMPT: + + ## again, upgrade to the new version !./bin/emqx install $VSN ?Made release permanent: "$VSN" ?SH-PROMPT From 815096063a76e6d577b84fa21e442c5532967c0e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 24 Feb 2022 16:05:55 +0800 Subject: [PATCH 44/47] chore: update the change logs --- CHANGES-4.3.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index e38cb1b0f..d9fe5c7f6 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -25,6 +25,7 @@ File format: ### Bug fixes +* Fix the `{error,eexist}` error when do release upgrade again if last run failed. [#7121] * Fix case where publishing to a non-existent topic alias would crash the connection [#6979] * Fix HTTP-API 500 error on querying the lwm2m client list on the another node [#7009] * Fix the ExProto connection registry is not released after the client process abnormally exits [#6983] From f121f9616e59f7f711f84498e1fa5a8948d76150 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 24 Feb 2022 15:57:44 +0800 Subject: [PATCH 45/47] chore: rename testcase name --- apps/emqx_exproto/src/emqx_exproto.appup.src | 12 +-- test/emqx_cm_SUITE.erl | 82 ++++++++++---------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index adeeeab3f..e66e8a58c 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,22 +1,22 @@ %% -*- mode: erlang -*- {VSN, - [{<<"4\\.3\\.[4-5]$">>, + [{<<"4\\.3\\.[4-5]">>, [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]$">>, + {<<"4\\.3\\.[2-3]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-1]$">>, + {<<"4\\.3\\.[0-1]">>, [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{<<"4\\.3\\.[4-5]$">>, + [{<<"4\\.3\\.[4-5]">>, [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]$">>, + {<<"4\\.3\\.[2-3]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[0-1]$">>, + {<<"4\\.3\\.[0-1]">>, [{load_module,emqx_exproto_gsvr,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_gcli,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl index c82e525f0..4fcaaf473 100644 --- a/test/emqx_cm_SUITE.erl +++ b/test/emqx_cm_SUITE.erl @@ -187,67 +187,67 @@ t_open_session_race_condition(_) -> ok = flush_emqx_pool(), ?assertEqual([], emqx_cm:lookup_channels(ClientId)). -t_call_session_discard_normal(_) -> - test_call_session(discard, normal). +t_stepdown_sessiondiscard_normal(_) -> + test_stepdown_session(discard, normal). -t_call_session_discard_shutdown(_) -> - test_call_session(discard, shutdown). +t_stepdown_sessiondiscard_shutdown(_) -> + test_stepdown_session(discard, shutdown). -t_call_session_discard_shutdown_with_reason(_) -> - test_call_session(discard, {shutdown, discard}). +t_stepdown_sessiondiscard_shutdown_with_reason(_) -> + test_stepdown_session(discard, {shutdown, discard}). -t_call_session_discard_timeout(_) -> - test_call_session(discard, timeout). +t_stepdown_sessiondiscard_timeout(_) -> + test_stepdown_session(discard, timeout). -t_call_session_discard_noproc(_) -> - test_call_session(discard, noproc). +t_stepdown_sessiondiscard_noproc(_) -> + test_stepdown_session(discard, noproc). -t_call_session_kick_normal(_) -> - test_call_session(kick, normal). +t_stepdown_sessionkick_normal(_) -> + test_stepdown_session(kick, normal). -t_call_session_kick_shutdown(_) -> - test_call_session(kick, shutdown). +t_stepdown_sessionkick_shutdown(_) -> + test_stepdown_session(kick, shutdown). -t_call_session_kick_shutdown_with_reason(_) -> - test_call_session(kick, {shutdown, discard}). +t_stepdown_sessionkick_shutdown_with_reason(_) -> + test_stepdown_session(kick, {shutdown, discard}). -t_call_session_kick_timeout(_) -> - test_call_session(kick, timeout). +t_stepdown_sessionkick_timeout(_) -> + test_stepdown_session(kick, timeout). -t_call_session_kick_noproc(_) -> - test_call_session(discard, noproc). +t_stepdown_sessionkick_noproc(_) -> + test_stepdown_session(discard, noproc). -t_call_session_takeover_begin_normal(_) -> - test_call_session({takeover, 'begin'}, normal). +t_stepdown_sessiontakeover_begin_normal(_) -> + test_stepdown_session({takeover, 'begin'}, normal). -t_call_session_takeover_begin_shutdown(_) -> - test_call_session({takeover, 'begin'}, shutdown). +t_stepdown_sessiontakeover_begin_shutdown(_) -> + test_stepdown_session({takeover, 'begin'}, shutdown). -t_call_session_takeover_begin_shutdown_with_reason(_) -> - test_call_session({takeover, 'begin'}, {shutdown, discard}). +t_stepdown_sessiontakeover_begin_shutdown_with_reason(_) -> + test_stepdown_session({takeover, 'begin'}, {shutdown, discard}). -t_call_session_takeover_begin_timeout(_) -> - test_call_session({takeover, 'begin'}, timeout). +t_stepdown_sessiontakeover_begin_timeout(_) -> + test_stepdown_session({takeover, 'begin'}, timeout). -t_call_session_takeover_begin_noproc(_) -> - test_call_session({takeover, 'begin'}, noproc). +t_stepdown_sessiontakeover_begin_noproc(_) -> + test_stepdown_session({takeover, 'begin'}, noproc). -t_call_session_takeover_end_normal(_) -> - test_call_session({takeover, 'end'}, normal). +t_stepdown_sessiontakeover_end_normal(_) -> + test_stepdown_session({takeover, 'end'}, normal). -t_call_session_takeover_end_shutdown(_) -> - test_call_session({takeover, 'end'}, shutdown). +t_stepdown_sessiontakeover_end_shutdown(_) -> + test_stepdown_session({takeover, 'end'}, shutdown). -t_call_session_takeover_end_shutdown_with_reason(_) -> - test_call_session({takeover, 'end'}, {shutdown, discard}). +t_stepdown_sessiontakeover_end_shutdown_with_reason(_) -> + test_stepdown_session({takeover, 'end'}, {shutdown, discard}). -t_call_session_takeover_end_timeout(_) -> - test_call_session({takeover, 'end'}, timeout). +t_stepdown_sessiontakeover_end_timeout(_) -> + test_stepdown_session({takeover, 'end'}, timeout). -t_call_session_takeover_end_noproc(_) -> - test_call_session({takeover, 'end'}, noproc). +t_stepdown_sessiontakeover_end_noproc(_) -> + test_stepdown_session({takeover, 'end'}, noproc). -test_call_session(Action, Reason) -> +test_stepdown_session(Action, Reason) -> ClientId = rand_client_id(), #{conninfo := ConnInfo} = ?ChanInfo, FakeSessionFun = From 6b93127ecb04b6b92cfdba19c10e5e0bf2af60b8 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 24 Feb 2022 14:58:53 -0300 Subject: [PATCH 46/47] fix(sys_mon): guard before calling `erlang:port_info` Port of #7108 . Sometimes, the `emqx_sys_mon:procinfo/1` might be called with something that is not a port, like `[]`. Not sure on the conditions for this to happen. ``` 2022-02-18T20:05:02.671592+00:00 [error] Generic server emqx_sys_mon terminating. Reason: {badarg,[{erlang,port_info,[[]],[{error_info,#{module => erl_erts_errors}}]},{emqx_sys_mon,portinfo,1,[{file,"/emqx/apps/emqx/src/emqx_sys_mon.erl"},{line,205}]},{emqx_sys_mon,'-handle_info/2-fun-5-',2,[{file,"/emqx/apps/emqx/src/emqx_sys_mon.erl"},{line,150}]},{emqx_sys_mon,suppress,3,[{file,"/emqx/apps/emqx/src/emqx_sys_mon.erl"},{line,184}]},{gen_server,try_dispatch,4,[{file,"gen_server.erl"},{line,695}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,771}]},{proc_lib,wake_up,3,[{file,"proc_lib.erl"},{line,236}]}]}. Last message: {monitor,<0.7796.0>,busy_dist_port,[]}. State: #{events => [{busy_dist_port,#Port<0.127>}],timer => #Ref<0.2758388682.1853620226.133920>}. ``` --- src/emqx_sys_mon.erl | 12 +++++++++--- test/emqx_sys_mon_SUITE.erl | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/emqx_sys_mon.erl b/src/emqx_sys_mon.erl index 710895bcd..4f07f0b18 100644 --- a/src/emqx_sys_mon.erl +++ b/src/emqx_sys_mon.erl @@ -119,7 +119,7 @@ handle_info({monitor, Port, long_schedule, Info}, State) when is_port(Port) -> suppress({long_schedule, Port}, fun() -> WarnMsg = io_lib:format("long_schedule warning: port = ~p, info: ~p", [Port, Info]), - ?LOG(warning, "~s~n~p", [WarnMsg, erlang:port_info(Port)]), + ?LOG(warning, "~s~n~p", [WarnMsg, portinfo(Port)]), safe_publish(long_schedule, WarnMsg) end, State); @@ -135,7 +135,7 @@ handle_info({monitor, SusPid, busy_port, Port}, State) -> suppress({busy_port, Port}, fun() -> WarnMsg = io_lib:format("busy_port warning: suspid = ~p, port = ~p", [SusPid, Port]), - ?LOG(warning, "~s~n~p~n~p", [WarnMsg, procinfo(SusPid), erlang:port_info(Port)]), + ?LOG(warning, "~s~n~p~n~p", [WarnMsg, procinfo(SusPid), portinfo(Port)]), safe_publish(busy_port, WarnMsg) end, State); @@ -143,7 +143,7 @@ handle_info({monitor, SusPid, busy_dist_port, Port}, State) -> suppress({busy_dist_port, Port}, fun() -> WarnMsg = io_lib:format("busy_dist_port warning: suspid = ~p, port = ~p", [SusPid, Port]), - ?LOG(warning, "~s~n~p~n~p", [WarnMsg, procinfo(SusPid), erlang:port_info(Port)]), + ?LOG(warning, "~s~n~p~n~p", [WarnMsg, procinfo(SusPid), portinfo(Port)]), safe_publish(busy_dist_port, WarnMsg) end, State); @@ -200,3 +200,9 @@ safe_publish(Event, WarnMsg) -> sysmon_msg(Topic, Payload) -> Msg = emqx_message:make(?SYSMON, Topic, Payload), emqx_message:set_flag(sys, Msg). + +portinfo(Port) -> + case is_port(Port) andalso erlang:port_info(Port) of + L when is_list(L) -> L; + _ -> [] + end. diff --git a/test/emqx_sys_mon_SUITE.erl b/test/emqx_sys_mon_SUITE.erl index 63e7c376a..e177e9527 100644 --- a/test/emqx_sys_mon_SUITE.erl +++ b/test/emqx_sys_mon_SUITE.erl @@ -33,6 +33,11 @@ {self(), busy_port, concat_str("busy_port warning: suspid = ~p, port = ~p", self(), list_to_port("#Port<0.4>")), list_to_port("#Port<0.4>")}, + %% for the case when the port is missing, for some + %% reason. + {self(), busy_port, + concat_str("busy_port warning: suspid = ~p, port = ~p", + self(), []), []}, {self(), busy_dist_port, concat_str("busy_dist_port warning: suspid = ~p, port = ~p", self(), list_to_port("#Port<0.4>")),list_to_port("#Port<0.4>")}, @@ -122,6 +127,16 @@ t_sys_mon(_Config) -> validate_sys_mon_info(PidOrPort, SysMonName,ValidateInfo, InfoOrPort) end, ?INPUTINFO). +%% Existing port, but closed. +t_sys_mon_dead_port(_Config) -> + process_flag(trap_exit, true), + Port = dead_port(), + {PidOrPort, SysMonName, ValidateInfo, InfoOrPort} = + {self(), busy_port, + concat_str("busy_port warning: suspid = ~p, port = ~p", + self(), Port), Port}, + validate_sys_mon_info(PidOrPort, SysMonName, ValidateInfo, InfoOrPort). + t_sys_mon2(_Config) -> ?SYSMON ! {timeout, ignored, reset}, ?SYSMON ! {ignored}, @@ -155,3 +170,8 @@ some_function(Parent, _Arg2) -> stop -> ok end. + +dead_port() -> + Port = erlang:open_port({spawn, "ls"}, []), + exit(Port, kill), + Port. From cf1cc7e9db497dfb825a08dc4b113c087537af64 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 24 Feb 2022 17:14:47 -0300 Subject: [PATCH 47/47] chore(appup): update appup files --- apps/emqx_exproto/src/emqx_exproto.appup.src | 6 ++++-- apps/emqx_rule_engine/src/emqx_rule_engine.appup.src | 6 ++++-- src/emqx.appup.src | 12 ++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index e66e8a58c..4b58335b4 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- {VSN, [{<<"4\\.3\\.[4-5]">>, - [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[2-3]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, @@ -12,7 +13,8 @@ {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{<<"4\\.3\\.[4-5]">>, - [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, + {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[2-3]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index c6990ceb7..7a758dcc4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- {VSN, [{"4.3.6", - [{update,emqx_rule_metrics,{advanced,["4.3.6"]}}, + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {update,emqx_rule_metrics,{advanced,["4.3.6"]}}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, @@ -58,7 +59,8 @@ {load_module,emqx_rule_engine_api,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.6", - [{update,emqx_rule_metrics,{advanced,["4.3.6"]}}, + [{load_module,emqx_rule_registry,brutal_purge,soft_purge,[]}, + {update,emqx_rule_metrics,{advanced,["4.3.6"]}}, {load_module,emqx_rule_events,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_engine,brutal_purge,soft_purge,[]}, {load_module,emqx_rule_runtime,brutal_purge,soft_purge,[]}, diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 6e9c43d01..dc045e08a 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -1,7 +1,8 @@ %% -*- mode: erlang -*- {VSN, [{"4.3.13", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -9,7 +10,8 @@ {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, @@ -374,7 +376,8 @@ {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.13", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, @@ -382,7 +385,8 @@ {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},