diff --git a/.ci/docker-compose-file/redis/.gitignore b/.ci/docker-compose-file/redis/.gitignore
new file mode 100644
index 000000000..b5947692d
--- /dev/null
+++ b/.ci/docker-compose-file/redis/.gitignore
@@ -0,0 +1,3 @@
+r7000i.log
+r7001i.log
+r7002i.log
diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml
new file mode 100644
index 000000000..177fdf6b8
--- /dev/null
+++ b/.github/actions/package-macos/action.yaml
@@ -0,0 +1,91 @@
+name: 'Create MacOS package'
+inputs:
+ profile: # emqx, emqx-enterprise
+ required: true
+ type: string
+ otp: # 24.2.1-1, 23.3.4.9-3
+ required: true
+ type: string
+ os:
+ required: false
+ type: string
+ default: macos-11
+ apple_id_password:
+ required: true
+ type: string
+ apple_developer_identity:
+ required: true
+ type: string
+ apple_developer_id_bundle:
+ required: true
+ type: string
+ apple_developer_id_bundle_password:
+ required: true
+ type: string
+
+runs:
+ using: composite
+ steps:
+ - name: prepare
+ shell: bash
+ run: |
+ brew update
+ brew install curl zip unzip kerl coreutils openssl@1.1
+ echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
+ echo "/usr/local/bin" >> $GITHUB_PATH
+ - uses: actions/cache@v2
+ id: cache
+ with:
+ path: ~/.kerl/${{ inputs.otp }}
+ key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit
+ - name: build erlang
+ if: steps.cache.outputs.cache-hit != 'true'
+ shell: bash
+ env:
+ KERL_BUILD_BACKEND: git
+ OTP_GITHUB_URL: https://github.com/emqx/otp
+ KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
+ run: |
+ kerl update releases
+ kerl build ${{ inputs.otp }}
+ kerl install ${{ inputs.otp }} $HOME/.kerl/${{ inputs.otp }}
+ - name: build ${{ inputs.profile }}
+ env:
+ AUTO_INSTALL_BUILD_DEPS: 1
+ APPLE_SIGN_BINARIES: 1
+ APPLE_ID: developers@emqx.io
+ APPLE_TEAM_ID: 26N6HYJLZA
+ APPLE_ID_PASSWORD: ${{ inputs.apple_id_password }}
+ APPLE_DEVELOPER_IDENTITY: ${{ inputs.apple_developer_identity }}
+ APPLE_DEVELOPER_ID_BUNDLE: ${{ inputs.apple_developer_id_bundle }}
+ APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }}
+ shell: bash
+ run: |
+ . $HOME/.kerl/${{ inputs.otp }}/activate
+ make ensure-rebar3
+ sudo cp rebar3 /usr/local/bin/rebar3
+ make ${{ inputs.profile }}-tgz
+ - name: test ${{ inputs.profile }}
+ shell: bash
+ run: |
+ pkg_name=$(find _packages/${{ inputs.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
+ mkdir emqx
+ unzip -d emqx $pkg_name > /dev/null
+ # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
+ ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
+ ready='no'
+ for i in {1..30}; do
+ if curl -fs 127.0.0.1:18083/status > /dev/null; then
+ ready='yes'
+ break
+ fi
+ sleep 1
+ done
+ if [ "$ready" != "yes" ]; then
+ echo "Timed out waiting for emqx to be ready"
+ cat emqx/log/erlang.log.1
+ exit 1
+ fi
+ ./emqx/bin/emqx_ctl status
+ ./emqx/bin/emqx stop
+ rm -rf emqx
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index b6e9f2e8c..9c7fae257 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -150,72 +150,26 @@ jobs:
name: source
path: .
- name: unzip source code
- run: unzip -q source.zip
+ run: |
+ ln -s . source
+ unzip -q source.zip
+ rm source source.zip
- name: prepare
run: |
- brew update
- brew install curl zip unzip kerl coreutils
- echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
- echo "/usr/local/bin" >> $GITHUB_PATH
git config --global credential.helper store
- - uses: actions/cache@v2
- id: cache
+ - uses: ./.github/actions/package-macos
with:
- path: ~/.kerl/${{ matrix.otp }}
- key: otp-install-${{ matrix.otp }}-${{ matrix.os }}
- - name: build erlang
- if: steps.cache.outputs.cache-hit != 'true'
- timeout-minutes: 60
- env:
- KERL_BUILD_BACKEND: git
- OTP_GITHUB_URL: https://github.com/emqx/otp
- run: |
- kerl update releases
- kerl build ${{ matrix.otp }}
- kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
-
- - name: build
- working-directory: source
- env:
- AUTO_INSTALL_BUILD_DEPS: 1
- APPLE_SIGN_BINARIES: 1
- APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
- APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
- APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- run: |
- . $HOME/.kerl/${{ matrix.otp }}/activate
- make ensure-rebar3
- sudo cp rebar3 /usr/local/bin/rebar3
- rm -rf _build/${{ matrix.profile }}/lib
- make ${{ matrix.profile }}-tgz
- - name: test
- working-directory: source
- run: |
- pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz)
- mkdir -p emqx
- tar -C emqx -zxf $pkg_name
- # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
- ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
- ready='no'
- for i in {1..18}; do
- if curl -fs 127.0.0.1:18083/status > /dev/null; then
- ready='yes'
- break
- fi
- sleep 1
- done
- if [ "$ready" != "yes" ]; then
- echo "Timed out waiting for emqx to be ready"
- cat emqx/log/erlang.log.1
- exit 1
- fi
- ./emqx/bin/emqx_ctl status
- ./emqx/bin/emqx stop
- rm -rf emqx
+ profile: ${{ matrix.profile }}
+ otp: ${{ matrix.otp }}
+ os: ${{ matrix.os }}
+ apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
+ apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
+ apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
+ apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- uses: actions/upload-artifact@v1
with:
name: ${{ matrix.profile }}-${{ matrix.otp }}
- path: source/_packages/${{ matrix.profile }}/.
+ path: _packages/${{ matrix.profile }}/.
linux:
needs: prepare
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index eb44cd2af..cf5df532c 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -133,75 +133,26 @@ jobs:
- emqx-enterprise
otp:
- 24.2.1-1
- macos:
+ os:
- macos-11
- runs-on: ${{ matrix.macos }}
+ runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
- name: prepare
run: |
- brew update
- brew install curl zip unzip kerl coreutils openssl@1.1
- echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
- echo "/usr/local/bin" >> $GITHUB_PATH
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
echo "BUILD_WITH_QUIC=1" >> $GITHUB_ENV
- - uses: actions/cache@v2
- id: cache
+ - uses: ./.github/actions/package-macos
with:
- path: ~/.kerl/${{ matrix.otp }}
- key: otp-install-${{ matrix.otp }}-${{ matrix.macos }}-static-ssl-disable-hipe-disable-jit
- - name: build erlang
- if: steps.cache.outputs.cache-hit != 'true'
- timeout-minutes: 60
- env:
- KERL_BUILD_BACKEND: git
- OTP_GITHUB_URL: https://github.com/emqx/otp
- KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
- run: |
- kerl update releases
- kerl build ${{ matrix.otp }}
- kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
- - name: build ${{ matrix.profile }}
- env:
- AUTO_INSTALL_BUILD_DEPS: 1
- APPLE_SIGN_BINARIES: 1
- APPLE_ID: developers@emqx.io
- APPLE_TEAM_ID: 26N6HYJLZA
- APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
- APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
- APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
- APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- run: |
- . $HOME/.kerl/${{ matrix.otp }}/activate
- make ensure-rebar3
- sudo cp rebar3 /usr/local/bin/rebar3
- make ${{ matrix.profile }}-tgz
- - name: test
- run: |
- pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
- mkdir emqx
- unzip -d emqx $pkg_name > /dev/null
- # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
- ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
- ready='no'
- for i in {1..30}; do
- if curl -fs 127.0.0.1:18083/status > /dev/null; then
- ready='yes'
- break
- fi
- sleep 1
- done
- if [ "$ready" != "yes" ]; then
- echo "Timed out waiting for emqx to be ready"
- cat emqx/log/erlang.log.1
- exit 1
- fi
- ./emqx/bin/emqx_ctl status
- ./emqx/bin/emqx stop
- rm -rf emqx
+ profile: ${{ matrix.profile }}
+ otp: ${{ matrix.otp }}
+ os: ${{ matrix.os }}
+ apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
+ apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
+ apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
+ apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- uses: actions/upload-artifact@v2
with:
name: macos
diff --git a/.gitignore b/.gitignore
index 26b146cef..d8b3806e3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -68,4 +68,3 @@ apps/emqx/test/emqx_static_checks_data/master.bpapi
# rendered configurations
*.conf.rendered
lux_logs/
-.ci/docker-compose-file/redis/*.log
diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md
index d2721b5bb..69a684dbd 100644
--- a/CHANGES-5.0.md
+++ b/CHANGES-5.0.md
@@ -1,6 +1,18 @@
# 5.0.9
+
+## Enhancements
+
+* Add `cert_common_name` and `cert_subject` placeholder support for authz_http and authz_mongo.[#8973](https://github.com/emqx/emqx/pull/8973)
+* Use milliseconds internally in emqx_delayed to store the publish time, improving precision.[#9060](https://github.com/emqx/emqx/pull/9060)
+* More rigorous checking of flapping to improve stability of the system. [#9136](https://github.com/emqx/emqx/pull/9136)
+
## Bug fixes
+* Check ACLs for last will testament topic before publishing the message. [#8930](https://github.com/emqx/emqx/pull/8930)
+* Fix GET /listeners API crash When some nodes still in initial configuration. [#9002](https://github.com/emqx/emqx/pull/9002)
+* Fix empty variable interpolation in authentication and authorization. Placeholders for undefined variables are rendered now as empty strings and do not cause errors anymore. [#8963](https://github.com/emqx/emqx/pull/8963)
+* Fix the latency statistics error of the slow subscription module when `stats_type` is `internal` or `response`. [#8986](https://github.com/emqx/emqx/pull/8986)
+* Redispatch shared subscription messages. [#9104](https://github.com/emqx/emqx/pull/9104)
* Ensure authentication type is an array, not struct. [#8923](https://github.com/emqx/emqx/pull/8923)
# 5.0.8
@@ -16,14 +28,21 @@
* Fix delayed publish inaccurate caused by os time change. [#8926](https://github.com/emqx/emqx/pull/8926)
* Fix that EMQX can't start when the retainer is disabled [#8911](https://github.com/emqx/emqx/pull/8911)
+* Fix that redis authn will deny the unknown users [#8934](https://github.com/emqx/emqx/pull/8934)
+* Fix ExProto UDP client keepalive checking error.
+ This causes the clients to not expire as long as a new UDP packet arrives [#8866](https://github.com/emqx/emqx/pull/8866)
+* Fix that MQTT Bridge message payload could be empty string. [#8949](https://github.com/emqx/emqx/pull/8949)
## Enhancements
* Print a warning message when boot with the default (insecure) Erlang cookie. [#8905](https://github.com/emqx/emqx/pull/8905)
* Change the `/gateway` API path to plural form. [#8823](https://github.com/emqx/emqx/pull/8823)
+* Don't allow updating config items when they already exist in `local-override.conf`. [#8851](https://github.com/emqx/emqx/pull/8851)
* Remove `node.etc_dir` from emqx.conf, because it is never used.
Also allow user to customize the logging directory [#8892](https://github.com/emqx/emqx/pull/8892)
* Added a new API `POST /listeners` for creating listener. [#8876](https://github.com/emqx/emqx/pull/8876)
+* Close ExProto client process immediately if it's keepalive timeouted. [#8866](https://github.com/emqx/emqx/pull/8866)
+* Upgrade grpc-erl driver to 0.6.7 to support batch operation in sending stream. [#8866](https://github.com/emqx/emqx/pull/8866)
# 5.0.7
diff --git a/Dockerfile.ubuntu20.04.runner b/Dockerfile.ubuntu20.04.runner
index 124021c89..1bb44a6e9 100644
--- a/Dockerfile.ubuntu20.04.runner
+++ b/Dockerfile.ubuntu20.04.runner
@@ -27,15 +27,14 @@ VOLUME ["/opt/emqx/log", "/opt/emqx/data"]
# emqx will occupy these port:
# - 1883 port for MQTT
-# - 8081 for mgmt API
# - 8083 for WebSocket/HTTP
# - 8084 for WSS/HTTPS
# - 8883 port for MQTT(SSL)
# - 11883 port for internal MQTT/TCP
-# - 18083 for dashboard
+# - 18083 for dashboard and API
# - 4370 default Erlang distrbution port
# - 5369 for backplain gen_rpc
-EXPOSE 1883 8081 8083 8084 8883 11883 18083 4370 5369
+EXPOSE 1883 8083 8084 8883 11883 18083 4370 5369
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
diff --git a/Makefile b/Makefile
index ead7847f8..74204afac 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
export EMQX_DEFAULT_RUNNER = debian:11-slim
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v1.0.8
+export EMQX_DASHBOARD_VERSION ?= v1.0.9
export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0
export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1
diff --git a/README-CN.md b/README-CN.md
index 71767869d..ea584c66a 100644
--- a/README-CN.md
+++ b/README-CN.md
@@ -32,12 +32,6 @@ EMQX 自 2013 年在 GitHub 发布开源版本以来，获得了来自 50 多个
docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx:latest
```
-或直接试用 EMQX 企业版（已内置 10 个并发连接的永不过期 License）
-
-```
-docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest
-```
-
接下来请参考 [入门指南](https://www.emqx.io/docs/zh/v5.0/getting-started/getting-started.html#启动-emqx) 开启您的 EMQX 之旅。
#### 在 Kubernetes 上运行 EMQX 集群
@@ -112,6 +106,27 @@ make
_build/emqx/rel/emqx/bin/emqx console
```
+### 在 Apple 芯片（M1,M2）上编译
+
+基于 Apple 芯片的 Homebrew 将[默认的 home 目录](https://github.com/Homebrew/brew/issues/9177)从 `/usr/local` 改成了 `/opt/homebrew`，这个改变导致了一些兼容性问题。
+
+具体到 EMQX 来说，主要影响的是 `unixodbc`，如果使用 Homebrew 安装的 `unixodbc` 包，那么在使用 [kerl](https://github.com/kerl/kerl) 编译 Erlang/OTP 的时候，kerl 会找不到 `unixodbc`。
+
+解决此问题的方法如下:
+
+```bash
+brew install unixodbc kerl
+sudo ln -s $(realpath $(brew --prefix unixodbc)) /usr/local/odbc
+export CC="/usr/bin/gcc -I$(brew --prefix unixodbc)/include"
+export LDFLAGS="-L$(brew --prefix unixodbc)/lib"
+kerl build 24.3
+mkdir ~/.kerl/installations
+kerl install 24.3 ~/.kerl/installations/24.3
+. ~/.kerl/installations/24.3/activate
+```
+
+然后再使用 `make` 继续编译就可以了。
+
## 开源许可
详见 [LICENSE](./LICENSE)。
diff --git a/README-RU.md b/README-RU.md
index cd8943795..76b1f0ae0 100644
--- a/README-RU.md
+++ b/README-RU.md
@@ -28,13 +28,7 @@
#### Установка EMQX с помощью Docker
```
-docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
-```
-
-Или запустите EMQX Enterprise со встроенной бессрочной лицензией на 10 соединений.
-
-```
-docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest
+docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
```
Чтобы ознакомиться с функциональностью EMQX, пожалуйста, следуйте [руководству по началу работы](https://www.emqx.io/docs/en/v5.0/getting-started/getting-started.html#start-emqx).
diff --git a/README.md b/README.md
index 4461f70c6..9543cf0de 100644
--- a/README.md
+++ b/README.md
@@ -33,12 +33,6 @@ The simplest way to set up EMQX is to create a managed deployment with EMQX Clou
docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx:latest
```
-Or install EMQX Enterprise with a built-in license for ten connections that never expire.
-
-```
-docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest
-```
-
Next, please follow the [getting started guide](https://www.emqx.io/docs/en/v5.0/getting-started/getting-started.html#start-emqx) to tour the EMQX features.
#### Run EMQX cluster on kubernetes
diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl
index bc14ef209..ad312e05d 100644
--- a/apps/emqx/include/emqx_release.hrl
+++ b/apps/emqx/include/emqx_release.hrl
@@ -32,7 +32,7 @@
%% `apps/emqx/src/bpapi/README.md'
%% Community edition
--define(EMQX_RELEASE_CE, "5.0.7").
+-define(EMQX_RELEASE_CE, "5.0.8").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.0.0-alpha.1").
diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config
index f9ffc4054..d20e980ea 100644
--- a/apps/emqx/rebar.config
+++ b/apps/emqx/rebar.config
@@ -27,7 +27,7 @@
{jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
{cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
- {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.4"}}},
+ {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.5"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src
index edf632740..cc1e7eaea 100644
--- a/apps/emqx/src/emqx.app.src
+++ b/apps/emqx/src/emqx.app.src
@@ -3,7 +3,7 @@
{id, "emqx"},
{description, "EMQX Core"},
% strict semver, bump manually!
- {vsn, "5.0.8"},
+ {vsn, "5.0.9"},
{modules, []},
{registered, []},
{applications, [
diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl
index 686833c45..494c77ec7 100644
--- a/apps/emqx/src/emqx_channel.erl
+++ b/apps/emqx/src/emqx_channel.erl
@@ -345,7 +345,8 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
fun check_connect/2,
fun enrich_client/2,
fun set_log_meta/2,
- fun check_banned/2
+ fun check_banned/2,
+ fun count_flapping_event/2
],
ConnPkt,
Channel#channel{conn_state = connecting}
@@ -354,12 +355,14 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
{ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} ->
?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
NChannel1 = NChannel#channel{
- will_msg = emqx_packet:will_msg(NConnPkt),
alias_maximum = init_alias_maximum(NConnPkt, ClientInfo)
},
case authenticate(?CONNECT_PACKET(NConnPkt), NChannel1) of
{ok, Properties, NChannel2} ->
- process_connect(Properties, NChannel2);
+ %% only store will_msg after successful authn
+ %% fix for: https://github.com/emqx/emqx/issues/8886
+ NChannel3 = NChannel2#channel{will_msg = emqx_packet:will_msg(NConnPkt)},
+ process_connect(Properties, NChannel3);
{continue, Properties, NChannel2} ->
handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, Properties}, NChannel2);
{error, ReasonCode} ->
@@ -995,8 +998,13 @@ maybe_nack(Delivers) ->
lists:filter(fun not_nacked/1, Delivers).
not_nacked({deliver, _Topic, Msg}) ->
- not (emqx_shared_sub:is_ack_required(Msg) andalso
- (ok == emqx_shared_sub:nack_no_connection(Msg))).
+ case emqx_shared_sub:is_ack_required(Msg) of
+ true ->
+ ok = emqx_shared_sub:nack_no_connection(Msg),
+ false;
+ false ->
+ true
+ end.
maybe_mark_as_delivered(Session, Delivers) ->
case emqx_session:info(is_persistent, Session) of
@@ -1165,10 +1173,11 @@ handle_call(
Channel = #channel{
conn_state = ConnState,
will_msg = WillMsg,
+ clientinfo = ClientInfo,
conninfo = #{proto_ver := ProtoVer}
}
) ->
- (WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
+ (WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg),
Channel1 =
case ConnState of
connected -> ensure_disconnected(kicked, Channel);
@@ -1219,6 +1228,8 @@ handle_call(
ChanInfo1 = info(NChannel),
emqx_cm:set_chan_info(ClientId, ChanInfo1#{sockinfo => SockInfo}),
reply(ok, reset_timer(alive_timer, NChannel));
+handle_call(get_mqueue, Channel) ->
+ reply({ok, get_mqueue(Channel)}, Channel);
handle_call(Req, Channel) ->
?SLOG(error, #{msg => "unexpected_call", call => Req}),
reply(ignored, Channel).
@@ -1250,14 +1261,11 @@ handle_info(
{sock_closed, Reason},
Channel =
#channel{
- conn_state = ConnState,
- clientinfo = ClientInfo = #{zone := Zone}
+ conn_state = ConnState
}
) when
ConnState =:= connected orelse ConnState =:= reauthenticating
->
- emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso
- emqx_flapping:detect(ClientInfo),
Channel1 = ensure_disconnected(Reason, maybe_publish_will_msg(Channel)),
case maybe_shutdown(Reason, Channel1) of
{ok, Channel2} -> {ok, {event, disconnected}, Channel2};
@@ -1359,8 +1367,10 @@ handle_timeout(
end;
handle_timeout(_TRef, expire_session, Channel) ->
shutdown(expired, Channel);
-handle_timeout(_TRef, will_message, Channel = #channel{will_msg = WillMsg}) ->
- (WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
+handle_timeout(
+ _TRef, will_message, Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg}
+) ->
+ (WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg),
{ok, clean_timer(will_timer, Channel#channel{will_msg = undefined})};
handle_timeout(
_TRef,
@@ -1434,12 +1444,14 @@ terminate({shutdown, kicked}, Channel) ->
run_terminate_hook(kicked, Channel);
terminate({shutdown, Reason}, Channel) when
Reason =:= discarded;
- Reason =:= takenover;
- Reason =:= not_authorized
+ Reason =:= takenover
->
run_terminate_hook(Reason, Channel);
-terminate(Reason, Channel = #channel{will_msg = WillMsg}) ->
- (WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
+terminate(Reason, Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg}) ->
+ %% since will_msg is set to undefined as soon as it is published,
+ %% if will_msg still exists when the session is terminated, it
+ %% must be published immediately.
+ WillMsg =/= undefined andalso publish_will_msg(ClientInfo, WillMsg),
(Reason =:= expired) andalso persist_if_session(Channel),
run_terminate_hook(Reason, Channel).
@@ -1622,6 +1634,14 @@ check_banned(_ConnPkt, #channel{clientinfo = ClientInfo}) ->
false -> ok
end.
+%%--------------------------------------------------------------------
+%% Flapping
+
+count_flapping_event(_ConnPkt, Channel = #channel{clientinfo = ClientInfo = #{zone := Zone}}) ->
+ emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso
+ emqx_flapping:detect(ClientInfo),
+ {ok, Channel}.
+
%%--------------------------------------------------------------------
%% Authenticate
@@ -2098,10 +2118,10 @@ ensure_disconnected(
maybe_publish_will_msg(Channel = #channel{will_msg = undefined}) ->
Channel;
-maybe_publish_will_msg(Channel = #channel{will_msg = WillMsg}) ->
+maybe_publish_will_msg(Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg}) ->
case will_delay_interval(WillMsg) of
0 ->
- ok = publish_will_msg(WillMsg),
+ ok = publish_will_msg(ClientInfo, WillMsg),
Channel#channel{will_msg = undefined};
I ->
ensure_timer(will_timer, timer:seconds(I), Channel)
@@ -2114,9 +2134,19 @@ will_delay_interval(WillMsg) ->
0
).
-publish_will_msg(Msg) ->
- _ = emqx_broker:publish(Msg),
- ok.
+publish_will_msg(ClientInfo, Msg = #message{topic = Topic}) ->
+ case emqx_access_control:authorize(ClientInfo, publish, Topic) of
+ allow ->
+ _ = emqx_broker:publish(Msg),
+ ok;
+ deny ->
+ ?tp(
+ warning,
+ last_will_testament_publish_denied,
+ #{topic => Topic}
+ ),
+ ok
+ end.
%%--------------------------------------------------------------------
%% Disconnect Reason
@@ -2207,3 +2237,6 @@ get_mqtt_conf(Zone, Key, Default) ->
set_field(Name, Value, Channel) ->
Pos = emqx_misc:index_of(Name, record_info(fields, channel)),
setelement(Pos + 1, Channel, Value).
+
+get_mqueue(#channel{session = Session}) ->
+ emqx_session:get_mqueue(Session).
diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl
index c5c67c7a5..fa1c63868 100644
--- a/apps/emqx/src/emqx_config.erl
+++ b/apps/emqx/src/emqx_config.erl
@@ -476,7 +476,7 @@ read_override_conf(#{} = Opts) ->
override_conf_file(Opts) when is_map(Opts) ->
Key =
- case maps:get(override_to, Opts, local) of
+ case maps:get(override_to, Opts, cluster) of
local -> local_override_conf_file;
cluster -> cluster_override_conf_file
end,
diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl
index ad46c1ee3..0311418a9 100644
--- a/apps/emqx/src/emqx_config_handler.erl
+++ b/apps/emqx/src/emqx_config_handler.erl
@@ -43,6 +43,7 @@
terminate/2,
code_change/3
]).
+-export([is_mutable/3]).
-define(MOD, {mod}).
-define(WKEY, '?').
@@ -229,15 +230,26 @@ process_update_request([_], _Handlers, {remove, _Opts}) ->
process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) ->
OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
BinKeyPath = bin_path(ConfKeyPath),
- NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf),
- OverrideConf = remove_from_override_config(BinKeyPath, Opts),
- {ok, NewRawConf, OverrideConf, Opts};
+ case check_permissions(remove, BinKeyPath, OldRawConf, Opts) of
+ allow ->
+ NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf),
+ OverrideConf = remove_from_override_config(BinKeyPath, Opts),
+ {ok, NewRawConf, OverrideConf, Opts};
+ {deny, Reason} ->
+ {error, {permission_denied, Reason}}
+ end;
process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) ->
OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of
{ok, NewRawConf} ->
- OverrideConf = update_override_config(NewRawConf, Opts),
- {ok, NewRawConf, OverrideConf, Opts};
+ BinKeyPath = bin_path(ConfKeyPath),
+ case check_permissions(update, BinKeyPath, NewRawConf, Opts) of
+ allow ->
+ OverrideConf = update_override_config(NewRawConf, Opts),
+ {ok, NewRawConf, OverrideConf, Opts};
+ {deny, Reason} ->
+ {error, {permission_denied, Reason}}
+ end;
Error ->
Error
end.
@@ -272,12 +284,11 @@ check_and_save_configs(
UpdateArgs,
Opts
) ->
- OldConf = emqx_config:get_root(ConfKeyPath),
Schema = schema(SchemaModule, ConfKeyPath),
{AppEnvs, NewConf} = emqx_config:check_config(Schema, NewRawConf),
+ OldConf = emqx_config:get_root(ConfKeyPath),
case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of
{ok, Result0} ->
- remove_from_local_if_cluster_change(ConfKeyPath, Opts),
ok = emqx_config:save_configs(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts),
Result1 = return_change_result(ConfKeyPath, UpdateArgs),
{ok, Result1#{post_config_update => Result0}};
@@ -430,16 +441,6 @@ merge_to_old_config(UpdateReq, RawConf) when is_map(UpdateReq), is_map(RawConf)
merge_to_old_config(UpdateReq, _RawConf) ->
{ok, UpdateReq}.
-%% local-override.conf priority is higher than cluster-override.conf
-%% If we want cluster to take effect, we must remove the local.
-remove_from_local_if_cluster_change(BinKeyPath, #{override_to := cluster} = Opts) ->
- Opts1 = Opts#{override_to => local},
- Local = remove_from_override_config(BinKeyPath, Opts1),
- _ = emqx_config:save_to_override_conf(Local, Opts1),
- ok;
-remove_from_local_if_cluster_change(_BinKeyPath, _Opts) ->
- ok.
-
remove_from_override_config(_BinKeyPath, #{persistent := false}) ->
undefined;
remove_from_override_config(BinKeyPath, Opts) ->
@@ -544,3 +545,98 @@ load_prev_handlers() ->
save_handlers(Handlers) ->
application:set_env(emqx, ?MODULE, Handlers).
+
+check_permissions(_Action, _ConfKeyPath, _NewRawConf, #{override_to := local}) ->
+ allow;
+check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) ->
+ case emqx_map_lib:deep_find(ConfKeyPath, NewRawConf) of
+ {ok, NewRaw} ->
+ LocalOverride = emqx_config:read_override_conf(#{override_to => local}),
+ case emqx_map_lib:deep_find(ConfKeyPath, LocalOverride) of
+ {ok, LocalRaw} ->
+ case is_mutable(Action, NewRaw, LocalRaw) of
+ ok ->
+ allow;
+ {error, Error} ->
+ ?SLOG(error, #{
+ msg => "prevent_remove_local_override_conf",
+ config_key_path => ConfKeyPath,
+ error => Error
+ }),
+ {deny, "Disable changed from local-override.conf"}
+ end;
+ {not_found, _, _} ->
+ allow
+ end;
+ {not_found, _, _} ->
+ allow
+ end.
+
+is_mutable(Action, NewRaw, LocalRaw) ->
+ try
+ KeyPath = [],
+ is_mutable(KeyPath, Action, NewRaw, LocalRaw)
+ catch
+ throw:Error -> Error
+ end.
+
+-define(REMOVE_FAILED, "remove_failed").
+-define(UPDATE_FAILED, "update_failed").
+
+is_mutable(KeyPath, Action, New = #{}, Local = #{}) ->
+ maps:foreach(
+ fun(Key, SubLocal) ->
+ case maps:find(Key, New) of
+ error -> ok;
+ {ok, SubNew} -> is_mutable(KeyPath ++ [Key], Action, SubNew, SubLocal)
+ end
+ end,
+ Local
+ );
+is_mutable(KeyPath, remove, Update, Origin) ->
+ throw({error, {?REMOVE_FAILED, KeyPath, Update, Origin}});
+is_mutable(_KeyPath, update, Val, Val) ->
+ ok;
+is_mutable(KeyPath, update, Update, Origin) ->
+ throw({error, {?UPDATE_FAILED, KeyPath, Update, Origin}}).
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+is_mutable_update_test() ->
+ Action = update,
+ ?assertEqual(ok, is_mutable(Action, #{}, #{})),
+ ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a => #{b => #{c => #{}}}})),
+ ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})),
+ ?assertEqual(
+ {error, {?UPDATE_FAILED, [a, b, c], 1, 2}},
+ is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}})
+ ),
+ ?assertEqual(
+ {error, {?UPDATE_FAILED, [a, b, d], 2, 3}},
+ is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}})
+ ),
+ ok.
+
+is_mutable_remove_test() ->
+ Action = remove,
+ ?assertEqual(ok, is_mutable(Action, #{}, #{})),
+ ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a1 => #{b => #{c => #{}}}})),
+ ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b1 => #{c => 1}}})),
+ ?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c1 => 1}}})),
+
+ ?assertEqual(
+ {error, {?REMOVE_FAILED, [a, b, c], 1, 1}},
+ is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})
+ ),
+ ?assertEqual(
+ {error, {?REMOVE_FAILED, [a, b, c], 1, 2}},
+ is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}})
+ ),
+ ?assertEqual(
+ {error, {?REMOVE_FAILED, [a, b, c], 1, 1}},
+ is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}})
+ ),
+ ok.
+
+-endif.
diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl
index aa2a2e0f9..2a809f88d 100644
--- a/apps/emqx/src/emqx_listeners.erl
+++ b/apps/emqx/src/emqx_listeners.erl
@@ -87,12 +87,18 @@ format_list(Listener) ->
].
do_list_raw() ->
- Key = <<"listeners">>,
- Raw = emqx_config:get_raw([Key], #{}),
- SchemaMod = emqx_config:get_schema_mod(Key),
- #{Key := RawWithDefault} = emqx_config:fill_defaults(SchemaMod, #{Key => Raw}, #{}),
- Listeners = maps:to_list(RawWithDefault),
- lists:flatmap(fun format_raw_listeners/1, Listeners).
+ %% GET /listeners from other nodes returns [] when init config is not loaded.
+ case emqx_app:get_init_config_load_done() of
+ true ->
+ Key = <<"listeners">>,
+ Raw = emqx_config:get_raw([Key], #{}),
+ SchemaMod = emqx_config:get_schema_mod(Key),
+ #{Key := RawWithDefault} = emqx_config:fill_defaults(SchemaMod, #{Key => Raw}, #{}),
+ Listeners = maps:to_list(RawWithDefault),
+ lists:flatmap(fun format_raw_listeners/1, Listeners);
+ false ->
+ []
+ end.
format_raw_listeners({Type0, Conf}) ->
Type = binary_to_atom(Type0),
@@ -515,12 +521,16 @@ merge_default(Options) ->
integer() | {tuple(), integer()} | string() | binary()
) -> io_lib:chars().
format_bind(Port) when is_integer(Port) ->
+ %% **Note**:
+ %% 'For TCP, UDP and IP networks, if the host is empty or a literal
+ %% unspecified IP address, as in ":80", "0.0.0.0:80" or "[::]:80" for
+ %% TCP and UDP, "", "0.0.0.0" or "::" for IP, the local system is
+ %% assumed.'
+ %%
+ %% Quoted from: https://pkg.go.dev/net
+ %% Decided to use this format to display the bind for all interfaces and
+ %% IPv4/IPv6 support
io_lib:format(":~w", [Port]);
-%% Print only the port number when bound on all interfaces
-format_bind({{0, 0, 0, 0}, Port}) ->
- format_bind(Port);
-format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, Port}) ->
- format_bind(Port);
format_bind({Addr, Port}) when is_list(Addr) ->
io_lib:format("~ts:~w", [Addr, Port]);
format_bind({Addr, Port}) when is_tuple(Addr), tuple_size(Addr) == 4 ->
@@ -532,6 +542,8 @@ format_bind(Str) when is_list(Str) ->
case emqx_schema:to_ip_port(Str) of
{ok, {Ip, Port}} ->
format_bind({Ip, Port});
+ {ok, Port} ->
+ format_bind(Port);
{error, _} ->
format_bind(list_to_integer(Str))
end;
diff --git a/apps/emqx/src/emqx_mqueue.erl b/apps/emqx/src/emqx_mqueue.erl
index f8556bc39..ae55d4b6e 100644
--- a/apps/emqx/src/emqx_mqueue.erl
+++ b/apps/emqx/src/emqx_mqueue.erl
@@ -66,7 +66,8 @@
in/2,
out/1,
stats/1,
- dropped/1
+ dropped/1,
+ to_list/1
]).
-define(NO_PRIORITY_TABLE, disabled).
@@ -109,7 +110,7 @@
dropped = 0 :: count(),
p_table = ?NO_PRIORITY_TABLE :: p_table(),
default_p = ?LOWEST_PRIORITY :: priority(),
- q = ?PQUEUE:new() :: pq(),
+ q = emqx_pqueue:new() :: pq(),
shift_opts :: #shift_opts{},
last_prio :: non_neg_integer() | undefined,
p_credit :: non_neg_integer() | undefined
@@ -118,7 +119,7 @@
-type mqueue() :: #mqueue{}.
-spec init(options()) -> mqueue().
-init(Opts = #{max_len := MaxLen0, store_qos0 := QoS_0}) ->
+init(Opts = #{max_len := MaxLen0, store_qos0 := Qos0}) ->
MaxLen =
case (is_integer(MaxLen0) andalso MaxLen0 > ?MAX_LEN_INFINITY) of
true -> MaxLen0;
@@ -126,7 +127,7 @@ init(Opts = #{max_len := MaxLen0, store_qos0 := QoS_0}) ->
end,
#mqueue{
max_len = MaxLen,
- store_qos0 = QoS_0,
+ store_qos0 = Qos0,
p_table = get_opt(priorities, Opts, ?NO_PRIORITY_TABLE),
default_p = get_priority_opt(Opts),
shift_opts = get_shift_opt(Opts)
@@ -152,6 +153,19 @@ len(#mqueue{len = Len}) -> Len.
max_len(#mqueue{max_len = MaxLen}) -> MaxLen.
+%% @doc Return all queued items in a list.
+-spec to_list(mqueue()) -> list().
+to_list(MQ) ->
+ to_list(MQ, []).
+
+to_list(MQ, Acc) ->
+ case out(MQ) of
+ {empty, _MQ} ->
+ lists:reverse(Acc);
+ {{value, Msg}, Q1} ->
+ to_list(Q1, [Msg | Acc])
+ end.
+
%% @doc Return number of dropped messages.
-spec dropped(mqueue()) -> count().
dropped(#mqueue{dropped = Dropped}) -> Dropped.
diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl
index 0bc028ec8..135992965 100644
--- a/apps/emqx/src/emqx_schema.erl
+++ b/apps/emqx/src/emqx_schema.erl
@@ -39,7 +39,8 @@
-type comma_separated_binary() :: [binary()].
-type comma_separated_atoms() :: [atom()].
-type bar_separated_list() :: list().
--type ip_port() :: tuple().
+-type ip_port() :: tuple() | integer().
+-type host_port() :: tuple().
-type cipher() :: map().
-typerefl_from_string({duration/0, emqx_schema, to_duration}).
@@ -52,6 +53,7 @@
-typerefl_from_string({comma_separated_binary/0, emqx_schema, to_comma_separated_binary}).
-typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}).
-typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}).
+-typerefl_from_string({host_port/0, emqx_schema, to_host_port}).
-typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}).
-typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}).
@@ -78,6 +80,7 @@
to_comma_separated_binary/1,
to_bar_separated_list/1,
to_ip_port/1,
+ to_host_port/1,
to_erl_cipher_suite/1,
to_comma_separated_atoms/1
]).
@@ -96,6 +99,7 @@
comma_separated_binary/0,
bar_separated_list/0,
ip_port/0,
+ host_port/0,
cipher/0,
comma_separated_atoms/0
]).
@@ -2167,33 +2171,60 @@ to_bar_separated_list(Str) ->
%% - :1883
%% - :::1883
to_ip_port(Str) ->
- case split_ip_port(Str) of
- {"", Port} ->
- {ok, {{0, 0, 0, 0}, list_to_integer(Port)}};
- {Ip, Port} ->
+ to_host_port(Str, ip_addr).
+
+%% @doc support the following format:
+%% - 127.0.0.1:1883
+%% - ::1:1883
+%% - [::1]:1883
+%% - :1883
+%% - :::1883
+%% - example.com:80
+to_host_port(Str) ->
+ to_host_port(Str, hostname).
+
+%% - example.com:80
+to_host_port(Str, IpOrHost) ->
+ case split_host_port(Str) of
+ {"", Port} when IpOrHost =:= ip_addr ->
+ %% this is a local address
+ {ok, list_to_integer(Port)};
+ {"", _Port} ->
+ %% must specify host part when it's a remote endpoint
+ {error, bad_host_port};
+ {MaybeIp, Port} ->
PortVal = list_to_integer(Port),
- case inet:parse_address(Ip) of
- {ok, R} ->
- {ok, {R, PortVal}};
- _ ->
+ case inet:parse_address(MaybeIp) of
+ {ok, IpTuple} ->
+ {ok, {IpTuple, PortVal}};
+ _ when IpOrHost =:= hostname ->
%% check is a rfc1035's hostname
- case inet_parse:domain(Ip) of
+ case inet_parse:domain(MaybeIp) of
true ->
- {ok, {Ip, PortVal}};
+ {ok, {MaybeIp, PortVal}};
_ ->
- {error, Str}
- end
+ {error, bad_hostname}
+ end;
+ _ ->
+ {error, bad_ip_port}
end;
_ ->
- {error, Str}
+ {error, bad_ip_port}
end.
-split_ip_port(Str0) ->
+split_host_port(Str0) ->
Str = re:replace(Str0, " ", "", [{return, list}, global]),
case lists:split(string:rchr(Str, $:), Str) of
- %% no port
+ %% no colon
{[], Str} ->
- error;
+ try
+ %% if it's just a port number, then return as-is
+ _ = list_to_integer(Str),
+ {"", Str}
+ catch
+ _:_ ->
+ error
+ end;
{IpPlusColon, PortString} ->
IpStr0 = lists:droplast(IpPlusColon),
case IpStr0 of
diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl
index 71ffeb24e..b285d0a88 100644
--- a/apps/emqx/src/emqx_session.erl
+++ b/apps/emqx/src/emqx_session.erl
@@ -60,7 +60,8 @@
info/2,
is_session/1,
stats/1,
- obtain_next_pkt_id/1
+ obtain_next_pkt_id/1,
+ get_mqueue/1
]).
-export([
@@ -801,7 +802,8 @@ replay(ClientInfo, Session = #session{inflight = Inflight}) ->
-spec terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok.
terminate(ClientInfo, Reason, Session) ->
run_terminate_hooks(ClientInfo, Reason, Session),
- redispatch_shared_messages(Session),
+ Reason =/= takenover andalso
+ redispatch_shared_messages(Session),
ok.
run_terminate_hooks(ClientInfo, discarded, Session) ->
@@ -811,29 +813,20 @@ run_terminate_hooks(ClientInfo, takenover, Session) ->
run_terminate_hooks(ClientInfo, Reason, Session) ->
run_hook('session.terminated', [ClientInfo, Reason, info(Session)]).
-redispatch_shared_messages(#session{inflight = Inflight}) ->
- InflightList = emqx_inflight:to_list(Inflight),
- lists:foreach(
- fun
- %% Only QoS1 messages get redispatched, because QoS2 messages
- %% must be sent to the same client, once they're in flight
- ({_, #inflight_data{message = #message{qos = ?QOS_2} = Msg}}) ->
- ?SLOG(warning, #{msg => qos2_lost_no_redispatch}, #{message => Msg});
- ({_, #inflight_data{message = #message{topic = Topic, qos = ?QOS_1} = Msg}}) ->
- case emqx_shared_sub:get_group(Msg) of
- {ok, Group} ->
- %% Note that dispatch is called with self() in failed subs
- %% This is done to avoid dispatching back to caller
- Delivery = #delivery{sender = self(), message = Msg},
- emqx_shared_sub:dispatch(Group, Topic, Delivery, [self()]);
- _ ->
- false
- end;
- (_) ->
- ok
- end,
- InflightList
- ).
+redispatch_shared_messages(#session{inflight = Inflight, mqueue = Q}) ->
+ AllInflights = emqx_inflight:to_list(fun sort_fun/2, Inflight),
+ F = fun
+ ({_PacketId, #inflight_data{message = #message{qos = ?QOS_1} = Msg}}) ->
+ %% For QoS 2, here is what the spec says:
+ %% If the Client's Session terminates before the Client reconnects,
+ %% the Server MUST NOT send the Application Message to any other
+ %% subscribed Client [MQTT-4.8.2-5].
+ {true, Msg};
+ ({_PacketId, #inflight_data{}}) ->
+ false
+ end,
+ InflightList = lists:filtermap(F, AllInflights),
+ emqx_shared_sub:redispatch(InflightList ++ emqx_mqueue:to_list(Q)).
-compile({inline, [run_hook/2]}).
run_hook(Name, Args) ->
@@ -892,7 +885,7 @@ on_delivery_completed(
).
mark_begin_deliver(Msg) ->
- emqx_message:set_header(deliver_begin_at, erlang:system_time(second), Msg).
+ emqx_message:set_header(deliver_begin_at, erlang:system_time(millisecond), Msg).
%%--------------------------------------------------------------------
%% Helper functions
@@ -925,3 +918,6 @@ age(Now, Ts) -> Now - Ts.
set_field(Name, Value, Session) ->
Pos = emqx_misc:index_of(Name, record_info(fields, session)),
setelement(Pos + 1, Session, Value).
+
+get_mqueue(#session{mqueue = Q}) ->
+ emqx_mqueue:to_list(Q).
diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl
index 0527cbfe7..975b403b9 100644
--- a/apps/emqx/src/emqx_shared_sub.erl
+++ b/apps/emqx/src/emqx_shared_sub.erl
@@ -39,15 +39,15 @@
-export([
dispatch/3,
dispatch/4,
- do_dispatch_with_ack/4
+ do_dispatch_with_ack/4,
+ redispatch/1
]).
-export([
maybe_ack/1,
maybe_nack_dropped/1,
nack_no_connection/1,
- is_ack_required/1,
- get_group/1
+ is_ack_required/1
]).
%% for testing
@@ -96,6 +96,9 @@
-define(ACK, shared_sub_ack).
-define(NACK(Reason), {shared_sub_nack, Reason}).
-define(NO_ACK, no_ack).
+-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
+
+-type redispatch_to() :: ?REDISPATCH_TO(emqx_topic:group(), emqx_topic:topic()).
-record(state, {pmon}).
@@ -144,7 +147,8 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) ->
false ->
{error, no_subscribers};
{Type, SubPid} ->
- case do_dispatch(SubPid, Group, Topic, Msg, Type) of
+ Msg1 = with_redispatch_to(Msg, Group, Topic),
+ case do_dispatch(SubPid, Group, Topic, Msg1, Type) of
ok ->
{ok, 1};
{error, _Reason} ->
@@ -223,16 +227,53 @@ without_group_ack(Msg) ->
get_group_ack(Msg) ->
emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK).
+with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) ->
+ Msg;
+with_redispatch_to(Msg, Group, Topic) ->
+ emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg).
+
+%% @hidden Redispatch is neede only for the messages with redispatch_to header added.
+is_redispatch_needed(#message{} = Msg) ->
+ case get_redispatch_to(Msg) of
+ ?REDISPATCH_TO(_, _) ->
+ true;
+ _ ->
+ false
+ end.
+
+%% @doc Redispatch shared deliveries to other members in the group.
+redispatch(Messages0) ->
+ Messages = lists:filter(fun is_redispatch_needed/1, Messages0),
+ case length(Messages) of
+ L when L > 0 ->
+ ?SLOG(info, #{
+ msg => "redispatching_shared_subscription_message",
+ count => L
+ }),
+ lists:foreach(fun redispatch_shared_message/1, Messages);
+ _ ->
+ ok
+ end.
+
+redispatch_shared_message(#message{} = Msg) ->
+ %% As long as it's still a #message{} record in inflight,
+ %% we should try to re-dispatch
+ ?REDISPATCH_TO(Group, Topic) = get_redispatch_to(Msg),
+ %% Note that dispatch is called with self() in failed subs
+ %% This is done to avoid dispatching back to caller
+ Delivery = #delivery{sender = self(), message = Msg},
+ dispatch(Group, Topic, Delivery, [self()]).
+
+%% @hidden Return the `redispatch_to` group-topic in the message header.
+%% `false` is returned if the message is not a shared dispatch.
+%% or when it's a QoS 0 message.
+-spec get_redispatch_to(emqx_types:message()) -> redispatch_to() | false.
+get_redispatch_to(Msg) ->
+ emqx_message:get_header(redispatch_to, Msg, false).
+
-spec is_ack_required(emqx_types:message()) -> boolean().
is_ack_required(Msg) -> ?NO_ACK =/= get_group_ack(Msg).
--spec get_group(emqx_types:message()) -> {ok, any()} | error.
-get_group(Msg) ->
- case get_group_ack(Msg) of
- ?NO_ACK -> error;
- {Group, _Sender, _Ref} -> {ok, Group}
- end.
-
%% @doc Negative ack dropped message due to inflight window or message queue being full.
-spec maybe_nack_dropped(emqx_types:message()) -> boolean().
maybe_nack_dropped(Msg) ->
diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl
index df1720772..a3fa3e5bc 100644
--- a/apps/emqx/test/emqx_channel_SUITE.erl
+++ b/apps/emqx/test/emqx_channel_SUITE.erl
@@ -207,14 +207,6 @@ init_per_suite(Config) ->
ok = meck:new(emqx_cm, [passthrough, no_history, no_link]),
ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end),
ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end),
- %% Access Control Meck
- ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
- ok = meck:expect(
- emqx_access_control,
- authenticate,
- fun(_) -> {ok, #{is_superuser => false}} end
- ),
- ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end),
%% Broker Meck
ok = meck:new(emqx_broker, [passthrough, no_history, no_link]),
%% Hooks Meck
@@ -234,7 +226,6 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
meck:unload([
- emqx_access_control,
emqx_metrics,
emqx_session,
emqx_broker,
@@ -244,11 +235,21 @@ end_per_suite(_Config) ->
]).
init_per_testcase(_TestCase, Config) ->
+ %% Access Control Meck
+ ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
+ ok = meck:expect(
+ emqx_access_control,
+ authenticate,
+ fun(_) -> {ok, #{is_superuser => false}} end
+ ),
+ ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end),
+ %% Set confs
OldConf = set_test_listener_confs(),
emqx_common_test_helpers:start_apps([]),
[{config, OldConf} | Config].
end_per_testcase(_TestCase, Config) ->
+ meck:unload([emqx_access_control]),
emqx_config:put(?config(config, Config)),
emqx_common_test_helpers:stop_apps([]),
Config.
@@ -1115,6 +1116,32 @@ t_ws_cookie_init(_) ->
),
?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)).
+%%--------------------------------------------------------------------
+%% Test cases for other mechnisms
+%%--------------------------------------------------------------------
+
+t_flapping_detect(_) ->
+ emqx_config:put_zone_conf(default, [flapping_detect, enable], true),
+ Parent = self(),
+ ok = meck:expect(
+ emqx_cm,
+ open_session,
+ fun(true, _ClientInfo, _ConnInfo) ->
+ {ok, #{session => session(), present => false}}
+ end
+ ),
+ ok = meck:expect(emqx_access_control, authenticate, fun(_) -> {error, not_authorized} end),
+ ok = meck:expect(emqx_flapping, detect, fun(_) -> Parent ! flapping_detect end),
+ IdleChannel = channel(#{conn_state => idle}),
+ {shutdown, not_authorized, _ConnAck, _Channel} =
+ emqx_channel:handle_in(?CONNECT_PACKET(connpkt()), IdleChannel),
+ receive
+ flapping_detect -> ok
+ after 2000 ->
+ ?assert(false, "Flapping detect should be exected in connecting progress")
+ end,
+ meck:unload([emqx_flapping]).
+
%%--------------------------------------------------------------------
%% Helper functions
%%--------------------------------------------------------------------
diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl
index ce998656a..f195e083c 100644
--- a/apps/emqx/test/emqx_common_test_helpers.erl
+++ b/apps/emqx/test/emqx_common_test_helpers.erl
@@ -632,9 +632,9 @@ setup_node(Node, Opts) when is_map(Opts) ->
%% Here we start the apps
EnvHandlerForRpc =
fun(App) ->
- %% We load configuration, and than set the special enviroment variable
+ %% We load configuration, and than set the special environment variable
%% which says that emqx shouldn't load configuration at startup
- %% Otherwise, configuration get's loaded and all preset env in envhandler is lost
+ %% Otherwise, configuration gets loaded and all preset env in EnvHandler is lost
LoadSchema andalso
begin
emqx_config:init_load(SchemaMod),
diff --git a/apps/emqx/test/emqx_config_handler_SUITE.erl b/apps/emqx/test/emqx_config_handler_SUITE.erl
index ae34bee7a..1fe79f74c 100644
--- a/apps/emqx/test/emqx_config_handler_SUITE.erl
+++ b/apps/emqx/test/emqx_config_handler_SUITE.erl
@@ -21,6 +21,8 @@
-define(MOD, {mod}).
-define(WKEY, '?').
+-define(LOCAL_CONF, "/tmp/local-override.conf").
+-define(CLUSTER_CONF, "/tmp/cluster-override.conf").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@@ -36,6 +38,8 @@ end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]).
init_per_testcase(_Case, Config) ->
+ _ = file:delete(?LOCAL_CONF),
+ _ = file:delete(?CLUSTER_CONF),
Config.
end_per_testcase(_Case, _Config) ->
@@ -196,6 +200,62 @@ t_sub_key_update_remove(_Config) ->
ok = emqx_config_handler:remove_handler(KeyPath2),
ok.
+t_local_override_update_remove(_Config) ->
+ application:set_env(emqx, local_override_conf_file, ?LOCAL_CONF),
+ application:set_env(emqx, cluster_override_conf_file, ?CLUSTER_CONF),
+ KeyPath = [sysmon, os, cpu_high_watermark],
+ ok = emqx_config_handler:add_handler(KeyPath, ?MODULE),
+ LocalOpts = #{override_to => local},
+ {ok, Res} = emqx:update_config(KeyPath, <<"70%">>, LocalOpts),
+ ?assertMatch(
+ #{
+ config := 0.7,
+ post_config_update := #{},
+ raw_config := <<"70%">>
+ },
+ Res
+ ),
+ ClusterOpts = #{override_to => cluster},
+ ?assertMatch(
+ {error, {permission_denied, _}}, emqx:update_config(KeyPath, <<"71%">>, ClusterOpts)
+ ),
+ ?assertMatch(0.7, emqx:get_config(KeyPath)),
+
+ KeyPath2 = [sysmon, os, cpu_low_watermark],
+ ok = emqx_config_handler:add_handler(KeyPath2, ?MODULE),
+ ?assertMatch(
+ {error, {permission_denied, _}}, emqx:update_config(KeyPath2, <<"40%">>, ClusterOpts)
+ ),
+
+ %% remove
+ ?assertMatch({error, {permission_denied, _}}, emqx:remove_config(KeyPath)),
+ ?assertEqual(
+ {ok, #{post_config_update => #{}}},
+ emqx:remove_config(KeyPath, #{override_to => local})
+ ),
+ ?assertEqual(
+ {ok, #{post_config_update => #{}}},
+ emqx:remove_config(KeyPath)
+ ),
+ ?assertError({config_not_found, KeyPath}, emqx:get_raw_config(KeyPath)),
+ OSKey = maps:keys(emqx:get_raw_config([sysmon, os])),
+ ?assertEqual(false, lists:member(<<"cpu_high_watermark">>, OSKey)),
+ ?assert(length(OSKey) > 0),
+
+ ?assertEqual(
+ {ok, #{config => 0.8, post_config_update => #{}, raw_config => <<"80%">>}},
+ emqx:reset_config(KeyPath, ClusterOpts)
+ ),
+ OSKey1 = maps:keys(emqx:get_raw_config([sysmon, os])),
+ ?assertEqual(true, lists:member(<<"cpu_high_watermark">>, OSKey1)),
+ ?assert(length(OSKey1) > 1),
+
+ ok = emqx_config_handler:remove_handler(KeyPath),
+ ok = emqx_config_handler:remove_handler(KeyPath2),
+ application:unset_env(emqx, local_override_conf_file),
+ application:unset_env(emqx, cluster_override_conf_file),
+ ok.
+
t_check_failed(_Config) ->
KeyPath = [sysmon, os, cpu_check_interval],
Opts = #{rawconf_with_defaults => true},
@@ -219,7 +279,7 @@ t_stop(_Config) ->
ok.
t_callback_crash(_Config) ->
- CrashPath = [sysmon, os, cpu_high_watermark],
+ CrashPath = [sysmon, os, procmem_high_watermark],
Opts = #{rawconf_with_defaults => true},
ok = emqx_config_handler:add_handler(CrashPath, ?MODULE),
Old = emqx:get_raw_config(CrashPath),
@@ -334,6 +394,8 @@ pre_config_update([sysmon, os, cpu_check_interval], UpdateReq, _RawConf) ->
{ok, UpdateReq};
pre_config_update([sysmon, os, cpu_low_watermark], UpdateReq, _RawConf) ->
{ok, UpdateReq};
+pre_config_update([sysmon, os, cpu_high_watermark], UpdateReq, _RawConf) ->
+ {ok, UpdateReq};
pre_config_update([sysmon, os, sysmem_high_watermark], UpdateReq, _RawConf) ->
{ok, UpdateReq};
pre_config_update([sysmon, os, mem_check_interval], _UpdateReq, _RawConf) ->
@@ -347,6 +409,8 @@ post_config_update([sysmon, os, cpu_check_interval], _UpdateReq, _NewConf, _OldC
{ok, ok};
post_config_update([sysmon, os, cpu_low_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs) ->
ok;
+post_config_update([sysmon, os, cpu_high_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs) ->
+ ok;
post_config_update([sysmon, os, sysmem_high_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs) ->
{error, post_config_update_error}.
diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl
index 6ea4d043d..5eb216be5 100644
--- a/apps/emqx/test/emqx_listeners_SUITE.erl
+++ b/apps/emqx/test/emqx_listeners_SUITE.erl
@@ -148,6 +148,32 @@ t_wss_conn(_) ->
{ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000),
ok = ssl:close(Socket).
+t_format_bind(_) ->
+ ?assertEqual(
+ ":1883",
+ lists:flatten(emqx_listeners:format_bind(1883))
+ ),
+ ?assertEqual(
+ "0.0.0.0:1883",
+ lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0}, 1883}))
+ ),
+ ?assertEqual(
+ "[::]:1883",
+ lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, 1883}))
+ ),
+ ?assertEqual(
+ "127.0.0.1:1883",
+ lists:flatten(emqx_listeners:format_bind({{127, 0, 0, 1}, 1883}))
+ ),
+ ?assertEqual(
+ ":1883",
+ lists:flatten(emqx_listeners:format_bind("1883"))
+ ),
+ ?assertEqual(
+ ":1883",
+ lists:flatten(emqx_listeners:format_bind(":1883"))
+ ).
+
render_config_file() ->
Path = local_path(["etc", "emqx.conf"]),
{ok, Temp} = file:read_file(Path),
diff --git a/apps/emqx/test/emqx_schema_tests.erl b/apps/emqx/test/emqx_schema_tests.erl
index 9118ac226..fdda9ef44 100644
--- a/apps/emqx/test/emqx_schema_tests.erl
+++ b/apps/emqx/test/emqx_schema_tests.erl
@@ -175,3 +175,30 @@ ssl_opts_gc_after_handshake_test_not_rancher_listener_test() ->
Checked
),
ok.
+
+to_ip_port_test_() ->
+ Ip = fun emqx_schema:to_ip_port/1,
+ Host = fun(Str) ->
+ case Ip(Str) of
+ {ok, {_, _} = Res} ->
+ %% assert
+ {ok, Res} = emqx_schema:to_host_port(Str);
+ _ ->
+ emqx_schema:to_host_port(Str)
+ end
+ end,
+ [
+ ?_assertEqual({ok, 80}, Ip("80")),
+ ?_assertEqual({error, bad_host_port}, Host("80")),
+ ?_assertEqual({ok, 80}, Ip(":80")),
+ ?_assertEqual({error, bad_host_port}, Host(":80")),
+ ?_assertEqual({error, bad_ip_port}, Ip("localhost:80")),
+ ?_assertEqual({ok, {"localhost", 80}}, Host("localhost:80")),
+ ?_assertEqual({ok, {"example.com", 80}}, Host("example.com:80")),
+ ?_assertEqual({ok, {{127, 0, 0, 1}, 80}}, Ip("127.0.0.1:80")),
+ ?_assertEqual({error, bad_ip_port}, Ip("$:1900")),
+ ?_assertEqual({error, bad_hostname}, Host("$:1900")),
+ ?_assertMatch({ok, {_, 1883}}, Ip("[::1]:1883")),
+ ?_assertMatch({ok, {_, 1883}}, Ip("::1:1883")),
+ ?_assertMatch({ok, {_, 1883}}, Ip(":::1883"))
+ ].
diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl
index f53e8f374..291286aa2 100644
--- a/apps/emqx/test/emqx_shared_sub_SUITE.erl
+++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl
@@ -25,10 +25,20 @@
-define(SUITE, ?MODULE).
--define(wait(For, Timeout),
- emqx_common_test_helpers:wait_for(
- ?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout
- )
+-define(WAIT(TIMEOUT, PATTERN, Res),
+ (fun() ->
+ receive
+ PATTERN ->
+ Res;
+ Other ->
+ ct:fail(#{
+ expected => ??PATTERN,
+ got => Other
+ })
+ after TIMEOUT ->
+ ct:fail({timeout, ??PATTERN})
+ end
+ end)()
).
-define(ack, shared_sub_ack).
@@ -45,10 +55,26 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]).
-t_is_ack_required(_) ->
+init_per_testcase(Case, Config) ->
+ try
+ ?MODULE:Case({'init', Config})
+ catch
+ error:function_clause ->
+ Config
+ end.
+
+end_per_testcase(Case, Config) ->
+ try
+ ?MODULE:Case({'end', Config})
+ catch
+ error:function_clause ->
+ ok
+ end.
+
+t_is_ack_required(Config) when is_list(Config) ->
?assertEqual(false, emqx_shared_sub:is_ack_required(#message{headers = #{}})).
-t_maybe_nack_dropped(_) ->
+t_maybe_nack_dropped(Config) when is_list(Config) ->
?assertEqual(false, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})),
Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
?assertEqual(true, emqx_shared_sub:maybe_nack_dropped(Msg)),
@@ -60,7 +86,7 @@ t_maybe_nack_dropped(_) ->
end
).
-t_nack_no_connection(_) ->
+t_nack_no_connection(Config) when is_list(Config) ->
Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
?assertEqual(ok, emqx_shared_sub:nack_no_connection(Msg)),
?assertEqual(
@@ -71,7 +97,7 @@ t_nack_no_connection(_) ->
end
).
-t_maybe_ack(_) ->
+t_maybe_ack(Config) when is_list(Config) ->
?assertEqual(#message{headers = #{}}, emqx_shared_sub:maybe_ack(#message{headers = #{}})),
Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
?assertEqual(
@@ -86,10 +112,7 @@ t_maybe_ack(_) ->
end
).
-% t_subscribers(_) ->
-% error('TODO').
-
-t_random_basic(_) ->
+t_random_basic(Config) when is_list(Config) ->
ok = ensure_config(random),
ClientId = <<"ClientId">>,
Topic = <<"foo">>,
@@ -121,7 +144,7 @@ t_random_basic(_) ->
%% After the connection for the 2nd session is also closed,
%% i.e. when all clients are offline, the following message(s)
%% should be delivered randomly.
-t_no_connection_nack(_) ->
+t_no_connection_nack(Config) when is_list(Config) ->
ok = ensure_config(sticky),
Publisher = <<"publisher">>,
Subscriber1 = <<"Subscriber1">>,
@@ -153,54 +176,22 @@ t_no_connection_nack(_) ->
%% This is the connection which was picked by broker to dispatch (sticky) for 1st message
?assertMatch([#{packet_id := 1}], recv_msgs(1)),
- %% Now kill the connection, expect all following messages to be delivered to the other
- %% subscriber.
- %emqx_mock_client:stop(ConnPid),
- %% sleep then make synced calls to session processes to ensure that
- %% the connection pid's 'EXIT' message is propagated to the session process
- %% also to be sure sessions are still alive
- % timer:sleep(2),
- % _ = emqx_session:info(SPid1),
- % _ = emqx_session:info(SPid2),
- % %% Now we know what is the other still alive connection
- % [TheOtherConnPid] = [SubConnPid1, SubConnPid2] -- [ConnPid],
- % %% Send some more messages
- % PacketIdList = lists:seq(2, 10),
- % lists:foreach(fun(Id) ->
- % SendF(Id),
- % ?wait(Received(Id, TheOtherConnPid), 1000)
- % end, PacketIdList),
- % %% Now close the 2nd (last connection)
- % emqx_mock_client:stop(TheOtherConnPid),
- % timer:sleep(2),
- % %% both sessions should have conn_pid = undefined
- % ?assertEqual({conn_pid, undefined}, lists:keyfind(conn_pid, 1, emqx_session:info(SPid1))),
- % ?assertEqual({conn_pid, undefined}, lists:keyfind(conn_pid, 1, emqx_session:info(SPid2))),
- % %% send more messages, but all should be queued in session state
- % lists:foreach(fun(Id) -> SendF(Id) end, PacketIdList),
- % {_, L1} = lists:keyfind(mqueue_len, 1, emqx_session:info(SPid1)),
- % {_, L2} = lists:keyfind(mqueue_len, 1, emqx_session:info(SPid2)),
- % ?assertEqual(length(PacketIdList), L1 + L2),
- % %% clean up
- % emqx_mock_client:close_session(PubConnPid),
- % emqx_sm:close_session(SPid1),
- % emqx_sm:close_session(SPid2),
ok.
-t_random(_) ->
+t_random(Config) when is_list(Config) ->
ok = ensure_config(random, true),
test_two_messages(random).
-t_round_robin(_) ->
+t_round_robin(Config) when is_list(Config) ->
ok = ensure_config(round_robin, true),
test_two_messages(round_robin).
-t_round_robin_per_group(_) ->
+t_round_robin_per_group(Config) when is_list(Config) ->
ok = ensure_config(round_robin_per_group, true),
test_two_messages(round_robin_per_group).
%% this would fail if executed with the standard round_robin strategy
-t_round_robin_per_group_even_distribution_one_group(_) ->
+t_round_robin_per_group_even_distribution_one_group(Config) when is_list(Config) ->
ok = ensure_config(round_robin_per_group, true),
Topic = <<"foo/bar">>,
Group = <<"group1">>,
@@ -264,7 +255,7 @@ t_round_robin_per_group_even_distribution_one_group(_) ->
),
ok.
-t_round_robin_per_group_even_distribution_two_groups(_) ->
+t_round_robin_per_group_even_distribution_two_groups(Config) when is_list(Config) ->
ok = ensure_config(round_robin_per_group, true),
Topic = <<"foo/bar">>,
{ok, ConnPid1} = emqtt:start_link([{clientid, <<"C0">>}]),
@@ -350,19 +341,19 @@ t_round_robin_per_group_even_distribution_two_groups(_) ->
),
ok.
-t_sticky(_) ->
+t_sticky(Config) when is_list(Config) ->
ok = ensure_config(sticky, true),
test_two_messages(sticky).
-t_hash(_) ->
+t_hash(Config) when is_list(Config) ->
ok = ensure_config(hash, false),
test_two_messages(hash).
-t_hash_clinetid(_) ->
+t_hash_clinetid(Config) when is_list(Config) ->
ok = ensure_config(hash_clientid, false),
test_two_messages(hash_clientid).
-t_hash_topic(_) ->
+t_hash_topic(Config) when is_list(Config) ->
ok = ensure_config(hash_topic, false),
ClientId1 = <<"ClientId1">>,
ClientId2 = <<"ClientId2">>,
@@ -407,7 +398,7 @@ t_hash_topic(_) ->
ok.
%% if the original subscriber dies, change to another one alive
-t_not_so_sticky(_) ->
+t_not_so_sticky(Config) when is_list(Config) ->
ok = ensure_config(sticky),
ClientId1 = <<"ClientId1">>,
ClientId2 = <<"ClientId2">>,
@@ -481,7 +472,7 @@ last_message(ExpectedPayload, Pids, Timeout) ->
<<"not yet?">>
end.
-t_dispatch(_) ->
+t_dispatch(Config) when is_list(Config) ->
ok = ensure_config(random),
Topic = <<"foo">>,
?assertEqual(
@@ -494,13 +485,13 @@ t_dispatch(_) ->
emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
).
-t_uncovered_func(_) ->
+t_uncovered_func(Config) when is_list(Config) ->
ignored = gen_server:call(emqx_shared_sub, ignored),
ok = gen_server:cast(emqx_shared_sub, ignored),
ignored = emqx_shared_sub ! ignored,
{mnesia_table_event, []} = emqx_shared_sub ! {mnesia_table_event, []}.
-t_per_group_config(_) ->
+t_per_group_config(Config) when is_list(Config) ->
ok = ensure_group_config(#{
<<"local_group">> => local,
<<"round_robin_group">> => round_robin,
@@ -521,7 +512,7 @@ t_per_group_config(_) ->
test_two_messages(round_robin_per_group, <<"round_robin_per_group_group">>),
test_two_messages(round_robin_per_group, <<"round_robin_per_group_group">>).
-t_local(_) ->
+t_local(Config) when is_list(Config) ->
GroupConfig = #{
<<"local_group">> => local,
<<"round_robin_group">> => round_robin,
@@ -567,7 +558,7 @@ t_local(_) ->
?assertNotEqual(UsedSubPid1, UsedSubPid2),
ok.
-t_remote(_) ->
+t_remote(Config) when is_list(Config) ->
%% This testcase verifies dispatching of shared messages to the remote nodes via backplane API.
%%
%% In this testcase we start two EMQX nodes: local and remote.
@@ -594,7 +585,7 @@ t_remote(_) ->
try
{ok, ClientPidLocal} = emqtt:connect(ConnPidLocal),
- {ok, ClientPidRemote} = emqtt:connect(ConnPidRemote),
+ {ok, _ClientPidRemote} = emqtt:connect(ConnPidRemote),
emqtt:subscribe(ConnPidRemote, {<<"$share/remote_group/", Topic/binary>>, 0}),
@@ -620,7 +611,7 @@ t_remote(_) ->
stop_slave(Node)
end.
-t_local_fallback(_) ->
+t_local_fallback(Config) when is_list(Config) ->
ok = ensure_group_config(#{
<<"local_group">> => local,
<<"round_robin_group">> => round_robin,
@@ -653,9 +644,14 @@ t_local_fallback(_) ->
%% This one tests that broker tries to select another shared subscriber
%% If the first one doesn't return an ACK
-t_redispatch(_) ->
- ok = ensure_config(sticky, true),
+t_redispatch_qos1_with_ack(Config) when is_list(Config) ->
+ test_redispatch_qos1(Config, true).
+t_redispatch_qos1_no_ack(Config) when is_list(Config) ->
+ test_redispatch_qos1(Config, false).
+
+test_redispatch_qos1(_Config, AckEnabled) ->
+ ok = ensure_config(sticky, AckEnabled),
Group = <<"group1">>,
Topic = <<"foo/bar">>,
ClientId1 = <<"ClientId1">>,
@@ -682,10 +678,169 @@ t_redispatch(_) ->
emqtt:stop(UsedSubPid2),
ok.
+t_qos1_random_dispatch_if_all_members_are_down(Config) when is_list(Config) ->
+ ok = ensure_config(sticky, true),
+ Group = <<"group1">>,
+ Topic = <<"foo/bar">>,
+ ClientId1 = <<"ClientId1">>,
+ ClientId2 = <<"ClientId2">>,
+ SubOpts = [{clean_start, false}],
+ {ok, ConnPub} = emqtt:start_link([{clientid, <<"pub">>}]),
+ {ok, _} = emqtt:connect(ConnPub),
+
+ {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1} | SubOpts]),
+ {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2} | SubOpts]),
+ {ok, _} = emqtt:connect(ConnPid1),
+ {ok, _} = emqtt:connect(ConnPid2),
+
+ emqtt:subscribe(ConnPid1, {<<"$share/", Group/binary, "/foo/bar">>, 1}),
+ emqtt:subscribe(ConnPid2, {<<"$share/", Group/binary, "/foo/bar">>, 1}),
+
+ ok = emqtt:stop(ConnPid1),
+ ok = emqtt:stop(ConnPid2),
+
+ [Pid1, Pid2] = emqx_shared_sub:subscribers(Group, Topic),
+ ?assert(is_process_alive(Pid1)),
+ ?assert(is_process_alive(Pid2)),
+
+ {ok, _} = emqtt:publish(ConnPub, Topic, <<"hello11">>, 1),
+ ct:sleep(100),
+ {ok, Msgs1} = gen_server:call(Pid1, get_mqueue),
+ {ok, Msgs2} = gen_server:call(Pid2, get_mqueue),
+ %% assert the message is in mqueue (because socket is closed)
+ ?assertMatch([#message{payload = <<"hello11">>}], Msgs1 ++ Msgs2),
+ emqtt:stop(ConnPub),
+ ok.
+
+%% No ack, QoS 2 subscriptions,
+%% client1 receives one message, send pubrec, then suspend
+%% client2 acts normal (auto_ack=true)
+%% Expected behaviour:
+%% the messages sent to client1's inflight and mq are re-dispatched after client1 is down
+t_dispatch_qos2({init, Config}) when is_list(Config) ->
+ emqx_config:put_zone_conf(default, [mqtt, max_inflight], 1),
+ Config;
+t_dispatch_qos2({'end', Config}) when is_list(Config) ->
+ emqx_config:put_zone_conf(default, [mqtt, max_inflight], 0);
+t_dispatch_qos2(Config) when is_list(Config) ->
+ ok = ensure_config(round_robin, _AckEnabled = false),
+ Topic = <<"foo/bar/1">>,
+ ClientId1 = <<"ClientId1">>,
+ ClientId2 = <<"ClientId2">>,
+
+ {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]),
+ {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]),
+ {ok, _} = emqtt:connect(ConnPid1),
+ {ok, _} = emqtt:connect(ConnPid2),
+
+ emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar/#">>, 2}),
+ emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar/#">>, 2}),
+
+ Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>),
+ Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>),
+ Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>),
+ Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>),
+ ct:sleep(100),
+
+ ok = sys:suspend(ConnPid1),
+
+ %% One message is inflight
+ ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message1)),
+ ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message2)),
+ ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message3)),
+ ?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message4)),
+
+ MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1),
+ MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2),
+ %% assert hello2 > hello1 or hello4 > hello3
+ ?assert(MsgRec2 > MsgRec1),
+
+ sys:resume(ConnPid1),
+ %% emqtt subscriber automatically sends PUBREC, but since auto_ack is set to false
+ %% so it will never send PUBCOMP, hence EMQX should not attempt to send
+ %% the 4th message yet since max_inflight is 1.
+ MsgRec3 = ?WAIT(2000, {publish, #{client_pid := ConnPid1, payload := P3}}, P3),
+ ct:sleep(100),
+ %% no message expected
+ ?assertEqual([], collect_msgs(0)),
+ %% now kill client 1
+ kill_process(ConnPid1),
+ %% client 2 should receive the message
+ MsgRec4 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P4}}, P4),
+ %% assert hello2 > hello1 or hello4 > hello3
+ ?assert(MsgRec4 > MsgRec3),
+ emqtt:stop(ConnPid2),
+ ok.
+
+t_dispatch_qos0({init, Config}) when is_list(Config) ->
+ Config;
+t_dispatch_qos0({'end', Config}) when is_list(Config) ->
+ ok;
+t_dispatch_qos0(Config) when is_list(Config) ->
+ ok = ensure_config(round_robin, _AckEnabled = false),
+ Topic = <<"foo/bar/1">>,
+ ClientId1 = <<"ClientId1">>,
+ ClientId2 = <<"ClientId2">>,
+
+ {ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]),
+ {ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]),
+ {ok, _} = emqtt:connect(ConnPid1),
+ {ok, _} = emqtt:connect(ConnPid2),
+
+ %% subscribe with QoS 0
+ emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar/#">>, 0}),
+ emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar/#">>, 0}),
+
+ %% publish with QoS 2, but should be downgraded to 0 as the subscribers
+ %% subscribe with QoS 0
+ Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>),
+ Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>),
+ Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>),
+ Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>),
+ ct:sleep(100),
+
+ ok = sys:suspend(ConnPid1),
+
+ ?assertMatch([_], emqx:publish(Message1)),
+ ?assertMatch([_], emqx:publish(Message2)),
+ ?assertMatch([_], emqx:publish(Message3)),
+ ?assertMatch([_], emqx:publish(Message4)),
+
+ MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1),
+ MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2),
+ %% assert hello2 > hello1 or hello4 > hello3
+ ?assert(MsgRec2 > MsgRec1),
+
+ kill_process(ConnPid1),
+ %% expect no redispatch
+ ?assertEqual([], collect_msgs(timer:seconds(2))),
+ emqtt:stop(ConnPid2),
+ ok.
+
%%--------------------------------------------------------------------
%% help functions
%%--------------------------------------------------------------------
+kill_process(Pid) ->
+ _ = unlink(Pid),
+ _ = monitor(process, Pid),
+ erlang:exit(Pid, kill),
+ receive
+ {'DOWN', _, process, Pid, _} ->
+ ok
+ end.
+
+collect_msgs(Timeout) ->
+ collect_msgs([], Timeout).
+
+collect_msgs(Acc, Timeout) ->
+ receive
+ Msg ->
+ collect_msgs([Msg | Acc], Timeout)
+ after Timeout ->
+ lists:reverse(Acc)
+ end.
+
ensure_config(Strategy) ->
ensure_config(Strategy, _AckEnabled = true).
diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl
index ba5f80a74..d59eea1af 100644
--- a/apps/emqx_authn/include/emqx_authn.hrl
+++ b/apps/emqx_authn/include/emqx_authn.hrl
@@ -38,8 +38,4 @@
-define(RESOURCE_GROUP, <<"emqx_authn">>).
--define(WITH_SUCCESSFUL_RENDER(Code),
- emqx_authn_utils:with_successful_render(?MODULE, fun() -> Code end)
-).
-
-endif.
diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src
index c4c59bd4a..bdb70cd4e 100644
--- a/apps/emqx_authn/src/emqx_authn.app.src
+++ b/apps/emqx_authn/src/emqx_authn.app.src
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_authn, [
{description, "EMQX Authentication"},
- {vsn, "0.1.6"},
+ {vsn, "0.1.7"},
{modules, []},
{registered, [emqx_authn_sup, emqx_authn_registry]},
{applications, [kernel, stdlib, emqx_resource, ehttpc, epgsql, mysql, jose]},
diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl
index c7f8eb824..6b5a8c5cd 100644
--- a/apps/emqx_authn/src/emqx_authn_api.erl
+++ b/apps/emqx_authn/src/emqx_authn_api.erl
@@ -34,7 +34,7 @@
% Swagger
-define(API_TAGS_GLOBAL, [<<"Authentication">>]).
--define(API_TAGS_SINGLE, [<<"Listener authentication">>]).
+-define(API_TAGS_SINGLE, [<<"Listener Authentication">>]).
-export([
api_spec/0,
diff --git a/apps/emqx_authn/src/emqx_authn_user_import_api.erl b/apps/emqx_authn/src/emqx_authn_user_import_api.erl
index 47d4de81c..dbd4c3656 100644
--- a/apps/emqx_authn/src/emqx_authn_user_import_api.erl
+++ b/apps/emqx_authn/src/emqx_authn_user_import_api.erl
@@ -30,7 +30,7 @@
% Swagger
-define(API_TAGS_GLOBAL, [<<"Authentication">>]).
--define(API_TAGS_SINGLE, [<<"Listener authentication">>]).
+-define(API_TAGS_SINGLE, [<<"Listener Authentication">>]).
-export([
api_spec/0,
diff --git a/apps/emqx_authn/src/emqx_authn_utils.erl b/apps/emqx_authn/src/emqx_authn_utils.erl
index b989da3b4..099da0077 100644
--- a/apps/emqx_authn/src/emqx_authn_utils.erl
+++ b/apps/emqx_authn/src/emqx_authn_utils.erl
@@ -34,8 +34,7 @@
ensure_apps_started/1,
cleanup_resources/0,
make_resource_id/1,
- without_password/1,
- with_successful_render/2
+ without_password/1
]).
-define(AUTHN_PLACEHOLDERS, [
@@ -112,7 +111,8 @@ parse_sql(Template, ReplaceWith) ->
Template,
#{
replace_with => ReplaceWith,
- placeholders => ?AUTHN_PLACEHOLDERS
+ placeholders => ?AUTHN_PLACEHOLDERS,
+ strip_double_quote => true
}
).
@@ -137,18 +137,6 @@ render_sql_params(ParamList, Credential) ->
#{return => rawlist, var_trans => fun handle_sql_var/2}
).
-with_successful_render(Provider, Fun) when is_function(Fun, 0) ->
- try
- Fun()
- catch
- error:{cannot_get_variable, Name} ->
- ?TRACE_AUTHN(error, "placeholder_interpolation_failed", #{
- provider => Provider,
- placeholder => Name
- }),
- ignore
- end.
-
%% true
is_superuser(#{<<"is_superuser">> := <<"true">>}) ->
#{is_superuser => true};
@@ -230,15 +218,15 @@ without_password(Credential, [Name | Rest]) ->
without_password(Credential, Rest)
end.
-handle_var({var, Name}, undefined) ->
- error({cannot_get_variable, Name});
+handle_var({var, _Name}, undefined) ->
+ <<>>;
handle_var({var, <<"peerhost">>}, PeerHost) ->
emqx_placeholder:bin(inet:ntoa(PeerHost));
handle_var(_, Value) ->
emqx_placeholder:bin(Value).
-handle_sql_var({var, Name}, undefined) ->
- error({cannot_get_variable, Name});
+handle_sql_var({var, _Name}, undefined) ->
+ <<>>;
handle_sql_var({var, <<"peerhost">>}, PeerHost) ->
emqx_placeholder:bin(inet:ntoa(PeerHost));
handle_sql_var(_, Value) ->
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
index 8489debcd..7527edcb3 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
@@ -187,29 +187,25 @@ authenticate(
request_timeout := RequestTimeout
} = State
) ->
- ?WITH_SUCCESSFUL_RENDER(
- begin
- Request = generate_request(Credential, State),
- Response = emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}),
- ?TRACE_AUTHN_PROVIDER("http_response", #{
- request => request_for_log(Credential, State),
- response => response_for_log(Response),
- resource => ResourceId
- }),
- case Response of
- {ok, 204, _Headers} ->
- {ok, #{is_superuser => false}};
- {ok, 200, Headers, Body} ->
- handle_response(Headers, Body);
- {ok, _StatusCode, _Headers} = Response ->
- ignore;
- {ok, _StatusCode, _Headers, _Body} = Response ->
- ignore;
- {error, _Reason} ->
- ignore
- end
- end
- ).
+ Request = generate_request(Credential, State),
+ Response = emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}),
+ ?TRACE_AUTHN_PROVIDER("http_response", #{
+ request => request_for_log(Credential, State),
+ response => response_for_log(Response),
+ resource => ResourceId
+ }),
+ case Response of
+ {ok, 204, _Headers} ->
+ {ok, #{is_superuser => false}};
+ {ok, 200, Headers, Body} ->
+ handle_response(Headers, Body);
+ {ok, _StatusCode, _Headers} = Response ->
+ ignore;
+ {ok, _StatusCode, _Headers, _Body} = Response ->
+ ignore;
+ {error, _Reason} ->
+ ignore
+ end.
destroy(#{resource_id := ResourceId}) ->
_ = emqx_resource:remove_local(ResourceId),
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
index 43a1ebd3b..f7249ae57 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
@@ -162,39 +162,35 @@ authenticate(
resource_id := ResourceId
} = State
) ->
- ?WITH_SUCCESSFUL_RENDER(
- begin
- Filter = emqx_authn_utils:render_deep(FilterTemplate, Credential),
- case emqx_resource:query(ResourceId, {find_one, Collection, Filter, #{}}) of
- undefined ->
- ignore;
- {error, Reason} ->
- ?TRACE_AUTHN_PROVIDER(error, "mongodb_query_failed", #{
+ Filter = emqx_authn_utils:render_deep(FilterTemplate, Credential),
+ case emqx_resource:query(ResourceId, {find_one, Collection, Filter, #{}}) of
+ undefined ->
+ ignore;
+ {error, Reason} ->
+ ?TRACE_AUTHN_PROVIDER(error, "mongodb_query_failed", #{
+ resource => ResourceId,
+ collection => Collection,
+ filter => Filter,
+ reason => Reason
+ }),
+ ignore;
+ Doc ->
+ case check_password(Password, Doc, State) of
+ ok ->
+ {ok, is_superuser(Doc, State)};
+ {error, {cannot_find_password_hash_field, PasswordHashField}} ->
+ ?TRACE_AUTHN_PROVIDER(error, "cannot_find_password_hash_field", #{
resource => ResourceId,
collection => Collection,
filter => Filter,
- reason => Reason
+ document => Doc,
+ password_hash_field => PasswordHashField
}),
ignore;
- Doc ->
- case check_password(Password, Doc, State) of
- ok ->
- {ok, is_superuser(Doc, State)};
- {error, {cannot_find_password_hash_field, PasswordHashField}} ->
- ?TRACE_AUTHN_PROVIDER(error, "cannot_find_password_hash_field", #{
- resource => ResourceId,
- collection => Collection,
- filter => Filter,
- document => Doc,
- password_hash_field => PasswordHashField
- }),
- ignore;
- {error, Reason} ->
- {error, Reason}
- end
+ {error, Reason} ->
+ {error, Reason}
end
- end
- ).
+ end.
%%------------------------------------------------------------------------------
%% Internal functions
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
index 4efa62670..e95302ad4 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
@@ -113,36 +113,32 @@ authenticate(
password_hash_algorithm := Algorithm
}
) ->
- ?WITH_SUCCESSFUL_RENDER(
- begin
- Params = emqx_authn_utils:render_sql_params(TmplToken, Credential),
- case emqx_resource:query(ResourceId, {prepared_query, ?PREPARE_KEY, Params, Timeout}) of
- {ok, _Columns, []} ->
- ignore;
- {ok, Columns, [Row | _]} ->
- Selected = maps:from_list(lists:zip(Columns, Row)),
- case
- emqx_authn_utils:check_password_from_selected_map(
- Algorithm, Selected, Password
- )
- of
- ok ->
- {ok, emqx_authn_utils:is_superuser(Selected)};
- {error, Reason} ->
- {error, Reason}
- end;
+ Params = emqx_authn_utils:render_sql_params(TmplToken, Credential),
+ case emqx_resource:query(ResourceId, {prepared_query, ?PREPARE_KEY, Params, Timeout}) of
+ {ok, _Columns, []} ->
+ ignore;
+ {ok, Columns, [Row | _]} ->
+ Selected = maps:from_list(lists:zip(Columns, Row)),
+ case
+ emqx_authn_utils:check_password_from_selected_map(
+ Algorithm, Selected, Password
+ )
+ of
+ ok ->
+ {ok, emqx_authn_utils:is_superuser(Selected)};
{error, Reason} ->
- ?TRACE_AUTHN_PROVIDER(error, "mysql_query_failed", #{
- resource => ResourceId,
- tmpl_token => TmplToken,
- params => Params,
- timeout => Timeout,
- reason => Reason
- }),
- ignore
- end
- end
- ).
+ {error, Reason}
+ end;
+ {error, Reason} ->
+ ?TRACE_AUTHN_PROVIDER(error, "mysql_query_failed", #{
+ resource => ResourceId,
+ tmpl_token => TmplToken,
+ params => Params,
+ timeout => Timeout,
+ reason => Reason
+ }),
+ ignore
+ end.
parse_config(
#{
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl
index f8b47959a..2962308ab 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl
@@ -115,35 +115,31 @@ authenticate(
password_hash_algorithm := Algorithm
}
) ->
- ?WITH_SUCCESSFUL_RENDER(
- begin
- Params = emqx_authn_utils:render_sql_params(PlaceHolders, Credential),
- case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Params}) of
- {ok, _Columns, []} ->
- ignore;
- {ok, Columns, [Row | _]} ->
- NColumns = [Name || #column{name = Name} <- Columns],
- Selected = maps:from_list(lists:zip(NColumns, erlang:tuple_to_list(Row))),
- case
- emqx_authn_utils:check_password_from_selected_map(
- Algorithm, Selected, Password
- )
- of
- ok ->
- {ok, emqx_authn_utils:is_superuser(Selected)};
- {error, Reason} ->
- {error, Reason}
- end;
+ Params = emqx_authn_utils:render_sql_params(PlaceHolders, Credential),
+ case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Params}) of
+ {ok, _Columns, []} ->
+ ignore;
+ {ok, Columns, [Row | _]} ->
+ NColumns = [Name || #column{name = Name} <- Columns],
+ Selected = maps:from_list(lists:zip(NColumns, erlang:tuple_to_list(Row))),
+ case
+ emqx_authn_utils:check_password_from_selected_map(
+ Algorithm, Selected, Password
+ )
+ of
+ ok ->
+ {ok, emqx_authn_utils:is_superuser(Selected)};
{error, Reason} ->
- ?TRACE_AUTHN_PROVIDER(error, "postgresql_query_failed", #{
- resource => ResourceId,
- params => Params,
- reason => Reason
- }),
- ignore
- end
- end
- ).
+ {error, Reason}
+ end;
+ {error, Reason} ->
+ ?TRACE_AUTHN_PROVIDER(error, "postgresql_query_failed", #{
+ resource => ResourceId,
+ params => Params,
+ reason => Reason
+ }),
+ ignore
+ end.
parse_config(
#{
diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl
index 4cc00322f..69d01f7e8 100644
--- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl
+++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl
@@ -133,15 +133,14 @@ authenticate(
password_hash_algorithm := Algorithm
}
) ->
- ?WITH_SUCCESSFUL_RENDER(
- begin
- NKey = emqx_authn_utils:render_str(KeyTemplate, Credential),
- Command = [CommandName, NKey | Fields],
- case emqx_resource:query(ResourceId, {cmd, Command}) of
- {ok, []} ->
- ignore;
- {ok, Values} ->
- Selected = merge(Fields, Values),
+ NKey = emqx_authn_utils:render_str(KeyTemplate, Credential),
+ Command = [CommandName, NKey | Fields],
+ case emqx_resource:query(ResourceId, {cmd, Command}) of
+ {ok, []} ->
+ ignore;
+ {ok, Values} ->
+ case merge(Fields, Values) of
+ Selected when Selected =/= #{} ->
case
emqx_authn_utils:check_password_from_selected_map(
Algorithm, Selected, Password
@@ -149,21 +148,28 @@ authenticate(
of
ok ->
{ok, emqx_authn_utils:is_superuser(Selected)};
- {error, _Reason} ->
- ignore
+ {error, _Reason} = Error ->
+ Error
end;
- {error, Reason} ->
- ?TRACE_AUTHN_PROVIDER(error, "redis_query_failed", #{
+ _ ->
+ ?TRACE_AUTHN_PROVIDER(info, "redis_query_not_matched", #{
resource => ResourceId,
cmd => Command,
keys => NKey,
- fields => Fields,
- reason => Reason
+ fields => Fields
}),
ignore
- end
- end
- ).
+ end;
+ {error, Reason} ->
+ ?TRACE_AUTHN_PROVIDER(error, "redis_query_failed", #{
+ resource => ResourceId,
+ cmd => Command,
+ keys => NKey,
+ fields => Fields,
+ reason => Reason
+ }),
+ ignore
+ end.
%%------------------------------------------------------------------------------
%% Internal functions
diff --git a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl
index db22a6cfe..b063bd833 100644
--- a/apps/emqx_authn/test/emqx_authn_http_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_http_SUITE.erl
@@ -166,6 +166,49 @@ test_user_auth(#{
?GLOBAL
).
+t_no_value_for_placeholder(_Config) ->
+ Handler = fun(Req0, State) ->
+ {ok, RawBody, Req1} = cowboy_req:read_body(Req0),
+ #{
+ <<"cert_subject">> := <<"">>,
+ <<"cert_common_name">> := <<"">>
+ } = jiffy:decode(RawBody, [return_maps]),
+ Req = cowboy_req:reply(
+ 200,
+ #{<<"content-type">> => <<"application/json">>},
+ jiffy:encode(#{result => allow, is_superuser => false}),
+ Req1
+ ),
+ {ok, Req, State}
+ end,
+
+ SpecificConfgParams = #{
+ <<"method">> => <<"post">>,
+ <<"headers">> => #{<<"content-type">> => <<"application/json">>},
+ <<"body">> => #{
+ <<"cert_subject">> => ?PH_CERT_SUBJECT,
+ <<"cert_common_name">> => ?PH_CERT_CN_NAME
+ }
+ },
+
+ AuthConfig = maps:merge(raw_http_auth_config(), SpecificConfgParams),
+
+ {ok, _} = emqx:update_config(
+ ?PATH,
+ {create_authenticator, ?GLOBAL, AuthConfig}
+ ),
+
+ ok = emqx_authn_http_test_server:set_handler(Handler),
+
+ Credentials = maps:without([cert_subject, cert_common_name], ?CREDENTIALS),
+
+ ?assertMatch({ok, _}, emqx_access_control:authenticate(Credentials)),
+
+ emqx_authn_test_lib:delete_authenticators(
+ [authentication],
+ ?GLOBAL
+ ).
+
t_destroy(_Config) ->
AuthConfig = raw_http_auth_config(),
@@ -247,27 +290,6 @@ t_update(_Config) ->
emqx_access_control:authenticate(?CREDENTIALS)
).
-t_interpolation_error(_Config) ->
- {ok, _} = emqx:update_config(
- ?PATH,
- {create_authenticator, ?GLOBAL, raw_http_auth_config()}
- ),
-
- Headers = #{<<"content-type">> => <<"application/json">>},
- Response = ?SERVER_RESPONSE_JSON(allow),
-
- ok = emqx_authn_http_test_server:set_handler(
- fun(Req0, State) ->
- Req = cowboy_req:reply(200, Headers, Response, Req0),
- {ok, Req, State}
- end
- ),
-
- ?assertMatch(
- ?EXCEPTION_DENY,
- emqx_access_control:authenticate(maps:without([username], ?CREDENTIALS))
- ).
-
t_is_superuser(_Config) ->
Config = raw_http_auth_config(),
{ok, _} = emqx:update_config(
@@ -431,26 +453,6 @@ samples() ->
result => {ok, #{is_superuser => false, user_property => #{}}}
},
- %% simple get request, no username
- #{
- handler => fun(Req0, State) ->
- #{
- username := <<"plain">>,
- password := <<"plain">>
- } = cowboy_req:match_qs([username, password], Req0),
-
- Req = cowboy_req:reply(
- 200,
- #{<<"content-type">> => <<"application/json">>},
- jiffy:encode(#{result => allow, is_superuser => false}),
- Req0
- ),
- {ok, Req, State}
- end,
- config_params => #{},
- result => {ok, #{is_superuser => false, user_property => #{}}}
- },
-
%% get request with json body response
#{
handler => fun(Req0, State) ->
diff --git a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl
index ddde18c49..2f7dd2391 100644
--- a/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl
@@ -288,20 +288,6 @@ raw_mongo_auth_config() ->
user_seeds() ->
[
- #{
- data => #{
- username => <<"plain">>,
- password_hash => <<"plainsalt">>,
- salt => <<"salt">>,
- is_superuser => <<"1">>
- },
- credentials => #{
- password => <<"plain">>
- },
- config_params => #{},
- result => {error, not_authorized}
- },
-
#{
data => #{
username => <<"plain">>,
diff --git a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
index 175aa7f1d..bd6a0159a 100644
--- a/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
@@ -258,20 +258,6 @@ raw_mysql_auth_config() ->
user_seeds() ->
[
- #{
- data => #{
- username => "plain",
- password_hash => "plainsalt",
- salt => "salt",
- is_superuser_str => "1"
- },
- credentials => #{
- password => <<"plain">>
- },
- config_params => #{},
- result => {error, not_authorized}
- },
-
#{
data => #{
username => "plain",
@@ -332,6 +318,32 @@ user_seeds() ->
result => {ok, #{is_superuser => true}}
},
+ %% strip double quote support
+ #{
+ data => #{
+ username => "sha256",
+ password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf",
+ salt => "salt",
+ is_superuser_int => 1
+ },
+ credentials => #{
+ username => <<"sha256">>,
+ password => <<"sha256">>
+ },
+ config_params => #{
+ <<"query">> =>
+ <<
+ "SELECT password_hash, salt, is_superuser_int as is_superuser\n"
+ " FROM users where username = \"${username}\" LIMIT 1"
+ >>,
+ <<"password_hash_algorithm">> => #{
+ <<"name">> => <<"sha256">>,
+ <<"salt_position">> => <<"prefix">>
+ }
+ },
+ result => {ok, #{is_superuser => true}}
+ },
+
#{
data => #{
username => "sha256",
diff --git a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl
index 02095c07d..41b07bfc3 100644
--- a/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl
@@ -320,20 +320,6 @@ raw_pgsql_auth_config() ->
user_seeds() ->
[
- #{
- data => #{
- username => "plain",
- password_hash => "plainsalt",
- salt => "salt",
- is_superuser_str => "1"
- },
- credentials => #{
- password => <<"plain">>
- },
- config_params => #{},
- result => {error, not_authorized}
- },
-
#{
data => #{
username => "plain",
@@ -394,6 +380,32 @@ user_seeds() ->
result => {ok, #{is_superuser => true}}
},
+ %% strip double quote support
+ #{
+ data => #{
+ username => "sha256",
+ password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf",
+ salt => "salt",
+ is_superuser_int => 1
+ },
+ credentials => #{
+ username => <<"sha256">>,
+ password => <<"sha256">>
+ },
+ config_params => #{
+ <<"query">> =>
+ <<
+ "SELECT password_hash, salt, is_superuser_int as is_superuser\n"
+ " FROM users where username = \"${username}\" LIMIT 1"
+ >>,
+ <<"password_hash_algorithm">> => #{
+ <<"name">> => <<"sha256">>,
+ <<"salt_position">> => <<"prefix">>
+ }
+ },
+ result => {ok, #{is_superuser => true}}
+ },
+
#{
data => #{
username => "sha256",
diff --git a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl
index f9ed8bcb1..cbf094549 100644
--- a/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl
+++ b/apps/emqx_authn/test/emqx_authn_redis_SUITE.erl
@@ -161,11 +161,13 @@ t_authenticate(_Config) ->
user_seeds()
).
-test_user_auth(#{
- credentials := Credentials0,
- config_params := SpecificConfigParams,
- result := Result
-}) ->
+test_user_auth(
+ #{
+ credentials := Credentials0,
+ config_params := SpecificConfigParams,
+ result := Result
+ } = Config
+) ->
AuthConfig = maps:merge(raw_redis_auth_config(), SpecificConfigParams),
{ok, _} = emqx:update_config(
@@ -183,14 +185,12 @@ test_user_auth(#{
?assertEqual(Result, emqx_access_control:authenticate(Credentials)),
- AuthnResult =
- case Result of
- {error, _} ->
- ignore;
- Any ->
- Any
- end,
- ?assertEqual(AuthnResult, emqx_authn_redis:authenticate(Credentials, State)),
+ case maps:get(redis_result, Config, undefined) of
+ undefined ->
+ ok;
+ RedisResult ->
+ ?assertEqual(RedisResult, emqx_authn_redis:authenticate(Credentials, State))
+ end,
emqx_authn_test_lib:delete_authenticators(
[authentication],
@@ -292,20 +292,6 @@ raw_redis_auth_config() ->
user_seeds() ->
[
- #{
- data => #{
- password_hash => <<"plainsalt">>,
- salt => <<"salt">>,
- is_superuser => <<"1">>
- },
- credentials => #{
- password => <<"plain">>
- },
- key => <<"mqtt_user:plain">>,
- config_params => #{},
- result => {error, not_authorized}
- },
-
#{
data => #{
password_hash => <<"plainsalt">>,
@@ -478,7 +464,7 @@ user_seeds() ->
<<"cmd">> => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>,
<<"password_hash_algorithm">> => #{<<"name">> => <<"bcrypt">>}
},
- result => {error, not_authorized}
+ result => {error, bad_username_or_password}
},
#{
@@ -547,6 +533,23 @@ user_seeds() ->
}
},
result => {ok, #{is_superuser => true}}
+ },
+
+ %% user not exists
+ #{
+ data => #{
+ password_hash => <<"plainsalt">>,
+ salt => <<"salt">>,
+ is_superuser => <<"1">>
+ },
+ credentials => #{
+ username => <<"not_exists">>,
+ password => <<"plain">>
+ },
+ key => <<"mqtt_user:plain">>,
+ config_params => #{},
+ result => {error, not_authorized},
+ redis_result => ignore
}
].
diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src
index 4a2186066..3dd2705d1 100644
--- a/apps/emqx_authz/src/emqx_authz.app.src
+++ b/apps/emqx_authz/src/emqx_authz.app.src
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_authz, [
{description, "An OTP application"},
- {vsn, "0.1.5"},
+ {vsn, "0.1.6"},
{registered, []},
{mod, {emqx_authz_app, []}},
{applications, [
diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl
index 3b175fc31..52f03fa90 100644
--- a/apps/emqx_authz/src/emqx_authz.erl
+++ b/apps/emqx_authz/src/emqx_authz.erl
@@ -391,14 +391,6 @@ do_authorize(
Matched ->
{Matched, Type}
catch
- error:{cannot_get_variable, Name} ->
- emqx_metrics_worker:inc(authz_metrics, Type, nomatch),
- ?SLOG(warning, #{
- msg => "placeholder_interpolation_failed",
- placeholder => Name,
- authorize_type => Type
- }),
- do_authorize(Client, PubSub, Topic, Tail);
Class:Reason:Stacktrace ->
emqx_metrics_worker:inc(authz_metrics, Type, nomatch),
?SLOG(warning, #{
diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl
index cb4c6f631..9ff65f8a5 100644
--- a/apps/emqx_authz/src/emqx_authz_api_sources.erl
+++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl
@@ -223,7 +223,7 @@ sources(get, _) ->
])
end;
(Source, AccIn) ->
- lists:append(AccIn, [drop_invalid_certs(Source)])
+ lists:append(AccIn, [Source])
end,
[],
get_raw_sources()
@@ -257,7 +257,7 @@ source(get, #{bindings := #{type := Type}}) ->
}}
end;
[Source] ->
- {200, drop_invalid_certs(Source)}
+ {200, Source}
end;
source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file">>} = Body}) ->
update_authz_file(Body);
@@ -511,11 +511,6 @@ update_config(Cmd, Sources) ->
}}
end.
-drop_invalid_certs(#{<<"ssl">> := SSL} = Source) when SSL =/= undefined ->
- Source#{<<"ssl">> => emqx_tls_lib:drop_invalid_certs(SSL)};
-drop_invalid_certs(Source) ->
- Source.
-
parameters_field() ->
[
{type,
diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl
index 69f21932a..246d9c294 100644
--- a/apps/emqx_authz/src/emqx_authz_http.erl
+++ b/apps/emqx_authz/src/emqx_authz_http.erl
@@ -45,7 +45,9 @@
?PH_PROTONAME,
?PH_MOUNTPOINT,
?PH_TOPIC,
- ?PH_ACTION
+ ?PH_ACTION,
+ ?PH_CERT_SUBJECT,
+ ?PH_CERT_CN_NAME
]).
description() ->
diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl
index ac450e4cc..931d83c07 100644
--- a/apps/emqx_authz/src/emqx_authz_mongodb.erl
+++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl
@@ -40,7 +40,9 @@
-define(PLACEHOLDERS, [
?PH_USERNAME,
?PH_CLIENTID,
- ?PH_PEERHOST
+ ?PH_PEERHOST,
+ ?PH_CERT_CN_NAME,
+ ?PH_CERT_SUBJECT
]).
description() ->
diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl
index d364bc5fa..1bf5d774c 100644
--- a/apps/emqx_authz/src/emqx_authz_utils.erl
+++ b/apps/emqx_authz/src/emqx_authz_utils.erl
@@ -110,7 +110,8 @@ parse_sql(Template, ReplaceWith, PlaceHolders) ->
Template,
#{
replace_with => ReplaceWith,
- placeholders => PlaceHolders
+ placeholders => PlaceHolders,
+ strip_double_quote => true
}
).
@@ -181,15 +182,15 @@ convert_client_var({dn, DN}) -> {cert_subject, DN};
convert_client_var({protocol, Proto}) -> {proto_name, Proto};
convert_client_var(Other) -> Other.
-handle_var({var, Name}, undefined) ->
- error({cannot_get_variable, Name});
+handle_var({var, _Name}, undefined) ->
+ <<>>;
handle_var({var, <<"peerhost">>}, IpAddr) ->
inet_parse:ntoa(IpAddr);
handle_var(_Name, Value) ->
emqx_placeholder:bin(Value).
-handle_sql_var({var, Name}, undefined) ->
- error({cannot_get_variable, Name});
+handle_sql_var({var, _Name}, undefined) ->
+ <<>>;
handle_sql_var({var, <<"peerhost">>}, IpAddr) ->
inet_parse:ntoa(IpAddr);
handle_sql_var(_Name, Value) ->
diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl
index 3ee1a94e8..f602acedc 100644
--- a/apps/emqx_authz/test/emqx_authz_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl
@@ -19,9 +19,12 @@
-compile(export_all).
-include("emqx_authz.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
@@ -61,10 +64,26 @@ end_per_suite(_Config) ->
meck:unload(emqx_resource),
ok.
+init_per_testcase(TestCase, Config) when
+ TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament;
+ TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament
+->
+ {ok, _} = emqx_authz:update(?CMD_REPLACE, []),
+ {ok, _} = emqx:update_config([authorization, deny_action], disconnect),
+ Config;
init_per_testcase(_, Config) ->
{ok, _} = emqx_authz:update(?CMD_REPLACE, []),
Config.
+end_per_testcase(TestCase, _Config) when
+ TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament;
+ TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament
+->
+ {ok, _} = emqx:update_config([authorization, deny_action], ignore),
+ ok;
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
set_special_configs(emqx_authz) ->
{ok, _} = emqx:update_config([authorization, cache, enable], false),
{ok, _} = emqx:update_config([authorization, no_match], deny),
@@ -139,6 +158,15 @@ set_special_configs(_App) ->
"\n{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}."
>>
}).
+-define(SOURCE7, #{
+ <<"type">> => <<"file">>,
+ <<"enable">> => true,
+ <<"rules">> =>
+ <<
+ "{allow,{username,\"some_client\"},publish,[\"some_client/lwt\"]}.\n"
+ "{deny, all}."
+ >>
+}).
%%------------------------------------------------------------------------------
%% Testcases
@@ -287,5 +315,87 @@ t_get_enabled_authzs_some_enabled(_Config) ->
{ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE4]),
?assertEqual([postgresql], emqx_authz:get_enabled_authzs()).
+t_subscribe_deny_disconnect_publishes_last_will_testament(_Config) ->
+ {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE7]),
+ {ok, C} = emqtt:start_link([
+ {username, <<"some_client">>},
+ {will_topic, <<"some_client/lwt">>},
+ {will_payload, <<"should be published">>}
+ ]),
+ {ok, _} = emqtt:connect(C),
+ ok = emqx:subscribe(<<"some_client/lwt">>),
+ process_flag(trap_exit, true),
+
+ try
+ emqtt:subscribe(C, <<"unauthorized">>),
+ error(should_have_disconnected)
+ catch
+ exit:{{shutdown, tcp_closed}, _} ->
+ ok
+ end,
+
+ receive
+ {deliver, <<"some_client/lwt">>, #message{payload = <<"should be published">>}} ->
+ ok
+ after 2_000 ->
+ error(lwt_not_published)
+ end,
+
+ ok.
+
+t_publish_deny_disconnect_publishes_last_will_testament(_Config) ->
+ {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE7]),
+ {ok, C} = emqtt:start_link([
+ {username, <<"some_client">>},
+ {will_topic, <<"some_client/lwt">>},
+ {will_payload, <<"should be published">>}
+ ]),
+ {ok, _} = emqtt:connect(C),
+ ok = emqx:subscribe(<<"some_client/lwt">>),
+ process_flag(trap_exit, true),
+
+ %% disconnect is async
+ Ref = monitor(process, C),
+ emqtt:publish(C, <<"some/topic">>, <<"unauthorized">>),
+ receive
+ {'DOWN', Ref, process, C, _} ->
+ ok
+ after 1_000 ->
+ error(client_should_have_been_disconnected)
+ end,
+ receive
+ {deliver, <<"some_client/lwt">>, #message{payload = <<"should be published">>}} ->
+ ok
+ after 2_000 ->
+ error(lwt_not_published)
+ end,
+
+ ok.
+
+t_publish_last_will_testament_denied_topic(_Config) ->
+ {ok, C} = emqtt:start_link([
+ {will_topic, <<"$SYS/lwt">>},
+ {will_payload, <<"should not be published">>}
+ ]),
+ {ok, _} = emqtt:connect(C),
+ ok = emqx:subscribe(<<"$SYS/lwt">>),
+ unlink(C),
+ ok = snabbkaffe:start_trace(),
+ {true, {ok, _}} = ?wait_async_action(
+ exit(C, kill),
+ #{?snk_kind := last_will_testament_publish_denied},
+ 1_000
+ ),
+ ok = snabbkaffe:stop(),
+
+ receive
+ {deliver, <<"$SYS/lwt">>, #message{payload = <<"should not be published">>}} ->
+ error(lwt_should_not_be_published_to_forbidden_topic)
+ after 1_000 ->
+ ok
+ end,
+
+ ok.
+
stop_apps(Apps) ->
lists:foreach(fun application:stop/1, Apps).
diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
index 4b5ad7cbf..628e8dbfe 100644
--- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl
@@ -22,6 +22,7 @@
-include("emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
+-include_lib("emqx/include/emqx_placeholder.hrl").
-define(HTTP_PORT, 33333).
-define(HTTP_PATH, "/authz/[...]").
@@ -303,7 +304,7 @@ t_json_body(_Config) ->
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
).
-t_form_body(_Config) ->
+t_placeholder_and_body(_Config) ->
ok = setup_handler_and_config(
fun(Req0, State) ->
?assertEqual(
@@ -321,7 +322,9 @@ t_form_body(_Config) ->
<<"proto_name">> := <<"MQTT">>,
<<"mountpoint">> := <<"MOUNTPOINT">>,
<<"topic">> := <<"t">>,
- <<"action">> := <<"publish">>
+ <<"action">> := <<"publish">>,
+ <<"CN">> := ?PH_CERT_CN_NAME,
+ <<"CS">> := ?PH_CERT_SUBJECT
},
jiffy:decode(PostVars, [return_maps])
),
@@ -336,7 +339,9 @@ t_form_body(_Config) ->
<<"proto_name">> => <<"${proto_name}">>,
<<"mountpoint">> => <<"${mountpoint}">>,
<<"topic">> => <<"${topic}">>,
- <<"action">> => <<"${action}">>
+ <<"action">> => <<"${action}">>,
+ <<"CN">> => ?PH_CERT_CN_NAME,
+ <<"CS">> => ?PH_CERT_SUBJECT
},
<<"headers">> => #{<<"content-type">> => <<"application/x-www-form-urlencoded">>}
}
@@ -349,6 +354,48 @@ t_form_body(_Config) ->
protocol => <<"MQTT">>,
mountpoint => <<"MOUNTPOINT">>,
zone => default,
+ listener => {tcp, default},
+ cn => ?PH_CERT_CN_NAME,
+ dn => ?PH_CERT_SUBJECT
+ },
+
+ ?assertEqual(
+ allow,
+ emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
+ ).
+
+t_no_value_for_placeholder(_Config) ->
+ ok = setup_handler_and_config(
+ fun(Req0, State) ->
+ ?assertEqual(
+ <<"/authz/users/">>,
+ cowboy_req:path(Req0)
+ ),
+
+ {ok, RawBody, Req1} = cowboy_req:read_body(Req0),
+
+ ?assertMatch(
+ #{
+ <<"mountpoint">> := <<"[]">>
+ },
+ jiffy:decode(RawBody, [return_maps])
+ ),
+ {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State}
+ end,
+ #{
+ <<"method">> => <<"post">>,
+ <<"body">> => #{
+ <<"mountpoint">> => <<"[${mountpoint}]">>
+ }
+ }
+ ),
+
+ ClientInfo = #{
+ clientid => <<"client id">>,
+ username => <<"user name">>,
+ peerhost => {127, 0, 0, 1},
+ protocol => <<"MQTT">>,
+ zone => default,
listener => {tcp, default}
},
diff --git a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl
index 5e5a6ca1e..f080f7e72 100644
--- a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl
@@ -18,8 +18,8 @@
-compile(nowarn_export_all).
-compile(export_all).
--include("emqx_connector.hrl").
-include("emqx_authz.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
@@ -188,6 +188,46 @@ t_lookups(_Config) ->
#{<<"filter">> => #{<<"peerhost">> => <<"${peerhost}">>}}
),
+ ok = emqx_authz_test_lib:test_samples(
+ ClientInfo,
+ [
+ {allow, subscribe, <<"a">>},
+ {deny, subscribe, <<"b">>}
+ ]
+ ),
+
+ ByCN = #{
+ <<"CN">> => <<"cn">>,
+ <<"topics">> => [<<"a">>],
+ <<"action">> => <<"all">>,
+ <<"permission">> => <<"allow">>
+ },
+
+ ok = setup_samples([ByCN]),
+ ok = setup_config(
+ #{<<"filter">> => #{<<"CN">> => ?PH_CERT_CN_NAME}}
+ ),
+
+ ok = emqx_authz_test_lib:test_samples(
+ ClientInfo,
+ [
+ {allow, subscribe, <<"a">>},
+ {deny, subscribe, <<"b">>}
+ ]
+ ),
+
+ ByDN = #{
+ <<"DN">> => <<"dn">>,
+ <<"topics">> => [<<"a">>],
+ <<"action">> => <<"all">>,
+ <<"permission">> => <<"allow">>
+ },
+
+ ok = setup_samples([ByDN]),
+ ok = setup_config(
+ #{<<"filter">> => #{<<"DN">> => ?PH_CERT_SUBJECT}}
+ ),
+
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
index ce8d03984..930426318 100644
--- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
@@ -202,6 +202,34 @@ t_lookups(_Config) ->
}
),
+ ok = emqx_authz_test_lib:test_samples(
+ ClientInfo,
+ [
+ {allow, subscribe, <<"a">>},
+ {deny, subscribe, <<"b">>}
+ ]
+ ),
+
+ %% strip double quote support
+
+ ok = init_table(),
+ ok = q(
+ <<
+ "INSERT INTO acl(clientid, topic, permission, action)"
+ "VALUES(?, ?, ?, ?)"
+ >>,
+ [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>]
+ ),
+
+ ok = setup_config(
+ #{
+ <<"query">> => <<
+ "SELECT permission, action, topic "
+ "FROM acl WHERE clientid = \"${clientid}\""
+ >>
+ }
+ ),
+
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
diff --git a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
index d4aaf7077..fa1672ba7 100644
--- a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
+++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
@@ -202,6 +202,34 @@ t_lookups(_Config) ->
}
),
+ ok = emqx_authz_test_lib:test_samples(
+ ClientInfo,
+ [
+ {allow, subscribe, <<"a">>},
+ {deny, subscribe, <<"b">>}
+ ]
+ ),
+
+ %% strip double quote support
+
+ ok = init_table(),
+ ok = insert(
+ <<
+ "INSERT INTO acl(clientid, topic, permission, action)"
+ "VALUES($1, $2, $3, $4)"
+ >>,
+ [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>]
+ ),
+
+ ok = setup_config(
+ #{
+ <<"query">> => <<
+ "SELECT permission, action, topic "
+ "FROM acl WHERE clientid = \"${clientid}\""
+ >>
+ }
+ ),
+
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src
index 937f201bb..1c5627a1f 100644
--- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src
+++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_auto_subscribe, [
{description, "An OTP application"},
- {vsn, "0.1.1"},
+ {vsn, "0.1.2"},
{registered, []},
{mod, {emqx_auto_subscribe_app, []}},
{applications, [
diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl
index 4edb709b9..4d48bfced 100644
--- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl
+++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl
@@ -44,14 +44,14 @@ schema("/mqtt/auto_subscribe") ->
'operationId' => auto_subscribe,
get => #{
description => ?DESC(list_auto_subscribe_api),
- tags => [<<"Auto subscribe">>],
+ tags => [<<"Auto Subscribe">>],
responses => #{
200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe")
}
},
put => #{
description => ?DESC(update_auto_subscribe_api),
- tags => [<<"Auto subscribe">>],
+ tags => [<<"Auto Subscribe">>],
'requestBody' => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"),
responses => #{
200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"),
diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl
index e28df9176..37a42ab3d 100644
--- a/apps/emqx_bridge/src/emqx_bridge_api.erl
+++ b/apps/emqx_bridge/src/emqx_bridge_api.erl
@@ -584,10 +584,9 @@ pick_bridges_by_id(Type, Name, BridgesAllNodes) ->
format_bridge_info([FirstBridge | _] = Bridges) ->
Res = maps:remove(node, FirstBridge),
- NRes = emqx_connector_ssl:drop_invalid_certs(Res),
NodeStatus = collect_status(Bridges),
NodeMetrics = collect_metrics(Bridges),
- NRes#{
+ Res#{
status => aggregate_status(NodeStatus),
node_status => NodeStatus,
metrics => aggregate_metrics(NodeMetrics),
diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src
index a8707025a..c0d5e1f44 100644
--- a/apps/emqx_conf/src/emqx_conf.app.src
+++ b/apps/emqx_conf/src/emqx_conf.app.src
@@ -1,6 +1,6 @@
{application, emqx_conf, [
{description, "EMQX configuration management"},
- {vsn, "0.1.4"},
+ {vsn, "0.1.5"},
{registered, []},
{mod, {emqx_conf_app, []}},
{applications, [kernel, stdlib]},
diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl
index 3061223cf..43a468762 100644
--- a/apps/emqx_conf/src/emqx_conf_app.erl
+++ b/apps/emqx_conf/src/emqx_conf_app.erl
@@ -152,11 +152,17 @@ copy_override_conf_from_core_node() ->
_ ->
[{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
#{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
- Msg = #{
+ ?SLOG(debug, #{
msg => "copy_overide_conf_from_core_node_success",
- node => Node
- },
- ?SLOG(debug, Msg),
+ node => Node,
+ cluster_override_conf_file => application:get_env(
+ emqx, cluster_override_conf_file
+ ),
+ local_override_conf_file => application:get_env(
+ emqx, local_override_conf_file
+ ),
+ data_dir => emqx:data_dir()
+ }),
ok = emqx_config:save_to_override_conf(
RawOverrideConf,
#{override_to => cluster}
diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl
index a00bfe6f3..ec61cb508 100644
--- a/apps/emqx_conf/src/emqx_conf_schema.erl
+++ b/apps/emqx_conf/src/emqx_conf_schema.erl
@@ -536,6 +536,15 @@ fields("node") ->
desc => ?DESC(node_applications)
}
)},
+ {"etc_dir",
+ sc(
+ string(),
+ #{
+ desc => ?DESC(node_etc_dir),
+ 'readOnly' => true,
+ deprecated => {since, "5.0.8"}
+ }
+ )},
{"cluster_call",
sc(
?R_REF("cluster_call"),
diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src
index 0734d47b8..06da66398 100644
--- a/apps/emqx_connector/src/emqx_connector.app.src
+++ b/apps/emqx_connector/src/emqx_connector.app.src
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_connector, [
{description, "An OTP application"},
- {vsn, "0.1.5"},
+ {vsn, "0.1.6"},
{registered, []},
{mod, {emqx_connector_app, []}},
{applications, [
diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl
index 5b07c5003..a5e2be521 100644
--- a/apps/emqx_connector/src/emqx_connector_mongo.erl
+++ b/apps/emqx_connector/src/emqx_connector_mongo.erl
@@ -345,7 +345,7 @@ init_worker_options([], Acc) ->
%% ===================================================================
%% Schema funcs
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
server(required) -> true;
server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
server(converter) -> fun to_server_raw/1;
diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl
index 21e201504..d21d373a0 100644
--- a/apps/emqx_connector/src/emqx_connector_mqtt.erl
+++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl
@@ -222,20 +222,20 @@ make_forward_confs(undefined) ->
make_forward_confs(FrowardConf) ->
FrowardConf.
-basic_config(#{
- server := Server,
- reconnect_interval := ReconnIntv,
- proto_ver := ProtoVer,
- bridge_mode := BridgeMode,
- username := User,
- password := Password,
- clean_start := CleanStart,
- keepalive := KeepAlive,
- retry_interval := RetryIntv,
- max_inflight := MaxInflight,
- replayq := ReplayQ,
- ssl := #{enable := EnableSsl} = Ssl
-}) ->
+basic_config(
+ #{
+ server := Server,
+ reconnect_interval := ReconnIntv,
+ proto_ver := ProtoVer,
+ bridge_mode := BridgeMode,
+ clean_start := CleanStart,
+ keepalive := KeepAlive,
+ retry_interval := RetryIntv,
+ max_inflight := MaxInflight,
+ replayq := ReplayQ,
+ ssl := #{enable := EnableSsl} = Ssl
+ } = Conf
+) ->
#{
replayq => ReplayQ,
%% connection opts
@@ -251,8 +251,9 @@ basic_config(#{
%% non-standard mqtt connection packets will be filtered out by LB.
%% So let's disable bridge_mode.
bridge_mode => BridgeMode,
- username => User,
- password => Password,
+ %% should be iolist for emqtt
+ username => maps:get(username, Conf, <<>>),
+ password => maps:get(password, Conf, <<>>),
clean_start => CleanStart,
keepalive => ms_to_s(KeepAlive),
retry_interval => RetryIntv,
diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl
index d6963d04e..79a306b05 100644
--- a/apps/emqx_connector/src/emqx_connector_mysql.erl
+++ b/apps/emqx_connector/src/emqx_connector_mysql.erl
@@ -55,7 +55,7 @@ fields(config) ->
emqx_connector_schema_lib:ssl_fields() ++
emqx_connector_schema_lib:prepare_statement_fields().
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
server(required) -> true;
server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
server(converter) -> fun to_server/1;
diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl
index 43ec6be8b..f642ba75c 100644
--- a/apps/emqx_connector/src/emqx_connector_pgsql.erl
+++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl
@@ -58,7 +58,7 @@ fields(config) ->
emqx_connector_schema_lib:ssl_fields() ++
emqx_connector_schema_lib:prepare_statement_fields().
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
server(required) -> true;
server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
server(converter) -> fun to_server/1;
diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl
index 67310dbac..95677b766 100644
--- a/apps/emqx_connector/src/emqx_connector_redis.erl
+++ b/apps/emqx_connector/src/emqx_connector_redis.erl
@@ -97,7 +97,7 @@ fields(sentinel) ->
redis_fields() ++
emqx_connector_schema_lib:ssl_fields().
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
server(required) -> true;
server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
server(converter) -> fun to_server_raw/1;
diff --git a/apps/emqx_connector/src/emqx_connector_ssl.erl b/apps/emqx_connector/src/emqx_connector_ssl.erl
index 929ffe82e..7f2fc537b 100644
--- a/apps/emqx_connector/src/emqx_connector_ssl.erl
+++ b/apps/emqx_connector/src/emqx_connector_ssl.erl
@@ -18,7 +18,6 @@
-export([
convert_certs/2,
- drop_invalid_certs/1,
clear_certs/2
]).
@@ -61,28 +60,6 @@ clear_certs(RltvDir, #{ssl := OldSSL} = _Config) ->
clear_certs(_RltvDir, _) ->
ok.
-drop_invalid_certs(#{<<"connector">> := Connector} = Config) when
- is_map(Connector)
-->
- SSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
- NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
- new_ssl_config(Config, NewSSL);
-drop_invalid_certs(#{connector := Connector} = Config) when
- is_map(Connector)
-->
- SSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
- NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
- new_ssl_config(Config, NewSSL);
-drop_invalid_certs(#{<<"ssl">> := SSL} = Config) ->
- NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
- new_ssl_config(Config, NewSSL);
-drop_invalid_certs(#{ssl := SSL} = Config) ->
- NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
- new_ssl_config(Config, NewSSL);
-%% for bridges use connector name
-drop_invalid_certs(Config) ->
- Config.
-
new_ssl_config(RltvDir, Config, SSL) ->
case emqx_tls_lib:ensure_ssl_files(RltvDir, SSL) of
{ok, NewSSL} ->
diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
index 438f33b5d..6fa511d00 100644
--- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
+++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
@@ -55,7 +55,7 @@ fields("connector") ->
)},
{server,
sc(
- emqx_schema:ip_port(),
+ emqx_schema:host_port(),
#{
required => true,
desc => ?DESC("server")
@@ -177,7 +177,7 @@ fields("ingress") ->
sc(
binary(),
#{
- default => <<"${payload}">>,
+ default => undefined,
desc => ?DESC("payload")
}
)}
@@ -224,7 +224,7 @@ fields("egress") ->
sc(
binary(),
#{
- default => <<"${payload}">>,
+ default => undefined,
desc => ?DESC("payload")
}
)}
diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src
index adf974243..9d5f85c7b 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard.app.src
+++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src
@@ -2,7 +2,7 @@
{application, emqx_dashboard, [
{description, "EMQX Web Dashboard"},
% strict semver, bump manually!
- {vsn, "5.0.5"},
+ {vsn, "5.0.6"},
{modules, []},
{registered, [emqx_dashboard_sup]},
{applications, [kernel, stdlib, mnesia, minirest, emqx]},
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl
index 139567828..2605ad91e 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl
@@ -51,7 +51,7 @@ schema("/error_codes") ->
get => #{
security => [],
description => <<"API Error Codes">>,
- tags => [<<"Error codes">>],
+ tags => [<<"Error Codes">>],
responses => #{
200 => hoconsc:array(hoconsc:ref(?MODULE, error_code))
}
@@ -63,7 +63,7 @@ schema("/error_codes/:code") ->
get => #{
security => [],
description => <<"API Error Codes">>,
- tags => [<<"Error codes">>],
+ tags => [<<"Error Codes">>],
parameters => [
{code,
hoconsc:mk(hoconsc:enum(emqx_dashboard_error_code:all()), #{
diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
index 52cbc4775..5674d273c 100644
--- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
+++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
@@ -656,8 +656,8 @@ typename_to_spec("file()", _Mod) ->
#{type => string, example => <<"/path/to/file">>};
typename_to_spec("ip_port()", _Mod) ->
#{type => string, example => <<"127.0.0.1:80">>};
-typename_to_spec("ip_ports()", _Mod) ->
- #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>};
+typename_to_spec("host_port()", _Mod) ->
+ #{type => string, example => <<"example.host.domain:80">>};
typename_to_spec("url()", _Mod) ->
#{type => string, example => <<"http://127.0.0.1">>};
typename_to_spec("connect_timeout()", Mod) ->
diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
index b3735109f..cb9c77657 100644
--- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
+++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
@@ -74,14 +74,6 @@ end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_management]),
mria:stop().
-set_special_configs(emqx_management) ->
- Listeners = #{http => #{port => 8081}},
- Config = #{
- listeners => Listeners,
- applications => [#{id => "admin", secret => "public"}]
- },
- emqx_config:put([emqx_management], Config),
- ok;
set_special_configs(emqx_dashboard) ->
emqx_dashboard_api_test_helpers:set_default_config(),
ok;
diff --git a/apps/emqx_exhook/include/emqx_exhook.hrl b/apps/emqx_exhook/include/emqx_exhook.hrl
index 6c386e688..7436b2a1c 100644
--- a/apps/emqx_exhook/include/emqx_exhook.hrl
+++ b/apps/emqx_exhook/include/emqx_exhook.hrl
@@ -43,6 +43,8 @@
{'message.dropped', {emqx_exhook_handler, on_message_dropped, []}}
]).
+-define(SERVER_FORCE_SHUTDOWN_TIMEOUT, 5000).
+
-endif.
-define(CMD_MOVE_FRONT, front).
diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src
index c4a43d846..3f844f61d 100644
--- a/apps/emqx_exhook/src/emqx_exhook.app.src
+++ b/apps/emqx_exhook/src/emqx_exhook.app.src
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_exhook, [
{description, "EMQX Extension for Hook"},
- {vsn, "5.0.4"},
+ {vsn, "5.0.5"},
{modules, []},
{registered, []},
{mod, {emqx_exhook_app, []}},
diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl
index 83d6c4aa8..921a4df68 100644
--- a/apps/emqx_exhook/src/emqx_exhook_api.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_api.erl
@@ -41,7 +41,7 @@
-import(hoconsc, [mk/1, mk/2, ref/1, enum/1, array/1, map/2]).
-import(emqx_dashboard_swagger, [schema_with_example/2, error_codes/2]).
--define(TAGS, [<<"exhooks">>]).
+-define(TAGS, [<<"ExHook">>]).
-define(NOT_FOURD, 'NOT_FOUND').
-define(BAD_REQUEST, 'BAD_REQUEST').
-define(BAD_RPC, 'BAD_RPC').
@@ -483,16 +483,11 @@ err_msg(Msg) -> emqx_misc:readable_error_msg(Msg).
get_raw_config() ->
RawConfig = emqx:get_raw_config([exhook, servers], []),
Schema = #{roots => emqx_exhook_schema:fields(exhook), fields => #{}},
- Conf = #{<<"servers">> => lists:map(fun drop_invalid_certs/1, RawConfig)},
+ Conf = #{<<"servers">> => RawConfig},
Options = #{only_fill_defaults => true},
#{<<"servers">> := Servers} = hocon_tconf:check_plain(Schema, Conf, Options),
Servers.
-drop_invalid_certs(#{<<"ssl">> := SSL} = Conf) when SSL =/= undefined ->
- Conf#{<<"ssl">> => emqx_tls_lib:drop_invalid_certs(SSL)};
-drop_invalid_certs(Conf) ->
- Conf.
-
position_example() ->
#{
front =>
diff --git a/apps/emqx_exhook/src/emqx_exhook_mgr.erl b/apps/emqx_exhook/src/emqx_exhook_mgr.erl
index cf83e8eb9..e58555ca1 100644
--- a/apps/emqx_exhook/src/emqx_exhook_mgr.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_mgr.erl
@@ -21,6 +21,7 @@
-include("emqx_exhook.hrl").
-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% APIs
-export([start_link/0]).
@@ -297,7 +298,8 @@ handle_info(refresh_tick, State) ->
handle_info(_Info, State) ->
{noreply, State}.
-terminate(_Reason, State = #{servers := Servers}) ->
+terminate(Reason, State = #{servers := Servers}) ->
+ _ = unload_exhooks(),
_ = maps:fold(
fun(Name, _, AccIn) ->
do_unload_server(Name, AccIn)
@@ -305,7 +307,7 @@ terminate(_Reason, State = #{servers := Servers}) ->
State,
Servers
),
- _ = unload_exhooks(),
+ ?tp(info, exhook_mgr_terminated, #{reason => Reason, servers => Servers}),
ok.
code_change(_OldVsn, State, _Extra) ->
diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl
index b15724ff2..9c89915aa 100644
--- a/apps/emqx_exhook/src/emqx_exhook_server.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_server.erl
@@ -179,13 +179,16 @@ filter(Ls) ->
-spec unload(server()) -> ok.
unload(#{name := Name, options := ReqOpts, hookspec := HookSpecs}) ->
- _ = do_deinit(Name, ReqOpts),
_ = may_unload_hooks(HookSpecs),
+ _ = do_deinit(Name, ReqOpts),
_ = emqx_exhook_sup:stop_grpc_client_channel(Name),
ok.
do_deinit(Name, ReqOpts) ->
- _ = do_call(Name, undefined, 'on_provider_unloaded', #{}, ReqOpts),
+ %% Override the request timeout to deinit grpc server to
+ %% avoid emqx_exhook_mgr force killed by upper supervisor
+ NReqOpts = ReqOpts#{timeout => ?SERVER_FORCE_SHUTDOWN_TIMEOUT},
+ _ = do_call(Name, undefined, 'on_provider_unloaded', #{}, NReqOpts),
ok.
do_init(ChannName, ReqOpts) ->
diff --git a/apps/emqx_exhook/src/emqx_exhook_sup.erl b/apps/emqx_exhook/src/emqx_exhook_sup.erl
index fb424ddff..cd49d89bb 100644
--- a/apps/emqx_exhook/src/emqx_exhook_sup.erl
+++ b/apps/emqx_exhook/src/emqx_exhook_sup.erl
@@ -16,6 +16,8 @@
-module(emqx_exhook_sup).
+-include("emqx_exhook.hrl").
+
-behaviour(supervisor).
-export([
@@ -28,11 +30,13 @@
stop_grpc_client_channel/1
]).
--define(CHILD(Mod, Type, Args), #{
+-define(DEFAULT_TIMEOUT, 5000).
+
+-define(CHILD(Mod, Type, Args, Timeout), #{
id => Mod,
start => {Mod, start_link, Args},
type => Type,
- shutdown => 15000
+ shutdown => Timeout
}).
%%--------------------------------------------------------------------
@@ -45,7 +49,7 @@ start_link() ->
init([]) ->
_ = emqx_exhook_metrics:init(),
_ = emqx_exhook_mgr:init_ref_counter_table(),
- Mngr = ?CHILD(emqx_exhook_mgr, worker, []),
+ Mngr = ?CHILD(emqx_exhook_mgr, worker, [], force_shutdown_timeout()),
{ok, {{one_for_one, 10, 100}, [Mngr]}}.
%%--------------------------------------------------------------------
@@ -70,3 +74,9 @@ stop_grpc_client_channel(Name) ->
_:_:_ ->
ok
end.
+
+%% Calculate the maximum timeout, which will help to shutdown the
+%% emqx_exhook_mgr process correctly.
+force_shutdown_timeout() ->
+ Factor = max(3, length(emqx:get_config([exhook, servers])) + 1),
+ Factor * ?SERVER_FORCE_SHUTDOWN_TIMEOUT.
diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
index 62606cf18..a0472b2c3 100644
--- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
+++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl
@@ -24,6 +24,7 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_hooks.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(DEFAULT_CLUSTER_NAME_ATOM, emqxcl).
@@ -313,6 +314,40 @@ t_cluster_name(_) ->
),
emqx_exhook_mgr:disable(<<"default">>).
+t_stop_timeout(_) ->
+ snabbkaffe:start_trace(),
+ meck:new(emqx_exhook_demo_svr, [passthrough, no_history]),
+ meck:expect(
+ emqx_exhook_demo_svr,
+ on_provider_unloaded,
+ fun(Req, Md) ->
+ %% ensure sleep time greater than emqx_exhook_mgr shutdown timeout
+ timer:sleep(20000),
+ meck:passthrough([Req, Md])
+ end
+ ),
+
+ %% stop application
+ application:stop(emqx_exhook),
+ ?block_until(#{?snk_kind := exhook_mgr_terminated}, 20000),
+
+ %% all exhook hooked point should be unloaded
+ Mods = lists:flatten(
+ lists:map(
+ fun({hook, _, Cbs}) ->
+ lists:map(fun({callback, {M, _, _}, _, _}) -> M end, Cbs)
+ end,
+ ets:tab2list(emqx_hooks)
+ )
+ ),
+ ?assertEqual(false, lists:any(fun(M) -> M == emqx_exhook_handler end, Mods)),
+
+ %% ensure started for other tests
+ emqx_common_test_helpers:start_apps([emqx_exhook]),
+
+ snabbkaffe:stop(),
+ meck:unload(emqx_exhook_demo_svr).
+
%%--------------------------------------------------------------------
%% Cases Helpers
%%--------------------------------------------------------------------
diff --git a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
index ea8398eeb..b566b7ab2 100644
--- a/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
+++ b/apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
@@ -80,7 +80,16 @@ stop() ->
stop(Name) ->
grpc:stop_server(Name),
- to_atom_name(Name) ! stop.
+ case whereis(to_atom_name(Name)) of
+ undefined ->
+ ok;
+ Pid ->
+ Ref = erlang:monitor(process, Pid),
+ Pid ! stop,
+ receive
+ {'DOWN', Ref, process, Pid, _Reason} -> ok
+ end
+ end.
take() ->
to_atom_name(?NAME) ! {take, self()},
diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf
index 0f04b3938..b7a65bee1 100644
--- a/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf
+++ b/apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf
@@ -2,70 +2,76 @@ emqx_gateway_api_authn {
get_authn {
desc {
- en: """Get the gateway authentication"""
- zh: """获取指定网关认证器"""
+ en: """Gets the configuration of the specified gateway authenticator.
+Returns 404 when gateway or authentication is not enabled."""
+ zh: """获取指定网关认证器的配置
+当网关或认证未启用时，返回 404。"""
}
}
update_authn {
desc {
- en: """Update authentication for the gateway"""
- zh: """更新网关认证器"""
+ en: """Update the configuration of the specified gateway authenticator, or disable the authenticator."""
+ zh: """更新指定网关认证器的配置，或停用认证器。"""
}
}
add_authn {
desc {
- en: """Add authentication for the gateway"""
- zh: """为指定网关新增认证器"""
+ en: """Enables the authenticator for client authentication for the specified gateway. <\br>
+When the authenticator is not configured or turned off, all client connections are assumed to be allowed. <\br>
+Note: Only one authenticator is allowed to be enabled at a time in the gateway, rather than allowing multiple authenticators to be configured to form an authentication chain as in MQTT."""
+ zh: """为指定网关开启认证器实现客户端认证的功能。<\br>
+当未配置认证器或关闭认证器时，则认为允许所有客户端的连接。<\br>
+注：在网关中仅支持添加一个认证器，而不是像 MQTT 一样允许配置多个认证器构成认证链。"""
}
}
delete_authn {
desc {
- en: """Remove the gateway authentication"""
- zh: """删除指定网关的认证器"""
+ en: """Delete the authenticator of the specified gateway."""
+ zh: """删除指定网关的认证器。"""
}
}
list_users {
desc {
- en: """Get the users for the authentication"""
+ en: """Get the users for the authenticator (only supported by built_in_database)."""
zh: """获取用户列表（仅支持 built_in_database 类型的认证器）"""
}
}
add_user {
desc {
- en: """Add user for the authentication"""
+ en: """Add user for the authenticator (only supports built_in_database)."""
zh: """添加用户（仅支持 built_in_database 类型的认证器）"""
}
}
get_user {
desc {
- en: """Get user info from the gateway authentication"""
+ en: """Get user info from the gateway authenticator (only supports built_in_database)"""
zh: """获取用户信息（仅支持 built_in_database 类型的认证器）"""
}
}
update_user {
desc {
- en: """Update the user info for the gateway authentication"""
+ en: """Update the user info for the gateway authenticator (only supports built_in_database)"""
zh: """更新用户信息（仅支持 built_in_database 类型的认证器）"""
}
}
delete_user {
desc {
- en: """Delete the user for the gateway authentication"""
+ en: """Delete the user for the gateway authenticator (only supports built_in_database)"""
zh: """删除用户（仅支持 built_in_database 类型的认证器）"""
}
}
import_users {
desc {
- en: """Import users into the gateway authentication"""
+ en: """Import users into the gateway authenticator (only supports built_in_database)"""
zh: """导入用户（仅支持 built_in_database 类型的认证器）"""
}
}
@@ -79,8 +85,8 @@ emqx_gateway_api_authn {
like_user_id {
desc {
- en: """Fuzzy search by user_id (username or clientid)"""
- zh: """用户 ID （username 或 clientid）模糊搜索"""
+ en: """Fuzzy search using user ID (username or clientid), only supports search by substring."""
+ zh: """使用用户 ID （username 或 clientid）模糊搜索，仅支持按子串的方式进行搜索。"""
}
}
@@ -90,5 +96,4 @@ emqx_gateway_api_authn {
zh: """是否是超级用户"""
}
}
-
}
diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf
index 5ab9277b2..f7271e6a0 100644
--- a/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf
+++ b/apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf
@@ -2,15 +2,17 @@ emqx_gateway_api {
list_gateway {
desc {
- en: """Get gateway list"""
- zh: """获取网关列表"""
+ en: """This API returns an overview info for the specified or all gateways.
+including current running status, number of connections, listener status, etc."""
+ zh: """该接口会返回指定或所有网关的概览状态，
+包括当前状态、连接数、监听器状态等。"""
}
}
enable_gateway {
desc {
- en: """Enable a gateway"""
- zh: """启用某网关"""
+ en: """Enable a gateway by confs."""
+ zh: """使用配置启动某一网关。"""
}
}
@@ -23,15 +25,17 @@ emqx_gateway_api {
delete_gateway {
desc {
- en: """Delete/Unload the gateway"""
- zh: """删除/禁用某网关"""
+ en: """Unload the specified gateway"""
+ zh: """停用指定网关"""
}
}
update_gateway {
desc {
- en: """Update the gateway configurations/status"""
- zh: """更新网关配置或启用状态"""
+ en: """Update the gateway basic configurations and running status.
+Note: The Authentication and Listener configurations should be updated by other special APIs. """
+ zh: """更新指定网关的基础配置、和启用的状态。
+注：认证、和监听器的配置更新需参考对应的 API 接口。"""
}
}
@@ -42,13 +46,33 @@ emqx_gateway_api {
}
}
+ gateway_name_in_qs {
+ desc {
+ en: """Gateway Name.
+It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto`
+"""
+ zh: """网关名称.
+可取值为 `stomp`、`mqttsn`、`coap`、`lwm2m`、`exproto`
+"""
+ }
+ }
+
gateway_status {
desc {
- en: """Gateway Status"""
+ en: """Gateway status"""
zh: """网关启用状态"""
}
}
+ gateway_status_in_qs {
+ desc {
+ en: """Filter gateways by status.
+It is enum with `running`, `stopped`, `unloaded`"""
+ zh: """通过网关状态筛选
+可选值为 `running`、`stopped`、`unloaded`"""
+ }
+ }
+
gateway_created_at {
desc {
en: """The Gateway created datetime"""
diff --git a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf b/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf
index 9c5de67c3..0b7c93afe 100644
--- a/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf
+++ b/apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf
@@ -2,105 +2,109 @@ emqx_gateway_api_listeners {
list_listeners {
desc {
- en: """Get the gateway listeners"""
- zh: """获取网关监听器列表"""
+ en: """Gets a list of gateway listeners. This interface returns all the configs of the listener (including the authenticator on that listener), as well as the status of that listener running in the cluster."""
+ zh: """获取网关监听器列表。该接口会返回监听器所有的配置（包括该监听器上的认证器），同时也会返回该监听器在集群中运行的状态。"""
}
}
add_listener {
desc {
- en: """Create the gateway listener"""
- zh: """为指定网关添加监听器"""
+ en: """Create the gateway listener.
+Note: For listener types not supported by a gateway, this API returns `400: BAD_REQUEST`."""
+ zh: """为指定网关添加监听器。
+注：对于某网关不支持的监听器类型，该接口会返回 `400: BAD_REQUEST`。"""
}
}
get_listener {
desc {
- en: """Get the gateway listener configurations"""
- zh: """获取指定监听器信息"""
+ en: """Get the gateway listener configs"""
+ zh: """获取指定网关监听器的配置。"""
}
}
delete_listener {
desc {
- en: """Delete the gateway listener"""
- zh: """删除监听器"""
+ en: """Delete the gateway listener. All connected clients under the deleted listener will be disconnected."""
+ zh: """删除指定监听器。被删除的监听器下所有已连接的客户端都会离线。"""
}
}
update_listener {
desc {
- en: """Update the gateway listener"""
- zh: """更新监听器"""
+ en: """Update the gateway listener. The listener being updated performs a restart and all clients connected to that listener will be disconnected."""
+ zh: """更新某网关监听器的配置。被更新的监听器会执行重启，所有已连接到该监听器上的客户端都会被断开。"""
}
}
get_listener_authn {
desc {
- en: """Get the listener's authentication info"""
- zh: """获取监听器的认证器信息"""
+ en: """Get the listener's authenticator configs."""
+ zh: """获取监听器的认证器配置。"""
}
}
add_listener_authn {
desc {
- en: """Add authentication for the listener"""
- zh: """为指定监听器添加认证器"""
+ en: """Enable authenticator for specified listener for client authentication.
+When authenticator is enabled for a listener, all clients connecting to that listener will use that authenticator for authentication."""
+ zh: """为指定监听器开启认证器以实现客户端认证的能力。
+当某一监听器开启认证后，所有连接到该监听器的客户端会使用该认证器进行认证。"""
}
}
update_listener_authn {
desc {
- en: """Update authentication for the listener"""
- zh: """更新指定监听上的认证器配置"""
+ en: """Update authenticator configs for the listener, or disable/enable it."""
+ zh: """更新指定监听器的认证器配置，或停用/启用该认证器。"""
}
}
delete_listener_authn {
desc {
- en: """Remove authentication for the listener"""
- zh: """为指定监听器移除认证器"""
+ en: """Remove authenticator for the listener."""
+ zh: """移除指定监听器的认证器。"""
}
}
list_users {
desc {
- en: """Get the users for the authentication"""
+ en: """Get the users for the authenticator (only supported by built_in_database)"""
zh: """获取用户列表（仅支持 built_in_database 类型的认证器）"""
}
}
add_user {
desc {
- en: """Add user for the authentication"""
+ en: """Add user for the authenticator (only supports built_in_database)"""
zh: """添加用户（仅支持 built_in_database 类型的认证器）"""
}
}
get_user {
desc {
- en: """Get user info from the gateway authentication"""
+ en: """Get user info from the gateway authenticator (only supports built_in_database)"""
zh: """获取用户信息（仅支持 built_in_database 类型的认证器）"""
}
}
update_user {
desc {
- en: """Update the user info for the gateway authentication"""
+ en: """Update the user info for the gateway authenticator (only supports built_in_database)"""
zh: """更新用户信息（仅支持 built_in_database 类型的认证器）"""
}
}
delete_user {
desc {
- en: """Delete the user for the gateway authentication"""
+ en: """Delete the user for the gateway authenticator (only supports built_in_database)"""
zh: """删除用户（仅支持 built_in_database 类型的认证器）"""
}
}
import_users {
desc {
- en: """Import users into the gateway authentication"""
+ en: """Import users into the gateway authenticator (only supports built_in_database)"""
zh: """导入用户（仅支持 built_in_database 类型的认证器）"""
}
}
@@ -139,5 +143,4 @@ emqx_gateway_api_listeners {
zh: """当前连接数"""
}
}
-
}
diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
index 9e65c7eea..99ac3a38f 100644
--- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
+++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
@@ -19,6 +19,7 @@
-include_lib("emqx/include/types.hrl").
-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% API
-export([
@@ -51,6 +52,9 @@
%% Internal callback
-export([wakeup_from_hib/2, recvloop/2]).
+%% for channel module
+-export([keepalive_stats/1]).
+
-record(state, {
%% TCP/SSL/UDP/DTLS Wrapped Socket
socket :: {esockd_transport, esockd:socket()} | {udp, _, _},
@@ -240,6 +244,11 @@ esockd_send(Data, #state{
esockd_send(Data, #state{socket = {esockd_transport, Sock}}) ->
esockd_transport:async_send(Sock, Data).
+keepalive_stats(recv) ->
+ emqx_pd:get_counter(recv_pkt);
+keepalive_stats(send) ->
+ emqx_pd:get_counter(send_pkt).
+
is_datadram_socket({esockd_transport, _}) -> false;
is_datadram_socket({udp, _, _}) -> true.
@@ -568,9 +577,15 @@ terminate(
channel = Channel
}
) ->
- ?SLOG(debug, #{msg => "conn_process_terminated", reason => Reason}),
_ = ChannMod:terminate(Reason, Channel),
_ = close_socket(State),
+ ClientId =
+ try ChannMod:info(clientid, Channel) of
+ Id -> Id
+ catch
+ _:_ -> undefined
+ end,
+ ?tp(debug, conn_process_terminated, #{reason => Reason, clientid => ClientId}),
exit(Reason).
%%--------------------------------------------------------------------
@@ -635,28 +650,22 @@ handle_timeout(
Keepalive,
State = #state{
chann_mod = ChannMod,
- socket = Socket,
channel = Channel
}
) when
Keepalive == keepalive;
Keepalive == keepalive_send
->
- Stat =
+ StatVal =
case Keepalive of
- keepalive -> recv_oct;
- keepalive_send -> send_oct
+ keepalive -> keepalive_stats(recv);
+ keepalive_send -> keepalive_stats(send)
end,
case ChannMod:info(conn_state, Channel) of
disconnected ->
{ok, State};
_ ->
- case esockd_getstat(Socket, [Stat]) of
- {ok, [{Stat, RecvOct}]} ->
- handle_timeout(TRef, {Keepalive, RecvOct}, State);
- {error, Reason} ->
- handle_info({sock_error, Reason}, State)
- end
+ handle_timeout(TRef, {Keepalive, StatVal}, State)
end;
handle_timeout(
_TRef,
diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl
index 5e94934da..27cb23ac8 100644
--- a/apps/emqx_gateway/src/coap/emqx_coap_api.erl
+++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl
@@ -48,8 +48,9 @@ schema(?PREFIX ++ "/request") ->
#{
operationId => request,
post => #{
- tags => [<<"CoAP">>],
+ tags => [<<"Gateways">>],
desc => ?DESC(send_coap_request),
+ summary => <<"Send a Request to a Client">>,
parameters => request_parameters(),
requestBody => request_body(),
responses => #{
diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src
index 47245c0a2..98dbd6909 100644
--- a/apps/emqx_gateway/src/emqx_gateway.app.src
+++ b/apps/emqx_gateway/src/emqx_gateway.app.src
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway, [
{description, "The Gateway management application"},
- {vsn, "0.1.5"},
+ {vsn, "0.1.6"},
{registered, []},
{mod, {emqx_gateway_app, []}},
{applications, [kernel, stdlib, grpc, emqx, emqx_authn]},
diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl
index 5ae8fe1e7..00d3d4b17 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api.erl
@@ -164,7 +164,9 @@ schema("/gateways") ->
'operationId' => gateway,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(list_gateway),
+ summary => <<"List All Gateways">>,
parameters => params_gateway_status_in_qs(),
responses =>
?STANDARD_RESP(
@@ -178,7 +180,9 @@ schema("/gateways") ->
},
post =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(enable_gateway),
+ summary => <<"Enable a Gateway">>,
%% TODO: distinguish create & response swagger schema
'requestBody' => schema_gateways_conf(),
responses =>
@@ -190,21 +194,27 @@ schema("/gateways/:name") ->
'operationId' => gateway_insta,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(get_gateway),
+ summary => <<"Get the Gateway">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(#{200 => schema_gateways_conf()})
},
delete =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(delete_gateway),
+ summary => <<"Unload the gateway">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(#{204 => <<"Deleted">>})
},
put =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(update_gateway),
+ summary => <<"Update the gateway confs">>,
parameters => params_gateway_name_in_path(),
'requestBody' => schema_update_gateways_conf(),
responses =>
@@ -224,8 +234,8 @@ params_gateway_name_in_path() ->
binary(),
#{
in => path,
- desc => ?DESC(gateway_name),
- example => <<"">>
+ desc => ?DESC(gateway_name_in_qs),
+ example => <<"stomp">>
}
)}
].
@@ -239,7 +249,7 @@ params_gateway_status_in_qs() ->
#{
in => query,
required => false,
- desc => ?DESC(gateway_status),
+ desc => ?DESC(gateway_status_in_qs),
example => <<"">>
}
)}
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
index 6fd073a3b..bb622894b 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl
@@ -75,6 +75,7 @@ authn(get, #{bindings := #{name := Name0}}) ->
Authn -> {200, Authn}
catch
error:{config_not_found, _} ->
+ %% FIXME: should return 404?
{204}
end
end);
@@ -181,19 +182,23 @@ schema("/gateways/:name/authentication") ->
'operationId' => authn,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(get_authn),
+ summary => <<"Get Authenticator Configuration">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(
#{
200 => schema_authn(),
- 204 => <<"Authentication does not initiated">>
+ 204 => <<"Authenticator doesn't initiated">>
}
)
},
put =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(update_authn),
+ summary => <<"Update Authenticator Configuration">>,
parameters => params_gateway_name_in_path(),
'requestBody' => schema_authn(),
responses =>
@@ -201,7 +206,9 @@ schema("/gateways/:name/authentication") ->
},
post =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(add_authn),
+ summary => <<"Create an Authenticator for a Gateway">>,
parameters => params_gateway_name_in_path(),
'requestBody' => schema_authn(),
responses =>
@@ -209,7 +216,9 @@ schema("/gateways/:name/authentication") ->
},
delete =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(delete_authn),
+ summary => <<"Delete the Gateway Authenticator">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(#{204 => <<"Deleted">>})
@@ -220,7 +229,9 @@ schema("/gateways/:name/authentication/users") ->
'operationId' => users,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(list_users),
+ summary => <<"List users for a Gateway Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_paging_in_qs() ++
params_fuzzy_in_qs(),
@@ -236,7 +247,9 @@ schema("/gateways/:name/authentication/users") ->
},
post =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(add_user),
+ summary => <<"Add User for a Gateway Authenticator">>,
parameters => params_gateway_name_in_path(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
ref(emqx_authn_api, request_user_create),
@@ -258,7 +271,9 @@ schema("/gateways/:name/authentication/users/:uid") ->
'operationId' => users_insta,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(get_user),
+ summary => <<"Get User Info for a Gateway Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_userid_in_path(),
responses =>
@@ -273,7 +288,9 @@ schema("/gateways/:name/authentication/users/:uid") ->
},
put =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(update_user),
+ summary => <<"Update User Info for a Gateway Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_userid_in_path(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -292,7 +309,9 @@ schema("/gateways/:name/authentication/users/:uid") ->
},
delete =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(delete_user),
+ summary => <<"Delete User for a Gateway Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_userid_in_path(),
responses =>
@@ -311,8 +330,8 @@ params_gateway_name_in_path() ->
binary(),
#{
in => path,
- desc => ?DESC(emqx_gateway_api, gateway_name),
- example => <<"">>
+ desc => ?DESC(emqx_gateway_api, gateway_name_in_qs),
+ example => <<"stomp">>
}
)}
].
@@ -325,7 +344,7 @@ params_userid_in_path() ->
#{
in => path,
desc => ?DESC(user_id),
- example => <<"">>
+ example => <<"test_username">>
}
)}
].
@@ -343,7 +362,7 @@ params_fuzzy_in_qs() ->
in => query,
required => false,
desc => ?DESC(like_user_id),
- example => <<"username">>
+ example => <<"test_">>
}
)},
{is_superuser,
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl
index 38036f7c7..0d90b4dc5 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl
@@ -122,7 +122,9 @@ schema("/gateways/:name/authentication/import_users") ->
'operationId' => import_users,
post =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(emqx_gateway_api_authn, import_users),
+ summary => <<"Import Users">>,
parameters => params_gateway_name_in_path(),
'requestBody' => emqx_dashboard_swagger:file_schema(filename),
responses =>
@@ -134,7 +136,9 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") ->
'operationId' => import_listener_users,
post =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(emqx_gateway_api_listeners, import_users),
+ summary => <<"Import Users">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
'requestBody' => emqx_dashboard_swagger:file_schema(filename),
@@ -144,6 +148,7 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") ->
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
+
%%--------------------------------------------------------------------
%% params defines
%%--------------------------------------------------------------------
@@ -155,7 +160,7 @@ params_gateway_name_in_path() ->
binary(),
#{
in => path,
- desc => ?DESC(emqx_gateway_api, gateway_name),
+ desc => ?DESC(emqx_gateway_api, gateway_name_in_qs),
example => <<"stomp">>
}
)}
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
index b7cf9fc64..ab82a4716 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl
@@ -467,7 +467,9 @@ schema("/gateways/:name/clients") ->
'operationId' => clients,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(list_clients),
+ summary => <<"List Gateway's Clients">>,
parameters => params_client_query(),
responses =>
?STANDARD_RESP(#{200 => schema_client_list()})
@@ -478,14 +480,18 @@ schema("/gateways/:name/clients/:clientid") ->
'operationId' => clients_insta,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(get_client),
+ summary => <<"Get Client Info">>,
parameters => params_client_insta(),
responses =>
?STANDARD_RESP(#{200 => schema_client()})
},
delete =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(kick_client),
+ summary => <<"Kick out Client">>,
parameters => params_client_insta(),
responses =>
?STANDARD_RESP(#{204 => <<"Kicked">>})
@@ -496,7 +502,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions") ->
'operationId' => subscriptions,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(list_subscriptions),
+ summary => <<"List Client's Subscription">>,
parameters => params_client_insta(),
responses =>
?STANDARD_RESP(
@@ -510,7 +518,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions") ->
},
post =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(add_subscription),
+ summary => <<"Add Subscription for Client">>,
parameters => params_client_insta(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
ref(subscription),
@@ -532,7 +542,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions/:topic") ->
'operationId' => subscriptions,
delete =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(delete_subscription),
+ summary => <<"Delete Client's Subscription">>,
parameters => params_topic_name_in_path() ++ params_client_insta(),
responses =>
?STANDARD_RESP(#{204 => <<"Unsubscribed">>})
diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
index 92903ec35..259ca8b36 100644
--- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
@@ -358,7 +358,9 @@ schema("/gateways/:name/listeners") ->
'operationId' => listeners,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(list_listeners),
+ summary => <<"List All Listeners">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(
@@ -372,7 +374,9 @@ schema("/gateways/:name/listeners") ->
},
post =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(add_listener),
+ summary => <<"Add a Listener">>,
parameters => params_gateway_name_in_path(),
%% XXX: How to distinguish the different listener supported by
%% different types of gateways?
@@ -396,7 +400,9 @@ schema("/gateways/:name/listeners/:id") ->
'operationId' => listeners_insta,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(get_listener),
+ summary => <<"Get the Listener Configs">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
responses =>
@@ -411,7 +417,9 @@ schema("/gateways/:name/listeners/:id") ->
},
delete =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(delete_listener),
+ summary => <<"Delete the Listener">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
responses =>
@@ -419,7 +427,9 @@ schema("/gateways/:name/listeners/:id") ->
},
put =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(update_listener),
+ summary => <<"Update the Listener Configs">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -442,7 +452,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
'operationId' => listeners_insta_authn,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(get_listener_authn),
+ summary => <<"Get the Listener's Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
responses =>
@@ -455,7 +467,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
},
post =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(add_listener_authn),
+ summary => <<"Create an Authenticator for a Listener">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
'requestBody' => schema_authn(),
@@ -464,7 +478,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
},
put =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(update_listener_authn),
+ summary => <<"Update the Listener Authenticator configs">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
'requestBody' => schema_authn(),
@@ -473,7 +489,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
},
delete =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(delete_listener_authn),
+ summary => <<"Delete the Listener's Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
responses =>
@@ -485,7 +503,9 @@ schema("/gateways/:name/listeners/:id/authentication/users") ->
'operationId' => users,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(list_users),
+ summary => <<"List Authenticator's Users">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path() ++
params_paging_in_qs(),
@@ -501,7 +521,9 @@ schema("/gateways/:name/listeners/:id/authentication/users") ->
},
post =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(add_user),
+ summary => <<"Add User for an Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -524,7 +546,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
'operationId' => users_insta,
get =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(get_user),
+ summary => <<"Get User Info">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path() ++
params_userid_in_path(),
@@ -540,7 +564,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
},
put =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(update_user),
+ summary => <<"Update User Info">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path() ++
params_userid_in_path(),
@@ -560,7 +586,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
},
delete =>
#{
+ tags => [<<"Gateways">>],
desc => ?DESC(delete_user),
+ summary => <<"Delete User">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path() ++
params_userid_in_path(),
@@ -570,6 +598,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
+
%%--------------------------------------------------------------------
%% params defines
@@ -580,8 +609,8 @@ params_gateway_name_in_path() ->
binary(),
#{
in => path,
- desc => ?DESC(emqx_gateway_api, gateway_name),
- example => <<"">>
+ desc => ?DESC(emqx_gateway_api, gateway_name_in_qs),
+ example => <<"stomp">>
}
)}
].
diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl
index 8850fa462..e5d6dd52a 100644
--- a/apps/emqx_gateway/src/emqx_gateway_schema.erl
+++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl
@@ -28,7 +28,7 @@
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
--type ip_port() :: tuple().
+-type ip_port() :: tuple() | integer().
-type duration() :: non_neg_integer().
-type duration_s() :: non_neg_integer().
-type bytesize() :: pos_integer().
diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl
index 861ae3189..be4cddcaa 100644
--- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl
+++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl
@@ -78,7 +78,8 @@
-define(TIMER_TABLE, #{
alive_timer => keepalive,
- force_timer => force_close
+ force_timer => force_close,
+ idle_timer => force_close_idle
}).
-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]).
@@ -151,14 +152,17 @@ init(
Ctx = maps:get(ctx, Options),
GRpcChann = maps:get(handler, Options),
PoolName = maps:get(pool_name, Options),
- NConnInfo = default_conninfo(ConnInfo),
+ IdleTimeout = emqx_gateway_utils:idle_timeout(Options),
+
+ NConnInfo = default_conninfo(ConnInfo#{idle_timeout => IdleTimeout}),
ListenerId =
case maps:get(listener, Options, undefined) of
undefined -> undefined;
{GwName, Type, LisName} -> emqx_gateway_utils:listener_id(GwName, Type, LisName)
end,
+
EnableAuthn = maps:get(enable_authn, Options, true),
- DefaultClientInfo = default_clientinfo(ConnInfo),
+ DefaultClientInfo = default_clientinfo(NConnInfo),
ClientInfo = DefaultClientInfo#{
listener => ListenerId,
enable_authn => EnableAuthn
@@ -183,7 +187,9 @@ init(
}
)
},
- try_dispatch(on_socket_created, wrap(Req), Channel).
+ start_idle_checking_timer(
+ try_dispatch(on_socket_created, wrap(Req), Channel)
+ ).
%% @private
peercert(NoSsl, ConnInfo) when
@@ -217,6 +223,12 @@ socktype(dtls) -> 'DTLS'.
address({Host, Port}) ->
#{host => inet:ntoa(Host), port => Port}.
+%% avoid udp connection process leak
+start_idle_checking_timer(Channel = #channel{conninfo = #{socktype := udp}}) ->
+ ensure_timer(idle_timer, Channel);
+start_idle_checking_timer(Channel) ->
+ Channel.
+
%%--------------------------------------------------------------------
%% Handle incoming packet
%%--------------------------------------------------------------------
@@ -285,10 +297,15 @@ handle_timeout(
{ok, reset_timer(alive_timer, NChannel)};
{error, timeout} ->
Req = #{type => 'KEEPALIVE'},
- {ok, try_dispatch(on_timer_timeout, wrap(Req), Channel)}
+ NChannel = remove_timer_ref(alive_timer, Channel),
+ %% close connection if keepalive timeout
+ Replies = [{event, disconnected}, {close, keepalive_timeout}],
+ {ok, Replies, try_dispatch(on_timer_timeout, wrap(Req), NChannel)}
end;
handle_timeout(_TRef, force_close, Channel = #channel{closed_reason = Reason}) ->
{shutdown, {error, {force_close, Reason}}, Channel};
+handle_timeout(_TRef, force_close_idle, Channel) ->
+ {shutdown, idle_timeout, Channel};
handle_timeout(_TRef, Msg, Channel) ->
?SLOG(warning, #{
msg => "unexpected_timeout_signal",
@@ -390,7 +407,7 @@ handle_call(
NConnInfo = ConnInfo#{keepalive => Interval},
NClientInfo = ClientInfo#{keepalive => Interval},
NChannel = Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo},
- {reply, ok, ensure_keepalive(NChannel)};
+ {reply, ok, [{event, updated}], ensure_keepalive(cancel_timer(idle_timer, NChannel))};
handle_call(
{subscribe_from_client, TopicFilter, Qos},
_From,
@@ -405,21 +422,21 @@ handle_call(
{reply, {error, ?RESP_PERMISSION_DENY, <<"Authorization deny">>}, Channel};
_ ->
{ok, _, NChannel} = do_subscribe([{TopicFilter, #{qos => Qos}}], Channel),
- {reply, ok, NChannel}
+ {reply, ok, [{event, updated}], NChannel}
end;
handle_call({subscribe, Topic, SubOpts}, _From, Channel) ->
{ok, [{NTopicFilter, NSubOpts}], NChannel} = do_subscribe([{Topic, SubOpts}], Channel),
- {reply, {ok, {NTopicFilter, NSubOpts}}, NChannel};
+ {reply, {ok, {NTopicFilter, NSubOpts}}, [{event, updated}], NChannel};
handle_call(
{unsubscribe_from_client, TopicFilter},
_From,
Channel = #channel{conn_state = connected}
) ->
{ok, NChannel} = do_unsubscribe([{TopicFilter, #{}}], Channel),
- {reply, ok, NChannel};
+ {reply, ok, [{event, updated}], NChannel};
handle_call({unsubscribe, Topic}, _From, Channel) ->
{ok, NChannel} = do_unsubscribe([Topic], Channel),
- {reply, ok, NChannel};
+ {reply, ok, [{event, update}], NChannel};
handle_call(subscriptions, _From, Channel = #channel{subscriptions = Subs}) ->
{reply, {ok, maps:to_list(Subs)}, Channel};
handle_call(
@@ -446,7 +463,7 @@ handle_call(
{reply, ok, Channel}
end;
handle_call(kick, _From, Channel) ->
- {shutdown, kicked, ok, ensure_disconnected(kicked, Channel)};
+ {reply, ok, [{event, disconnected}, {close, kicked}], Channel};
handle_call(discard, _From, Channel) ->
{shutdown, discarded, ok, Channel};
handle_call(Req, _From, Channel) ->
@@ -648,7 +665,8 @@ ensure_keepalive(Channel = #channel{clientinfo = ClientInfo}) ->
ensure_keepalive_timer(Interval, Channel) when Interval =< 0 ->
Channel;
ensure_keepalive_timer(Interval, Channel) ->
- Keepalive = emqx_keepalive:init(timer:seconds(Interval)),
+ StatVal = emqx_gateway_conn:keepalive_stats(recv),
+ Keepalive = emqx_keepalive:init(StatVal, timer:seconds(Interval)),
ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}).
ensure_timer(Name, Channel = #channel{timers = Timers}) ->
@@ -666,11 +684,17 @@ ensure_timer(Name, Time, Channel = #channel{timers = Timers}) ->
Channel#channel{timers = Timers#{Name => TRef}}.
reset_timer(Name, Channel) ->
- ensure_timer(Name, clean_timer(Name, Channel)).
+ ensure_timer(Name, remove_timer_ref(Name, Channel)).
-clean_timer(Name, Channel = #channel{timers = Timers}) ->
+cancel_timer(Name, Channel = #channel{timers = Timers}) ->
+ emqx_misc:cancel_timer(maps:get(Name, Timers, undefined)),
+ remove_timer_ref(Name, Channel).
+
+remove_timer_ref(Name, Channel = #channel{timers = Timers}) ->
Channel#channel{timers = maps:remove(Name, Timers)}.
+interval(idle_timer, #channel{conninfo = #{idle_timeout := IdleTimeout}}) ->
+ IdleTimeout;
interval(force_timer, _) ->
15000;
interval(alive_timer, #channel{keepalive = Keepalive}) ->
@@ -725,7 +749,7 @@ enrich_clientinfo(InClientInfo = #{proto_name := ProtoName}, ClientInfo) ->
default_conninfo(ConnInfo) ->
ConnInfo#{
clean_start => true,
- clientid => undefined,
+ clientid => anonymous_clientid(),
username => undefined,
conn_props => #{},
connected => true,
@@ -739,14 +763,15 @@ default_conninfo(ConnInfo) ->
default_clientinfo(#{
peername := {PeerHost, _},
- sockname := {_, SockPort}
+ sockname := {_, SockPort},
+ clientid := ClientId
}) ->
#{
zone => default,
protocol => exproto,
peerhost => PeerHost,
sockport => SockPort,
- clientid => undefined,
+ clientid => ClientId,
username => undefined,
is_bridge => false,
is_superuser => false,
@@ -764,3 +789,6 @@ proto_name_to_protocol(<<>>) ->
exproto;
proto_name_to_protocol(ProtoName) when is_binary(ProtoName) ->
binary_to_atom(ProtoName).
+
+anonymous_clientid() ->
+ iolist_to_binary(["exproto-", emqx_misc:gen_id()]).
diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl
index d1bf4ba94..cf8ed76a7 100644
--- a/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl
+++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl
@@ -56,12 +56,19 @@ start_link(Pool, Id) ->
[]
).
+-spec async_call(atom(), map(), map()) -> ok.
async_call(
FunName,
Req = #{conn := Conn},
Options = #{pool_name := PoolName}
) ->
- cast(pick(PoolName, Conn), {rpc, FunName, Req, Options, self()}).
+ case pick(PoolName, Conn) of
+ false ->
+ reply(self(), FunName, {error, no_available_grpc_client});
+ Pid when is_pid(Pid) ->
+ cast(Pid, {rpc, FunName, Req, Options, self()})
+ end,
+ ok.
%%--------------------------------------------------------------------
%% cast, pick
@@ -72,6 +79,7 @@ async_call(
cast(Deliver, Msg) ->
gen_server:cast(Deliver, Msg).
+-spec pick(term(), term()) -> pid() | false.
pick(PoolName, Conn) ->
gproc_pool:pick_worker(PoolName, Conn).
diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl
index 1aa0bac93..b563d9ea6 100644
--- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl
+++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl
@@ -45,8 +45,9 @@ schema(?PATH("/lookup")) ->
#{
'operationId' => lookup,
get => #{
- tags => [<<"LwM2M">>],
+ tags => [<<"Gateways">>],
desc => ?DESC(lookup_resource),
+ summary => <<"List Client's Resources">>,
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
{path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})},
@@ -69,8 +70,9 @@ schema(?PATH("/observe")) ->
#{
'operationId' => observe,
post => #{
- tags => [<<"LwM2M">>],
+ tags => [<<"Gateways">>],
desc => ?DESC(observe_resource),
+ summary => <<"Observe a Resource">>,
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
{path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})},
@@ -87,8 +89,9 @@ schema(?PATH("/read")) ->
#{
'operationId' => read,
post => #{
- tags => [<<"LwM2M">>],
+ tags => [<<"Gateways">>],
desc => ?DESC(read_resource),
+ summary => <<"Read Value from a Resource Path">>,
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
{path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})}
@@ -104,7 +107,8 @@ schema(?PATH("/write")) ->
'operationId' => write,
post => #{
desc => ?DESC(write_resource),
- tags => [<<"LwM2M">>],
+ tags => [<<"Gateways">>],
+ summary => <<"Write a Value to Resource Path">>,
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
{path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})},
diff --git a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl b/apps/emqx_gateway/test/emqx_exproto_SUITE.erl
index 66f780d3f..0e863a14c 100644
--- a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl
+++ b/apps/emqx_gateway/test/emqx_exproto_SUITE.erl
@@ -20,6 +20,7 @@
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx_hooks.hrl").
+-include_lib("eunit/include/eunit.hrl").
-import(
emqx_exproto_echo_svr,
@@ -38,6 +39,7 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(TCPOPTS, [binary, {active, false}]).
-define(DTLSOPTS, [binary, {active, false}, {protocol, dtls}]).
@@ -62,6 +64,9 @@
all() ->
[{group, Name} || Name <- metrics()].
+suite() ->
+ [{timetrap, {seconds, 30}}].
+
groups() ->
Cases = emqx_common_test_helpers:all(?MODULE),
[{Name, Cases} || Name <- metrics()].
@@ -87,6 +92,7 @@ set_special_cfg(emqx_gateway) ->
[gateway, exproto],
#{
server => #{bind => 9100},
+ idle_timeout => 5000,
handler => #{address => "http://127.0.0.1:9001"},
listeners => listener_confs(LisType)
}
@@ -223,14 +229,16 @@ t_acl_deny(Cfg) ->
close(Sock).
t_keepalive_timeout(Cfg) ->
+ ok = snabbkaffe:start_trace(),
SockType = proplists:get_value(listener_type, Cfg),
Sock = open(SockType),
+ ClientId1 = <<"keepalive_test_client1">>,
Client = #{
proto_name => <<"demo">>,
proto_ver => <<"v0.1">>,
- clientid => <<"test_client_1">>,
- keepalive => 2
+ clientid => ClientId1,
+ keepalive => 5
},
Password = <<"123456">>,
@@ -238,16 +246,42 @@ t_keepalive_timeout(Cfg) ->
ConnAckBin = frame_connack(0),
send(Sock, ConnBin),
- {ok, ConnAckBin} = recv(Sock, 5000),
+ {ok, ConnAckBin} = recv(Sock),
- DisconnectBin = frame_disconnect(),
- {ok, DisconnectBin} = recv(Sock, 10000),
-
- SockType =/= udp andalso
- begin
- {error, closed} = recv(Sock, 5000)
- end,
- ok.
+ case SockType of
+ udp ->
+ %% another udp client should not affect the first
+ %% udp client keepalive check
+ timer:sleep(4000),
+ Sock2 = open(SockType),
+ ConnBin2 = frame_connect(
+ Client#{clientid => <<"keepalive_test_client2">>},
+ Password
+ ),
+ send(Sock2, ConnBin2),
+ %% first client will be keepalive timeouted in 6s
+ ?assertMatch(
+ {ok, #{
+ clientid := ClientId1,
+ reason := {shutdown, {sock_closed, keepalive_timeout}}
+ }},
+ ?block_until(#{?snk_kind := conn_process_terminated}, 8000)
+ );
+ _ ->
+ ?assertMatch(
+ {ok, #{
+ clientid := ClientId1,
+ reason := {shutdown, {sock_closed, keepalive_timeout}}
+ }},
+ ?block_until(#{?snk_kind := conn_process_terminated}, 12000)
+ ),
+ Trace = snabbkaffe:collect_trace(),
+ %% conn process should be terminated
+ ?assertEqual(1, length(?of_kind(conn_process_terminated, Trace))),
+ %% socket port should be closed
+ ?assertEqual({error, closed}, recv(Sock, 5000))
+ end,
+ snabbkaffe:stop().
t_hook_connected_disconnected(Cfg) ->
SockType = proplists:get_value(listener_type, Cfg),
@@ -337,6 +371,8 @@ t_hook_session_subscribed_unsubscribed(Cfg) ->
error(hook_is_not_running)
end,
+ send(Sock, frame_disconnect()),
+
close(Sock),
emqx_hooks:del('session.subscribed', {?MODULE, hook_fun3}),
emqx_hooks:del('session.unsubscribed', {?MODULE, hook_fun4}).
@@ -373,6 +409,48 @@ t_hook_message_delivered(Cfg) ->
close(Sock),
emqx_hooks:del('message.delivered', {?MODULE, hook_fun5}).
+t_idle_timeout(Cfg) ->
+ ok = snabbkaffe:start_trace(),
+ SockType = proplists:get_value(listener_type, Cfg),
+ Sock = open(SockType),
+
+ %% need to create udp client by sending something
+ case SockType of
+ udp ->
+ %% nothing to do
+ ok = meck:new(emqx_exproto_gcli, [passthrough, no_history]),
+ ok = meck:expect(
+ emqx_exproto_gcli,
+ async_call,
+ fun(FunName, _Req, _GClient) ->
+ self() ! {hreply, FunName, ok},
+ ok
+ end
+ ),
+ %% send request, but nobody can respond to it
+ ClientId = <<"idle_test_client1">>,
+ Client = #{
+ proto_name => <<"demo">>,
+ proto_ver => <<"v0.1">>,
+ clientid => ClientId,
+ keepalive => 5
+ },
+ Password = <<"123456">>,
+ ConnBin = frame_connect(Client, Password),
+ send(Sock, ConnBin),
+ ?assertMatch(
+ {ok, #{reason := {shutdown, idle_timeout}}},
+ ?block_until(#{?snk_kind := conn_process_terminated}, 10000)
+ ),
+ ok = meck:unload(emqx_exproto_gcli);
+ _ ->
+ ?assertMatch(
+ {ok, #{reason := {shutdown, idle_timeout}}},
+ ?block_until(#{?snk_kind := conn_process_terminated}, 10000)
+ )
+ end,
+ snabbkaffe:stop().
+
%%--------------------------------------------------------------------
%% Utils
@@ -422,6 +500,9 @@ send({ssl, Sock}, Bin) ->
send({dtls, Sock}, Bin) ->
ssl:send(Sock, Bin).
+recv(Sock) ->
+ recv(Sock, infinity).
+
recv({tcp, Sock}, Ts) ->
gen_tcp:recv(Sock, 0, Ts);
recv({udp, Sock}, Ts) ->
diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf
index c6f518c86..0ab09520e 100644
--- a/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf
+++ b/apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf
@@ -2,78 +2,82 @@ emqx_mgmt_api_alarms {
list_alarms_api {
desc {
- en: """List alarms"""
- zh: """列出告警，获取告警列表"""
+ en: """List currently activated alarms or historical alarms, determined by query parameters."""
+ zh: """列出当前激活的告警或历史告警，由查询参数决定。"""
}
}
delete_alarms_api {
desc {
- en: """Remove all deactivated alarms"""
- zh: """删除所有历史告警（非活跃告警）"""
+ en: """Remove all historical alarms."""
+ zh: """删除所有历史告警。"""
}
}
delete_alarms_api_response204 {
desc {
- en: """Remove all deactivated alarms ok"""
- zh: """删除所有历史告警（非活跃告警）成功"""
+ en: """Historical alarms have been cleared successfully."""
+ zh: """历史告警已成功清除。"""
}
}
get_alarms_qs_activated {
desc {
- en: """Activate alarms, or deactivate alarms. Default is false"""
- zh: """活跃中的告警，或历史告警（非活跃告警），默认为 false"""
+ en: """It is used to specify the alarm type of the query.
+When true, it returns the currently activated alarm,
+and when it is false, it returns the historical alarm.
+The default is false."""
+ zh: """用于指定查询的告警类型，
+为 true 时返回当前激活的告警，为 false 时返回历史告警，默认为 false。"""
}
}
node {
desc {
- en: """Alarm in node"""
- zh: """告警节点名称"""
+ en: """The name of the node that triggered this alarm."""
+ zh: """触发此告警的节点名称。"""
}
}
name {
desc {
- en: """Alarm name"""
- zh: """告警名称"""
+ en: """Alarm name, used to distinguish different alarms."""
+ zh: """告警名称，用于区分不同的告警。"""
}
}
message {
desc {
- en: """Alarm readable information"""
- zh: """告警信息"""
+ en: """Alarm message, which describes the alarm content in a human-readable format."""
+ zh: """告警消息，以人类可读的方式描述告警内容。"""
}
}
details {
desc {
- en: """Alarm details information"""
- zh: """告警详细信息"""
+ en: """Alarm details, provides more alarm information, mainly for program processing."""
+ zh: """告警详情，提供了更多的告警信息，主要提供给程序处理。"""
}
}
duration {
desc {
- en: """Alarms duration time; UNIX time stamp, millisecond"""
- zh: """告警持续时间，单位：毫秒"""
+ en: """Indicates how long the alarm has lasted, in milliseconds."""
+ zh: """表明告警已经持续了多久，单位：毫秒。"""
}
}
activate_at {
desc {
- en: """Alarms activate time, RFC 3339"""
- zh: """告警开始时间，使用 rfc3339 标准时间格式"""
+ en: """Alarm start time, using rfc3339 standard time format."""
+ zh: """告警开始时间，使用 rfc3339 标准时间格式。"""
}
}
deactivate_at {
desc {
- en: """Alarms deactivate time, RFC 3339"""
- zh: """告警结束时间，使用 rfc3339 标准时间格式"""
+ en: """Alarm end time, using rfc3339 standard time format."""
+ zh: """告警结束时间，使用 rfc3339 标准时间格式。"""
}
}
diff --git a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf
index 686293d0b..3045cb293 100644
--- a/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf
+++ b/apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf
@@ -2,112 +2,97 @@ emqx_mgmt_api_banned {
list_banned_api {
desc {
- en: """List banned."""
- zh: """列出黑名单"""
- }
- label {
- en: """List Banned"""
- zh: """列出黑名单"""
+ en: """List all currently banned client IDs, usernames and IP addresses."""
+ zh: """列出目前所有被封禁的客户端 ID、用户名和 IP 地址。"""
}
}
create_banned_api {
desc {
- en: """Create banned."""
- zh: """创建黑名单"""
+ en: """Add a client ID, username or IP address to the blacklist."""
+ zh: """添加一个客户端 ID、用户名或者 IP 地址到黑名单。"""
}
}
create_banned_api_response400 {
desc {
- en: """Banned already existed, or bad args."""
- zh: """黑名单已存在，或参数格式有错误"""
+ en: """Bad request, possibly due to wrong parameters or the existence of a banned object."""
+ zh: """错误的请求，可能是参数错误或封禁对象已存在等原因。"""
}
}
delete_banned_api {
desc {
- en: """Delete banned"""
- zh: """删除黑名单"""
+ en: """Remove a client ID, username or IP address from the blacklist."""
+ zh: """将一个客户端 ID、用户名或者 IP 地址从黑名单中删除。"""
}
}
delete_banned_api_response404 {
desc {
- en: """Banned not found. May be the banned time has been exceeded"""
- zh: """黑名单未找到，可能为已经超期失效"""
- }
- }
-
- create_banned {
- desc {
- en: """List banned."""
- zh: """列出黑名单"""
- }
- label {
- en: """List Banned"""
- zh: """列出黑名单"""
+ en: """The banned object was not found in the blacklist."""
+ zh: """未在黑名单中找到该封禁对象。"""
}
}
as {
desc {
- en: """Banned type clientid, username, peerhost"""
- zh: """黑名单类型，可选 clientid、username、peerhost"""
+ en: """Ban method, which can be client ID, username or IP address."""
+ zh: """封禁方式，可以通过客户端 ID、用户名或者 IP 地址等方式进行封禁。"""
}
label {
- en: """Banned Type"""
- zh: """黑名单类型"""
+ en: """Ban Method"""
+ zh: """封禁方式"""
}
}
who {
desc {
- en: """Client info as banned type"""
- zh: """设备信息"""
+ en: """Ban object, specific client ID, username or IP address."""
+ zh: """封禁对象，具体的客户端 ID、用户名或者 IP 地址。"""
}
label {
- en: """Banned Info"""
- zh: """黑名单信息"""
+ en: """Ban Object"""
+ zh: """封禁对象"""
}
}
by {
desc {
- en: """Commander"""
- zh: """黑名单创建者"""
+ en: """Initiator of the ban."""
+ zh: """封禁的发起者。"""
}
label {
- en: """Commander"""
- zh: """黑名单创建者"""
+ en: """Ban Initiator"""
+ zh: """封禁发起者"""
}
}
reason {
desc {
- en: """Banned reason"""
- zh: """黑名单创建原因"""
+ en: """Ban reason, record the reason why the current object was banned."""
+ zh: """封禁原因，记录当前对象被封禁的原因。"""
}
label {
- en: """Reason"""
- zh: """原因"""
+ en: """Ban Reason"""
+ zh: """封禁原因"""
}
}
at {
desc {
- en: """Create banned time, rfc3339, now if not specified"""
- zh: """黑名单创建时间，默认为当前"""
+ en: """The start time of the ban, the format is rfc3339, the default is the time when the operation was initiated."""
+ zh: """封禁的起始时间，格式为 rfc3339，默认为发起操作的时间。"""
}
label {
- en: """Create banned time"""
- zh: """黑名单创建时间"""
+ en: """Ban Time"""
+ zh: """封禁时间"""
}
}
until {
desc {
- en: """Cancel banned time, rfc3339, now + 5 minute if not specified"""
- zh: """黑名单结束时间，默认为创建时间 + 5 分钟"""
+ en: """The end time of the ban, the format is rfc3339, the default is the time when the operation was initiated + 5 minutes."""
+ zh: """封禁的结束时间，式为 rfc3339，默认为发起操作的时间 + 5 分钟。"""
}
label {
- en: """Cancel banned time"""
- zh: """黑名单结束时间"""
+ en: """Ban End Time"""
+ zh: """封禁结束时间"""
}
}
}
diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src
index d9f09f00e..5f8b30bf4 100644
--- a/apps/emqx_management/src/emqx_management.app.src
+++ b/apps/emqx_management/src/emqx_management.app.src
@@ -2,7 +2,7 @@
{application, emqx_management, [
{description, "EMQX Management API and CLI"},
% strict semver, bump manually!
- {vsn, "5.0.5"},
+ {vsn, "5.0.6"},
{modules, []},
{registered, [emqx_management_sup]},
{applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},
diff --git a/apps/emqx_management/src/emqx_mgmt_api_app.erl b/apps/emqx_management/src/emqx_mgmt_api_app.erl
index 89311a8d1..7050ea1af 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_app.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_app.erl
@@ -22,7 +22,7 @@
-export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]).
-export([api_key/2, api_key_by_name/2]).
-export([validate_name/1]).
--define(TAGS, [<<"API keys">>]).
+-define(TAGS, [<<"API Keys">>]).
namespace() -> "api_key".
diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl
index 7435e5e0d..8eb801952 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl
@@ -141,7 +141,8 @@ schema("/configs_reset/:rootname") ->
],
responses => #{
200 => <<"Rest config successfully">>,
- 400 => emqx_dashboard_swagger:error_codes(['NO_DEFAULT_VALUE', 'REST_FAILED'])
+ 400 => emqx_dashboard_swagger:error_codes(['NO_DEFAULT_VALUE', 'REST_FAILED']),
+ 403 => emqx_dashboard_swagger:error_codes(['REST_FAILED'])
}
}
};
@@ -160,7 +161,8 @@ schema("/configs/global_zone") ->
'requestBody' => Schema,
responses => #{
200 => Schema,
- 400 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED'])
+ 400 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED']),
+ 403 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED'])
}
}
};
@@ -226,7 +228,8 @@ schema(Path) ->
'requestBody' => Schema,
responses => #{
200 => Schema,
- 400 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED'])
+ 400 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED']),
+ 403 => emqx_dashboard_swagger:error_codes(['UPDATE_FAILED'])
}
}
}.
@@ -254,6 +257,8 @@ config(put, #{body := Body}, Req) ->
case emqx_conf:update(Path, Body, ?OPTS) of
{ok, #{raw_config := RawConf}} ->
{200, RawConf};
+ {error, {permission_denied, Reason}} ->
+ {403, #{code => 'UPDATE_FAILED', message => Reason}};
{error, Reason} ->
{400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Reason)}}
end.
@@ -297,6 +302,8 @@ config_reset(post, _Params, Req) ->
case emqx_conf:reset(Path, ?OPTS) of
{ok, _} ->
{200};
+ {error, {permission_denied, Reason}} ->
+ {403, #{code => 'REST_FAILED', message => Reason}};
{error, no_default_value} ->
{400, #{code => 'NO_DEFAULT_VALUE', message => <<"No Default Value.">>}};
{error, Reason} ->
diff --git a/apps/emqx_management/src/emqx_mgmt_api_status.erl b/apps/emqx_management/src/emqx_mgmt_api_status.erl
index e38ccfd69..68bad03e4 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_status.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_status.erl
@@ -20,6 +20,13 @@
path/0
]).
+%% Note: Because swagger now requires an HTTP prefix (e.g. /api/v5),
+%% but the `/status` does not require this fixed prefix.
+%%
+%% Changing the swagger framework was too big, so we implemented the `/status`
+%% in a simple way first
+%%
+%% XXX: So the HTTP API docs generated by swagger can not find this API now
path() ->
"/status".
diff --git a/apps/emqx_management/src/emqx_mgmt_api_sys.erl b/apps/emqx_management/src/emqx_mgmt_api_sys.erl
index 43fd9ee14..c7aeb1d95 100644
--- a/apps/emqx_management/src/emqx_mgmt_api_sys.erl
+++ b/apps/emqx_management/src/emqx_mgmt_api_sys.erl
@@ -31,7 +31,7 @@
-export([sys/2]).
--define(TAGS, [<<"System topics">>]).
+-define(TAGS, [<<"System Topics">>]).
namespace() -> "sys".
diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl
index 10d04db85..a623b6fbf 100644
--- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl
+++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl
@@ -100,6 +100,68 @@ t_wss_crud_listeners_by_id(_) ->
Type = <<"wss">>,
crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type).
+t_api_listeners_list_not_ready(_Config) ->
+ net_kernel:start(['listeners@127.0.0.1', longnames]),
+ ct:timetrap({seconds, 120}),
+ snabbkaffe:fix_ct_logging(),
+ Cluster = [{Name, Opts}, {Name1, Opts1}] = cluster([core, core]),
+ ct:pal("Starting ~p", [Cluster]),
+ Node1 = emqx_common_test_helpers:start_slave(Name, Opts),
+ Node2 = emqx_common_test_helpers:start_slave(Name1, Opts1),
+ try
+ L1 = get_tcp_listeners(Node1),
+
+ %% test init_config not ready.
+ _ = rpc:call(Node1, application, set_env, [emqx, init_config_load_done, false]),
+ assert_config_load_not_done(Node1),
+
+ L2 = get_tcp_listeners(Node1),
+ L3 = get_tcp_listeners(Node2),
+
+ Comment = #{
+ node1 => rpc:call(Node1, mria_mnesia, running_nodes, []),
+ node2 => rpc:call(Node2, mria_mnesia, running_nodes, [])
+ },
+
+ ?assert(length(L1) > length(L2), Comment),
+ ?assertEqual(length(L2), length(L3), Comment)
+ after
+ emqx_common_test_helpers:stop_slave(Node1),
+ emqx_common_test_helpers:stop_slave(Node2)
+ end.
+
+get_tcp_listeners(Node) ->
+ Query = #{query_string => #{<<"type">> => tcp}},
+ {200, L} = rpc:call(Node, emqx_mgmt_api_listeners, list_listeners, [get, Query]),
+ [#{node_status := NodeStatus}] = L,
+ ct:pal("Node:~p:~p", [Node, L]),
+ NodeStatus.
+
+assert_config_load_not_done(Node) ->
+ Done = rpc:call(Node, emqx_app, get_init_config_load_done, []),
+ ?assertNot(Done, #{node => Node}).
+
+cluster(Specs) ->
+ Env = [
+ {emqx, init_config_load_done, false},
+ {emqx, boot_modules, []}
+ ],
+ emqx_common_test_helpers:emqx_cluster(Specs, [
+ {env, Env},
+ {apps, [emqx_conf]},
+ {load_schema, false},
+ {join_to, true},
+ {env_handler, fun
+ (emqx) ->
+ application:set_env(emqx, boot_modules, []),
+ %% test init_config not ready.
+ application:set_env(emqx, init_config_load_done, false),
+ ok;
+ (_) ->
+ ok
+ end}
+ ]).
+
crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type) ->
OriginPath = emqx_mgmt_api_test_util:api_path(["listeners", ListenerId]),
NewPath = emqx_mgmt_api_test_util:api_path(["listeners", NewListenerId]),
diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl
index ac7f75158..76646bc64 100644
--- a/apps/emqx_modules/src/emqx_delayed.erl
+++ b/apps/emqx_modules/src/emqx_delayed.erl
@@ -91,6 +91,7 @@
-define(SERVER, ?MODULE).
-define(MAX_INTERVAL, 4294967).
-define(FORMAT_FUN, {?MODULE, format_delayed}).
+-define(NOW, erlang:system_time(milli_seconds)).
%%--------------------------------------------------------------------
%% Mnesia bootstrap
@@ -118,12 +119,13 @@ on_message_publish(
{PubAt, Delayed} =
case binary_to_integer(Delay) of
Interval when Interval < ?MAX_INTERVAL ->
- {Interval + erlang:round(Ts / 1000), Interval};
+ {Interval * 1000 + Ts, Interval};
Timestamp ->
%% Check malicious timestamp?
- case (Timestamp - erlang:round(Ts / 1000)) > ?MAX_INTERVAL of
+ Internal = Timestamp - erlang:round(Ts / 1000),
+ case Internal > ?MAX_INTERVAL of
true -> error(invalid_delayed_timestamp);
- false -> {Timestamp, Timestamp - erlang:round(Ts / 1000)}
+ false -> {Timestamp * 1000, Internal}
end
end,
PubMsg = Msg#message{topic = Topic1},
@@ -189,14 +191,14 @@ format_delayed(
WithPayload
) ->
PublishTime = to_rfc3339(PublishTimeStamp div 1000),
- ExpectTime = to_rfc3339(ExpectTimeStamp),
- RemainingTime = ExpectTimeStamp - erlang:system_time(second),
+ ExpectTime = to_rfc3339(ExpectTimeStamp div 1000),
+ RemainingTime = ExpectTimeStamp - ?NOW,
Result = #{
msgid => emqx_guid:to_hexstr(Id),
node => node(),
publish_at => PublishTime,
delayed_interval => Delayed,
- delayed_remaining => RemainingTime,
+ delayed_remaining => RemainingTime div 1000,
expected_at => ExpectTime,
topic => Topic,
qos => Qos,
@@ -296,7 +298,7 @@ handle_cast(Msg, State) ->
%% Do Publish...
handle_info({timeout, TRef, do_publish}, State = #{publish_timer := TRef}) ->
- DeletedKeys = do_publish(mnesia:dirty_first(?TAB), erlang:system_time(seconds)),
+ DeletedKeys = do_publish(mnesia:dirty_first(?TAB), ?NOW),
lists:foreach(fun(Key) -> mria:dirty_delete(?TAB, Key) end, DeletedKeys),
{noreply, ensure_publish_timer(State#{publish_timer := undefined, publish_at := 0})};
handle_info(stats, State = #{stats_fun := StatsFun}) ->
@@ -347,18 +349,18 @@ ensure_publish_timer(State) ->
ensure_publish_timer('$end_of_table', State) ->
State#{publish_timer := undefined, publish_at := 0};
ensure_publish_timer({Ts, _Id}, State = #{publish_timer := undefined}) ->
- ensure_publish_timer(Ts, erlang:system_time(seconds), State);
+ ensure_publish_timer(Ts, ?NOW, State);
ensure_publish_timer({Ts, _Id}, State = #{publish_timer := TRef, publish_at := PubAt}) when
Ts < PubAt
->
ok = emqx_misc:cancel_timer(TRef),
- ensure_publish_timer(Ts, erlang:system_time(seconds), State);
+ ensure_publish_timer(Ts, ?NOW, State);
ensure_publish_timer(_Key, State) ->
State.
ensure_publish_timer(Ts, Now, State) ->
Interval = max(1, Ts - Now),
- TRef = emqx_misc:start_timer(timer:seconds(Interval), do_publish),
+ TRef = emqx_misc:start_timer(Interval, do_publish),
State#{publish_timer := TRef, publish_at := Now + Interval}.
do_publish(Key, Now) ->
diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src
index 2fa38dae3..4a1644152 100644
--- a/apps/emqx_modules/src/emqx_modules.app.src
+++ b/apps/emqx_modules/src/emqx_modules.app.src
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_modules, [
{description, "EMQX Modules"},
- {vsn, "5.0.4"},
+ {vsn, "5.0.5"},
{modules, []},
{applications, [kernel, stdlib, emqx]},
{mod, {emqx_modules_app, []}},
diff --git a/apps/emqx_modules/test/emqx_delayed_SUITE.erl b/apps/emqx_modules/test/emqx_delayed_SUITE.erl
index d1af9a064..e5e3db98c 100644
--- a/apps/emqx_modules/test/emqx_delayed_SUITE.erl
+++ b/apps/emqx_modules/test/emqx_delayed_SUITE.erl
@@ -202,3 +202,37 @@ t_get_basic_usage_info(_Config) ->
),
?assertEqual(#{delayed_message_count => 4}, emqx_delayed:get_basic_usage_info()),
ok.
+
+t_delayed_precision(_) ->
+ MaxSpan = 1250,
+ FutureDiff = subscribe_proc(),
+ DelayedMsg0 = emqx_message:make(
+ ?MODULE, 1, <<"$delayed/1/delayed/test">>, <<"delayed/test">>
+ ),
+ _ = on_message_publish(DelayedMsg0),
+ ?assert(FutureDiff() =< MaxSpan).
+
+subscribe_proc() ->
+ Self = self(),
+ Ref = erlang:make_ref(),
+ erlang:spawn(fun() ->
+ Topic = <<"delayed/+">>,
+ emqx_broker:subscribe(Topic),
+ Self !
+ {Ref,
+ receive
+ {deliver, Topic, Msg} ->
+ erlang:system_time(milli_seconds) - Msg#message.timestamp
+ after 2000 ->
+ 2000
+ end},
+ emqx_broker:unsubscribe(Topic)
+ end),
+ fun() ->
+ receive
+ {Ref, Diff} ->
+ Diff
+ after 2000 ->
+ 2000
+ end
+ end.
diff --git a/apps/emqx_plugin_libs/src/emqx_placeholder.erl b/apps/emqx_plugin_libs/src/emqx_placeholder.erl
index 70a1e41a5..dd99724e2 100644
--- a/apps/emqx_plugin_libs/src/emqx_placeholder.erl
+++ b/apps/emqx_plugin_libs/src/emqx_placeholder.erl
@@ -39,7 +39,10 @@
sql_data/1
]).
--define(EX_PLACE_HOLDER, "(\\$\\{[a-zA-Z0-9\\._]+\\}|\"\\$\\{[a-zA-Z0-9\\._]+\\}\")").
+-define(EX_PLACE_HOLDER, "(\\$\\{[a-zA-Z0-9\\._]+\\})").
+
+-define(EX_PLACE_HOLDER_DOUBLE_QUOTE, "(\\$\\{[a-zA-Z0-9\\._]+\\}|\"\\$\\{[a-zA-Z0-9\\._]+\\}\")").
+
%% Space and CRLF
-define(EX_WITHE_CHARS, "\\s").
@@ -57,7 +60,8 @@
-type preproc_sql_opts() :: #{
placeholders => list(binary()),
- replace_with => '?' | '$n'
+ replace_with => '?' | '$n',
+ strip_double_quote => boolean()
}.
-type preproc_deep_opts() :: #{
@@ -89,7 +93,7 @@ preproc_tmpl(Str) ->
preproc_tmpl(Str, Opts) ->
RE = preproc_var_re(Opts),
Tokens = re:split(Str, RE, [{return, binary}, group, trim]),
- do_preproc_tmpl(Tokens, []).
+ do_preproc_tmpl(Opts, Tokens, []).
-spec proc_tmpl(tmpl_token(), map()) -> binary().
proc_tmpl(Tokens, Data) ->
@@ -140,10 +144,11 @@ preproc_sql(Sql, ReplaceWith) when is_atom(ReplaceWith) ->
preproc_sql(Sql, #{replace_with => ReplaceWith});
preproc_sql(Sql, Opts) ->
RE = preproc_var_re(Opts),
+ Strip = maps:get(strip_double_quote, Opts, false),
ReplaceWith = maps:get(replace_with, Opts, '?'),
case re:run(Sql, RE, [{capture, all_but_first, binary}, global]) of
{match, PlaceHolders} ->
- PhKs = [parse_nested(unwrap(Phld)) || [Phld | _] <- PlaceHolders],
+ PhKs = [parse_nested(unwrap(Phld, Strip)) || [Phld | _] <- PlaceHolders],
{replace_with(Sql, RE, ReplaceWith), [{var, Phld} || Phld <- PhKs]};
nomatch ->
{Sql, []}
@@ -234,29 +239,36 @@ get_phld_var(Fun, Data) when is_function(Fun) ->
get_phld_var(Phld, Data) ->
emqx_rule_maps:nested_get(Phld, Data).
-preproc_var_re(#{placeholders := PHs}) ->
+preproc_var_re(#{placeholders := PHs, strip_double_quote := true}) ->
Res = [ph_to_re(PH) || PH <- PHs],
QuoteRes = ["\"" ++ Re ++ "\"" || Re <- Res],
"(" ++ string:join(Res ++ QuoteRes, "|") ++ ")";
+preproc_var_re(#{placeholders := PHs}) ->
+ "(" ++ string:join([ph_to_re(PH) || PH <- PHs], "|") ++ ")";
+preproc_var_re(#{strip_double_quote := true}) ->
+ ?EX_PLACE_HOLDER_DOUBLE_QUOTE;
preproc_var_re(#{}) ->
?EX_PLACE_HOLDER.
ph_to_re(VarPH) ->
re:replace(VarPH, "[\\$\\{\\}]", "\\\\&", [global, {return, list}]).
-do_preproc_tmpl([], Acc) ->
+do_preproc_tmpl(_Opts, [], Acc) ->
lists:reverse(Acc);
-do_preproc_tmpl([[Str, Phld] | Tokens], Acc) ->
+do_preproc_tmpl(Opts, [[Str, Phld] | Tokens], Acc) ->
+ Strip = maps:get(strip_double_quote, Opts, false),
do_preproc_tmpl(
+ Opts,
Tokens,
put_head(
var,
- parse_nested(unwrap(Phld)),
+ parse_nested(unwrap(Phld, Strip)),
put_head(str, Str, Acc)
)
);
-do_preproc_tmpl([[Str] | Tokens], Acc) ->
+do_preproc_tmpl(Opts, [[Str] | Tokens], Acc) ->
do_preproc_tmpl(
+ Opts,
Tokens,
put_head(str, Str, Acc)
).
@@ -293,10 +305,10 @@ parse_nested(Attr) ->
Nested -> {path, [{key, P} || P <- Nested]}
end.
-unwrap(<<"${", Val/binary>>) ->
- binary:part(Val, {0, byte_size(Val) - 1});
-unwrap(<<"\"${", Val/binary>>) ->
- binary:part(Val, {0, byte_size(Val) - 2}).
+unwrap(<<"\"${", Val/binary>>, _StripDoubleQuote = true) ->
+ binary:part(Val, {0, byte_size(Val) - 2});
+unwrap(<<"${", Val/binary>>, _StripDoubleQuote) ->
+ binary:part(Val, {0, byte_size(Val) - 1}).
quote_sql(Str) ->
quote(Str, <<"\\\\'">>).
diff --git a/apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl b/apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl
index ea642a9b0..bb83ce40c 100644
--- a/apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl
+++ b/apps/emqx_plugin_libs/test/emqx_placeholder_SUITE.erl
@@ -150,20 +150,25 @@ t_preproc_sql6(_) ->
emqx_placeholder:proc_sql(ParamsTokens, Selected)
).
-t_preproc_sql7(_) ->
+t_preproc_sql_strip_double_quote(_) ->
Selected = #{a => <<"a">>, b => <<"b">>},
+ Opts = #{replace_with => '$n', placeholders => [<<"${a}">>]},
+
+ %% no strip_double_quote option: "${key}" -> "value"
{PrepareStatement, ParamsTokens} = emqx_placeholder:preproc_sql(
<<"a:\"${a}\",b:\"${b}\"">>,
- #{
- replace_with => '$n',
- placeholders => [<<"${a}">>]
- }
+ Opts
),
- ?assertEqual(<<"a:$1,b:\"${b}\"">>, PrepareStatement),
- ?assertEqual(
- [<<"a">>],
- emqx_placeholder:proc_sql(ParamsTokens, Selected)
- ).
+ ?assertEqual(<<"a:\"$1\",b:\"${b}\"">>, PrepareStatement),
+ ?assertEqual([<<"a">>], emqx_placeholder:proc_sql(ParamsTokens, Selected)),
+
+ %% strip_double_quote = true: "${key}" -> value
+ {PrepareStatement1, ParamsTokens1} = emqx_placeholder:preproc_sql(
+ <<"a:\"${a}\",b:\"${b}\"">>,
+ Opts#{strip_double_quote => true}
+ ),
+ ?assertEqual(<<"a:$1,b:\"${b}\"">>, PrepareStatement1),
+ ?assertEqual([<<"a">>], emqx_placeholder:proc_sql(ParamsTokens1, Selected)).
t_preproc_tmpl_deep(_) ->
Selected = #{a => <<"1">>, b => 1, c => 1.0, d => #{d1 => <<"hi">>}},
diff --git a/apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf b/apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf
index b6dd2d3b5..c0009a040 100644
--- a/apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf
+++ b/apps/emqx_rule_engine/i18n/emqx_rule_engine_schema.conf
@@ -261,6 +261,17 @@ of the rule, then the string "undefined" is used.
}
}
+ rule_engine_jq_implementation_module {
+ desc {
+ en: "The implementation module for the jq rule engine function. The two options are jq_nif and jq_port. With the jq_nif option an Erlang NIF library is used while with the jq_port option an implementation based on Erlang port programs is used. The jq_nif option (the default option) is the fastest implementation of the two but jq_port is safer as the jq programs will not execute in the same process as the Erlang VM."
+ zh: "jq 规则引擎功能的实现模块。可用的两个选项是 jq_nif 和 jq_port。jq_nif 使用 Erlang NIF 库访问 jq 库，而 jq_port 使用基于 Erlang Port 的实现。jq_nif 方式（默认选项）是这两个选项中最快的实现，但 jq_port 方式更安全，因为这种情况下 jq 程序不会在 Erlang VM 进程中执行。"
+ }
+ label: {
+ en: "JQ Implementation Module"
+ zh: "JQ 实现模块"
+ }
+ }
+
desc_rule_engine {
desc {
en: """Configuration for the EMQX Rule Engine."""
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
index 61c0f4ac1..28f90fdb9 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src
@@ -2,7 +2,7 @@
{application, emqx_rule_engine, [
{description, "EMQX Rule Engine"},
% strict semver, bump manually!
- {vsn, "5.0.1"},
+ {vsn, "5.0.2"},
{modules, []},
{registered, [emqx_rule_engine_sup, emqx_rule_engine]},
{applications, [kernel, stdlib, rulesql, getopt]},
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl
index f991c7b4f..236e4b9aa 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl
@@ -320,6 +320,10 @@ init([]) ->
{write_concurrency, true},
{read_concurrency, true}
]),
+ ok = emqx_config_handler:add_handler(
+ [rule_engine, jq_implementation_module],
+ emqx_rule_engine_schema
+ ),
{ok, #{}}.
handle_call({insert_rule, Rule}, _From, State) ->
diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
index c6e7a2bc0..eec41bde8 100644
--- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
+++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl
@@ -25,7 +25,8 @@
namespace/0,
roots/0,
fields/1,
- desc/1
+ desc/1,
+ post_config_update/5
]).
-export([validate_sql/1]).
@@ -49,6 +50,15 @@ fields("rule_engine") ->
default => "10s",
desc => ?DESC("rule_engine_jq_function_default_timeout")
}
+ )},
+ {jq_implementation_module,
+ ?HOCON(
+ hoconsc:enum([jq_nif, jq_port]),
+ #{
+ default => jq_nif,
+ mapping => "jq.jq_implementation_module",
+ desc => ?DESC("rule_engine_jq_implementation_module")
+ }
)}
];
fields("rules") ->
@@ -209,3 +219,13 @@ validate_sql(Sql) ->
{ok, _Result} -> ok;
{error, Reason} -> {error, Reason}
end.
+
+post_config_update(
+ [rule_engine, jq_implementation_module],
+ _Req,
+ NewSysConf,
+ _OldSysConf,
+ _AppEnvs
+) ->
+ jq:set_implementation_module(NewSysConf),
+ ok.
diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src
index e87d293d3..866655b61 100644
--- a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src
+++ b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src
@@ -1,7 +1,7 @@
{application, emqx_slow_subs, [
{description, "EMQX Slow Subscribers Statistics"},
% strict semver, bump manually!
- {vsn, "1.0.1"},
+ {vsn, "1.0.2"},
{modules, []},
{registered, [emqx_slow_subs_sup]},
{applications, [kernel, stdlib, emqx]},
diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.erl b/apps/emqx_slow_subs/src/emqx_slow_subs.erl
index 2a07518b3..9a82abeb9 100644
--- a/apps/emqx_slow_subs/src/emqx_slow_subs.erl
+++ b/apps/emqx_slow_subs/src/emqx_slow_subs.erl
@@ -166,11 +166,11 @@ init([]) ->
expire_timer => undefined
},
- Enable = emqx:get_config([slow_subs, enable]),
- {ok, check_enable(Enable, InitState)}.
+ Cfg = emqx:get_config([slow_subs]),
+ {ok, check_enable(Cfg, InitState)}.
-handle_call({update_settings, #{enable := Enable}}, _From, State) ->
- State2 = check_enable(Enable, State),
+handle_call({update_settings, Cfg}, _From, State) ->
+ State2 = check_enable(Cfg, State),
{reply, ok, State2};
handle_call(clear_history, _, State) ->
do_clear_history(),
@@ -206,12 +206,14 @@ code_change(_OldVsn, State, _Extra) ->
expire_tick() ->
erlang:send_after(?EXPIRE_CHECK_INTERVAL, self(), ?FUNCTION_NAME).
-load(State) ->
+load(
#{
top_k_num := MaxSizeT,
stats_type := StatsType,
threshold := Threshold
- } = emqx:get_config([slow_subs]),
+ },
+ State
+) ->
MaxSize = erlang:min(MaxSizeT, ?MAX_SIZE),
ok = emqx_hooks:put(
'delivery.completed',
@@ -334,15 +336,15 @@ do_clear_history() ->
ets:delete_all_objects(?INDEX_TAB),
ets:delete_all_objects(?TOPK_TAB).
-check_enable(Enable, #{enable := IsEnable} = State) ->
+check_enable(#{enable := Enable} = Cfg, #{enable := IsEnable} = State) ->
case {IsEnable, Enable} of
{false, true} ->
- load(State);
+ load(Cfg, State);
{true, false} ->
unload(State);
{true, true} ->
S1 = unload(State),
- load(S1);
+ load(Cfg, S1);
_ ->
State
end.
diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl
index cd2f76f11..c24d043e6 100644
--- a/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl
+++ b/apps/emqx_slow_subs/src/emqx_slow_subs_api.erl
@@ -25,7 +25,7 @@
-export([api_spec/0, paths/0, schema/1, fields/1, namespace/0]).
-export([slow_subs/2, get_history/0, settings/2]).
--define(TAGS, [<<"Slow subscriptions">>]).
+-define(TAGS, [<<"Slow Subscriptions">>]).
-import(hoconsc, [mk/2, ref/1, ref/2]).
-import(emqx_mgmt_util, [bad_request/0]).
diff --git a/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl b/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl
index 0547eb1f8..f7ef78ed8 100644
--- a/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl
+++ b/apps/emqx_slow_subs/test/emqx_slow_subs_SUITE.erl
@@ -26,13 +26,15 @@
-define(NOW, erlang:system_time(millisecond)).
-define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard).
+-define(LANTENCY, 101).
-define(BASE_CONF, <<
""
"\n"
"slow_subs {\n"
" enable = true\n"
- " top_k_num = 5,\n"
+ " top_k_num = 5\n"
+ " threshold = 100ms\n"
" expire_interval = 5m\n"
" stats_type = whole\n"
" }"
@@ -64,10 +66,10 @@ end_per_suite(_Config) ->
init_per_testcase(t_expire, Config) ->
{ok, _} = emqx_cluster_rpc:start_link(),
- Cfg = emqx_config:get([slow_subs]),
- emqx_slow_subs:update_settings(Cfg#{expire_interval := 1500}),
+ update_config(<<"expire_interval">>, <<"1500ms">>),
Config;
init_per_testcase(_, Config) ->
+ {ok, _} = emqx_cluster_rpc:start_link(),
Config.
end_per_testcase(_, _) ->
@@ -84,38 +86,7 @@ end_per_testcase(_, _) ->
%% Test Cases
%%--------------------------------------------------------------------
t_pub(_) ->
- %% Sub topic first
- Subs = [{<<"/test1/+">>, ?QOS_1}, {<<"/test2/+">>, ?QOS_2}],
- Clients = start_client(Subs),
- timer:sleep(1000),
- Now = ?NOW,
- %% publish
-
- lists:foreach(
- fun(I) ->
- Topic = list_to_binary(io_lib:format("/test1/~p", [I])),
- Msg = emqx_message:make(undefined, ?QOS_1, Topic, <<"Hello">>),
- emqx:publish(Msg#message{timestamp = Now - 500}),
- timer:sleep(100)
- end,
- lists:seq(1, 10)
- ),
-
- lists:foreach(
- fun(I) ->
- Topic = list_to_binary(io_lib:format("/test2/~p", [I])),
- Msg = emqx_message:make(undefined, ?QOS_2, Topic, <<"Hello">>),
- emqx:publish(Msg#message{timestamp = Now - 500}),
- timer:sleep(100)
- end,
- lists:seq(1, 10)
- ),
-
- timer:sleep(1000),
- Size = ets:info(?TOPK_TAB, size),
- ?assert(Size =< 10 andalso Size >= 3, io_lib:format("the size is :~p~n", [Size])),
-
- [Client ! stop || Client <- Clients],
+ _ = [stats_with_type(Type) || Type <- [whole, internal, response]],
ok.
t_expire(_) ->
@@ -135,16 +106,12 @@ t_expire(_) ->
?assertEqual(0, Size),
ok.
-start_client(Subs) ->
- [spawn(fun() -> client(I, Subs) end) || I <- lists:seq(1, 10)].
+start_client(Type, Subs) ->
+ [spawn(fun() -> client(I, Type, Subs) end) || I <- lists:seq(1, 10)].
-client(I, Subs) ->
- {ok, C} = emqtt:start_link([
- {host, "localhost"},
- {clientid, io_lib:format("slow_subs_~p", [I])},
- {username, <<"plain">>},
- {password, <<"plain">>}
- ]),
+client(I, Type, Subs) ->
+ ConnOptions = make_conn_options(Type, I),
+ {ok, C} = emqtt:start_link(ConnOptions),
{ok, _} = emqtt:connect(C),
Len = erlang:length(Subs),
@@ -155,3 +122,81 @@ client(I, Subs) ->
stop ->
ok
end.
+
+stats_with_type(Type) ->
+ emqx_slow_subs:clear_history(),
+ update_stats_type(Type),
+ %% Sub topic first
+ Subs = [{<<"/test1/+">>, ?QOS_1}, {<<"/test2/+">>, ?QOS_2}],
+ Clients = start_client(Type, Subs),
+ timer:sleep(1000),
+ Now = ?NOW,
+ %% publish
+
+ lists:foreach(
+ fun(I) ->
+ Topic = list_to_binary(io_lib:format("/test1/~p", [I])),
+ Msg = emqx_message:make(undefined, ?QOS_1, Topic, <<"Hello">>),
+ emqx:publish(Msg#message{timestamp = Now - ?LANTENCY}),
+ timer:sleep(100)
+ end,
+ lists:seq(1, 10)
+ ),
+
+ lists:foreach(
+ fun(I) ->
+ Topic = list_to_binary(io_lib:format("/test2/~p", [I])),
+ Msg = emqx_message:make(undefined, ?QOS_2, Topic, <<"Hello">>),
+ emqx:publish(Msg#message{timestamp = Now - ?LANTENCY}),
+ timer:sleep(100)
+ end,
+ lists:seq(1, 10)
+ ),
+
+ timer:sleep(1000),
+ Size = ets:info(?TOPK_TAB, size),
+ ?assert(
+ Size =< 10 andalso Size >= 3,
+ lists:flatten(io_lib:format("with_type:~p, the size is :~p~n", [Type, Size]))
+ ),
+
+ ?assert(
+ lists:all(
+ fun(#{timespan := Ts}) ->
+ Ts >= 101 andalso Ts < ?NOW - Now
+ end,
+ emqx_slow_subs_api:get_history()
+ )
+ ),
+
+ [Client ! stop || Client <- Clients],
+ ok.
+
+update_stats_type(Type) ->
+ update_config(<<"stats_type">>, erlang:atom_to_binary(Type)).
+
+update_config(Key, Value) ->
+ Raw = #{
+ <<"enable">> => true,
+ <<"expire_interval">> => <<"5m">>,
+ <<"stats_type">> => <<"whole">>,
+ <<"threshold">> => <<"100ms">>,
+ <<"top_k_num">> => 5
+ },
+ emqx_slow_subs:update_settings(Raw#{Key => Value}).
+
+make_conn_options(response, I) ->
+ [
+ {msg_handler, #{
+ publish => fun(_) -> timer:sleep(?LANTENCY) end,
+ disconnected => fun(_) -> ok end
+ }}
+ | make_conn_options(whole, I)
+ ];
+make_conn_options(_, I) ->
+ [
+ {host, "localhost"},
+ {clientid, io_lib:format("slow_subs_~p", [I])},
+ {username, <<"plain">>},
+ {password, <<"plain">>}
+ ].
diff --git a/apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf b/apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf
new file mode 100644
index 000000000..2721188bd
--- /dev/null
+++ b/apps/emqx_statsd/i18n/emqx_statsd_api_i18n.conf
@@ -0,0 +1,16 @@
+emqx_statsd_api {
+
+ get_statsd_config_api {
+ desc {
+ en: """List the configuration of StatsD metrics collection and push service."""
+ zh: """列出 StatsD 指标采集和推送服务的的配置。"""
+ }
+ }
+
+ update_statsd_config_api {
+ desc {
+ en: """Update the configuration of StatsD metrics collection and push service."""
+ zh: """更新 StatsD 指标采集和推送服务的配置。"""
+ }
+ }
+}
diff --git a/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf b/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf
index 4ccad1682..9c6eb5afb 100644
--- a/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf
+++ b/apps/emqx_statsd/i18n/emqx_statsd_schema_i18n.conf
@@ -1,9 +1,23 @@
emqx_statsd_schema {
+ get_statsd_config_api {
+ desc {
+ en: """List the configuration of StatsD metrics collection and push service."""
+ zh: """列出 StatsD 指标采集和推送服务的的配置。"""
+ }
+ }
+
+ update_statsd_config_api {
+ desc {
+ en: """Update the configuration of StatsD metrics collection and push service."""
+ zh: """更新 StatsD 指标采集和推送服务的配置。"""
+ }
+ }
+
statsd {
desc {
- en: """Settings for reporting metrics to StatsD"""
- zh: """StatsD 监控数据推送"""
+ en: """StatsD metrics collection and push configuration."""
+ zh: """StatsD 指标采集与推送配置。"""
}
label {
en: """StatsD"""
@@ -13,29 +27,29 @@ emqx_statsd_schema {
server {
desc {
- en: """URL of StatsD server"""
- zh: """StatsD 服务器地址"""
+ en: """StatsD server address."""
+ zh: """StatsD 服务器地址。"""
}
}
sample_interval {
desc {
- en: """Data collection interval."""
- zh: """数据收集间隔"""
+ en: """The sampling interval for metrics."""
+ zh: """指标的采样间隔。"""
}
}
flush_interval {
desc {
- en: """Data reporting interval."""
- zh: """数据推送间隔"""
+ en: """The push interval for metrics."""
+ zh: """指标的推送间隔。"""
}
}
enable {
desc {
- en: """Turn StatsD data pushing on or off"""
- zh: """开启或关闭 StatsD 数据推送"""
+ en: """Enable or disable StatsD metrics collection and push service."""
+ zh: """启用或禁用 StatsD 指标采集和推送服务。"""
}
}
}
diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src
index 21a972266..76b04204b 100644
--- a/apps/emqx_statsd/src/emqx_statsd.app.src
+++ b/apps/emqx_statsd/src/emqx_statsd.app.src
@@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_statsd, [
{description, "An OTP application"},
- {vsn, "5.0.1"},
+ {vsn, "5.0.2"},
{registered, []},
{mod, {emqx_statsd_app, []}},
{applications, [
diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl
index ee6007d7d..2f2e42303 100644
--- a/apps/emqx_statsd/src/emqx_statsd_api.erl
+++ b/apps/emqx_statsd/src/emqx_statsd_api.erl
@@ -20,6 +20,7 @@
-include("emqx_statsd.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
-import(hoconsc, [mk/2, ref/2]).
@@ -48,14 +49,14 @@ schema("/statsd") ->
'operationId' => statsd,
get =>
#{
- description => <<"Get statsd config">>,
+ description => ?DESC(get_statsd_config_api),
tags => ?API_TAG_STATSD,
responses =>
#{200 => statsd_config_schema()}
},
put =>
#{
- description => <<"Set statsd config">>,
+ description => ?DESC(update_statsd_config_api),
tags => ?API_TAG_STATSD,
'requestBody' => statsd_config_schema(),
responses =>
diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl
index e8cc32f99..9efde5afc 100644
--- a/apps/emqx_statsd/src/emqx_statsd_schema.erl
+++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl
@@ -21,8 +21,6 @@
-behaviour(hocon_schema).
--export([to_ip_port/1]).
-
-export([
namespace/0,
roots/0,
@@ -30,8 +28,6 @@
desc/1
]).
--typerefl_from_string({ip_port/0, emqx_statsd_schema, to_ip_port}).
-
namespace() -> "statsd".
roots() -> ["statsd"].
@@ -55,7 +51,7 @@ fields("statsd") ->
desc("statsd") -> ?DESC(statsd);
desc(_) -> undefined.
-server(type) -> emqx_schema:ip_port();
+server(type) -> emqx_schema:host_port();
server(required) -> true;
server(default) -> "127.0.0.1:8125";
server(desc) -> ?DESC(?FUNCTION_NAME);
@@ -72,14 +68,3 @@ flush_interval(required) -> true;
flush_interval(default) -> "10s";
flush_interval(desc) -> ?DESC(?FUNCTION_NAME);
flush_interval(_) -> undefined.
-
-to_ip_port(Str) ->
- case string:tokens(Str, ":") of
- [Ip, Port] ->
- case inet:parse_address(Ip) of
- {ok, R} -> {ok, {R, list_to_integer(Port)}};
- _ -> {error, Str}
- end;
- _ ->
- {error, Str}
- end.
diff --git a/bin/emqx b/bin/emqx
index 63aae6e70..811bb2981 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -7,7 +7,11 @@ set -euo pipefail
DEBUG="${DEBUG:-0}"
[ "$DEBUG" -eq 1 ] && set -x
-RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
+if [ "$(uname -s)" == 'Darwin' ]; then
+ RUNNER_ROOT_DIR="$(cd "$(dirname "$(stat -f%R "$0" || echo "$0")")"/..; pwd -P)"
+else
+ RUNNER_ROOT_DIR="$(cd "$(dirname "$(realpath "$0" || echo "$0")")"/..; pwd -P)"
+fi
# shellcheck disable=SC1090,SC1091
. "$RUNNER_ROOT_DIR"/releases/emqx_vars
@@ -24,9 +28,6 @@ REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN"
WHOAMI=$(whoami)
-# Make sure log directory exists
-mkdir -p "$RUNNER_LOG_DIR"
-
# hocon try to read environment variables starting with "EMQX_"
export HOCON_ENV_OVERRIDE_PREFIX='EMQX_'
@@ -262,6 +263,9 @@ if [ "$ES" -ne 0 ]; then
exit $ES
fi
+# Make sure log directory exists
+mkdir -p "$RUNNER_LOG_DIR"
+
COMPATIBILITY_CHECK='
io:format("BEAM_OK~n", []),
try
diff --git a/build b/build
index dd2b96604..bd5307014 100755
--- a/build
+++ b/build
@@ -364,6 +364,7 @@ export_release_vars() {
# compiling the project, so that `emqx_release.erl' picks up
# `emqx_vsn' as if it was compiled by rebar3.
erl_opts+=( "{compile_info,[{emqx_vsn,\"${PKG_VSN}\"}]}" )
+ erl_opts+=( "{d,snk_kind,msg}" )
ERL_COMPILER_OPTIONS="[$(join , "${erl_opts[@]}")]"
export ERL_COMPILER_OPTIONS
diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md
index 9c3762fdd..a579af70d 100644
--- a/deploy/charts/emqx-enterprise/README.md
+++ b/deploy/charts/emqx-enterprise/README.md
@@ -60,10 +60,9 @@ The following table lists the configurable parameters of the emqx chart and thei
| `service.type` | Kubernetes Service type. | ClusterIP |
| `service.mqtt` | Port for MQTT. | 1883 |
| `service.mqttssl` | Port for MQTT(SSL). | 8883 |
-| `service.mgmt` | Port for mgmt API. | 8081 |
| `service.ws` | Port for WebSocket/HTTP. | 8083 |
| `service.wss` | Port for WSS/HTTPS. | 8084 |
-| `service.dashboard` | Port for dashboard. | 18083 |
+| `service.dashboard` | Port for dashboard and API. | 18083 |
| `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil |
| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil |
| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil |
diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml
index aa61a62ea..7827d6afb 100644
--- a/deploy/charts/emqx-enterprise/values.yaml
+++ b/deploy/charts/emqx-enterprise/values.yaml
@@ -123,19 +123,16 @@ service:
## Port for MQTT(SSL)
##
mqttssl: 8883
- ## Port for mgmt API
- ##
- mgmt: 8081
## Port for WebSocket/HTTP
##
ws: 8083
## Port for WSS/HTTPS
##
wss: 8084
- ## Port for dashboard
+ ## Port for dashboard and API
##
dashboard: 18083
- ## Port for dashboard HTTPS
+ ## Port for dashboard and API over HTTPS
##
# dashboardtls: 18084
## Specify the nodePort(s) value for the LoadBalancer and NodePort service types.
diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml
index 544d848fd..fa265e663 100644
--- a/deploy/charts/emqx/Chart.yaml
+++ b/deploy/charts/emqx/Chart.yaml
@@ -14,8 +14,8 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
-version: 5.0.7
+version: 5.0.8
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application.
-appVersion: 5.0.7
+appVersion: 5.0.8
diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md
index ed331619d..84b5ab4c9 100644
--- a/deploy/charts/emqx/README.md
+++ b/deploy/charts/emqx/README.md
@@ -60,10 +60,9 @@ The following table lists the configurable parameters of the emqx chart and thei
| `service.type` | Kubernetes Service type. | ClusterIP |
| `service.mqtt` | Port for MQTT. | 1883 |
| `service.mqttssl` | Port for MQTT(SSL). | 8883 |
-| `service.mgmt` | Port for mgmt API. | 8081 |
| `service.ws` | Port for WebSocket/HTTP. | 8083 |
| `service.wss` | Port for WSS/HTTPS. | 8084 |
-| `service.dashboard` | Port for dashboard. | 18083 |
+| `service.dashboard` | Port for dashboard and API. | 18083 |
| `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil |
| `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil |
| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil |
diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml
index 5bf7377f4..b648f070f 100644
--- a/deploy/charts/emqx/values.yaml
+++ b/deploy/charts/emqx/values.yaml
@@ -125,19 +125,16 @@ service:
## Port for MQTT(SSL)
##
mqttssl: 8883
- ## Port for mgmt API
- ##
- mgmt: 8081
## Port for WebSocket/HTTP
##
ws: 8083
## Port for WSS/HTTPS
##
wss: 8084
- ## Port for dashboard
+ ## Port for dashboard and API
##
dashboard: 18083
- ## Port for dashboard HTTPS
+ ## Port for dashboard and API over HTTPS
##
# dashboardtls: 18084
## Specify the nodePort(s) value for the LoadBalancer and NodePort service types.
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 6c5baa391..51b4dbd0c 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -16,6 +16,7 @@ RUN export PROFILE=${EMQX_NAME%%-elixir} \
&& cd /emqx \
&& rm -rf $EMQX_LIB_PATH \
&& make $EMQX_NAME1 \
+ && rm -f $EMQX_REL_PATH/*.tar.gz \
&& mkdir -p /emqx-rel \
&& mv $EMQX_REL_PATH /emqx-rel
@@ -48,15 +49,14 @@ VOLUME ["/opt/emqx/log", "/opt/emqx/data"]
# emqx will occupy these port:
# - 1883 port for MQTT
-# - 8081 for mgmt API
# - 8083 for WebSocket/HTTP
# - 8084 for WSS/HTTPS
# - 8883 port for MQTT(SSL)
# - 11883 port for internal MQTT/TCP
-# - 18083 for dashboard
+# - 18083 for dashboard and API
# - 4370 default Erlang distribution port
# - 5369 for backplain gen_rpc
-EXPOSE 1883 8081 8083 8084 8883 11883 18083 4370 5369
+EXPOSE 1883 8083 8084 8883 11883 18083 4370 5369
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
diff --git a/deploy/docker/Dockerfile.alpine b/deploy/docker/Dockerfile.alpine
index 6368a0b51..dc16afc54 100644
--- a/deploy/docker/Dockerfile.alpine
+++ b/deploy/docker/Dockerfile.alpine
@@ -62,15 +62,14 @@ VOLUME ["/opt/emqx/log", "/opt/emqx/data"]
# emqx will occupy these port:
# - 1883 port for MQTT
-# - 8081 for mgmt API
# - 8083 for WebSocket/HTTP
# - 8084 for WSS/HTTPS
# - 8883 port for MQTT(SSL)
# - 11883 port for internal MQTT/TCP
-# - 18083 for dashboard
+# - 18083 for dashboard and API
# - 4370 default Erlang distrbution port
# - 5369 for backplain gen_rpc
-EXPOSE 1883 8081 8083 8084 8883 11883 18083 4370 5369
+EXPOSE 1883 8083 8084 8883 11883 18083 4370 5369
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
diff --git a/mix.exs b/mix.exs
index 3f55de64b..a2863f137 100644
--- a/mix.exs
+++ b/mix.exs
@@ -52,9 +52,9 @@ defmodule EMQXUmbrella.MixProject do
{:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
{:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
{:esockd, github: "emqx/esockd", tag: "5.9.4", override: true},
- {:ekka, github: "emqx/ekka", tag: "0.13.4", override: true},
+ {:ekka, github: "emqx/ekka", tag: "0.13.5", override: true},
{:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
- {:grpc, github: "emqx/grpc-erl", tag: "0.6.6", override: true},
+ {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true},
{:minirest, github: "emqx/minirest", tag: "1.3.7", override: true},
{:ecpool, github: "emqx/ecpool", tag: "0.5.2"},
{:replayq, "0.3.4", override: true},
@@ -88,7 +88,7 @@ defmodule EMQXUmbrella.MixProject do
{:ranch,
github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
# in conflict by grpc and eetcd
- {:gpb, "4.11.2", override: true, runtime: false}
+ {:gpb, "4.19.5", override: true, runtime: false}
] ++
umbrella_apps() ++ enterprise_apps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
end
@@ -616,7 +616,7 @@ defmodule EMQXUmbrella.MixProject do
defp jq_dep() do
if enable_jq?(),
- do: [{:jq, github: "emqx/jq", tag: "v0.3.5", override: true}],
+ do: [{:jq, github: "emqx/jq", tag: "v0.3.6", override: true}],
else: []
end
diff --git a/pkg-vsn.sh b/pkg-vsn.sh
index 7f50f03e2..281160de8 100755
--- a/pkg-vsn.sh
+++ b/pkg-vsn.sh
@@ -73,6 +73,12 @@ while [ "$#" -gt 0 ]; do
esac
done
+# return immediately if version is already set
+if [[ "${PKG_VSN:-novalue}" != novalue && "${LONG_VERSION:-novalue}" != 'yes' ]]; then
+ echo "$PKG_VSN"
+ exit 0
+fi
+
case "${PROFILE}" in
*enterprise*)
RELEASE_EDITION="EMQX_RELEASE_EE"
diff --git a/rebar.config b/rebar.config
index 271e274da..9be69d963 100644
--- a/rebar.config
+++ b/rebar.config
@@ -46,7 +46,7 @@
{deps,
[ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.1"}}}
, {redbug, "2.0.7"}
- , {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
+ , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
, {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}}
, {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.7"}}}
, {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.3.0"}}}
@@ -54,9 +54,9 @@
, {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}}
- , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.4"}}}
+ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.5"}}}
, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
- , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.6"}}}
+ , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}}
, {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.7"}}}
, {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}
, {replayq, "0.3.4"}
diff --git a/rebar.config.erl b/rebar.config.erl
index ce1930ed6..eb79e6028 100644
--- a/rebar.config.erl
+++ b/rebar.config.erl
@@ -42,7 +42,7 @@ quicer() ->
{quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}.
jq() ->
- {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.5"}}}.
+ {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.6"}}}.
deps(Config) ->
{deps, OldDeps} = lists:keyfind(deps, 1, Config),
diff --git a/rel/emqx_conf.template.en.md b/rel/emqx_conf.template.en.md
index 76d25680b..520ed4578 100644
--- a/rel/emqx_conf.template.en.md
+++ b/rel/emqx_conf.template.en.md
@@ -143,7 +143,19 @@ to even set complex values from environment variables.
For example, this environment variable sets an array value.
```
-export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]"
+export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]'
+```
+
+However this also means a string value should be quoted if it happen to contain special
+characters such as `=` and `:`.
+
+For example, a string value `"localhost:1883"` would be
+parsed into object (struct): `{"localhost": 1883}`.
+
+To keep it as a string, one should quote the value like below:
+
+```
+EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"'
```
::: tip Tip
diff --git a/rel/emqx_conf.template.zh.md b/rel/emqx_conf.template.zh.md
index ac4c5ce39..86162a4a9 100644
--- a/rel/emqx_conf.template.zh.md
+++ b/rel/emqx_conf.template.zh.md
@@ -127,14 +127,24 @@ authentication.1.enable = true
例如 `node.name` 的重载变量名是 `EMQX_NODE__NAME`。
-环境变量的值，是解析成HOCON值的。所以这也使得环境变量可以用来传递复杂数据类型的值。
+环境变量的值，是按 HOCON 值解析的，这也使得环境变量可以用来传递复杂数据类型的值。
例如，下面这个环境变量传入一个数组类型的值。
```
-export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS="[\"TLS_AES_256_GCM_SHA384\"]"
+export EMQX_LISTENERS__SSL__L1__AUTHENTICATION__SSL__CIPHERS='["TLS_AES_256_GCM_SHA384"]'
```
+这也意味着有些带特殊字符（例如`:` 和 `=`），则需要用双引号对这个值包起来。
+
+例如`localhost:1883` 会被解析成一个结构体 `{"localhost": 1883}`。
+想要把它当字符串使用时，就必需使用引号，如下：
+
+```
+EMQX_BRIDGES__MQTT__MYBRIDGE__CONNECTOR_SERVER='"localhost:1883"'
+```
+
+
::: tip Tip
未定义的根路径会被EMQX忽略，例如 `EMQX_UNKNOWN_ROOT__FOOBAR` 这个环境变量会被EMQX忽略，
因为 `UNKNOWN_ROOT` 不是预先定义好的根路径。
diff --git a/rel/emqx_vars b/rel/emqx_vars
index 1ec95b4de..e3965d40c 100644
--- a/rel/emqx_vars
+++ b/rel/emqx_vars
@@ -9,9 +9,9 @@ ERL_OPTS="{{ erl_opts }}"
RUNNER_BIN_DIR="{{ runner_bin_dir }}"
RUNNER_LIB_DIR="{{ runner_lib_dir }}"
IS_ELIXIR="${IS_ELIXIR:-{{ is_elixir }}}"
-## Allow users to pre-set `RUNNER_LOG_DIR` because it only affects boot commands like `start` and `console`,
+## Allow users to pre-set `EMQX_LOG_DIR` because it only affects boot commands like `start` and `console`,
## but not other commands such as `ping` and `ctl`.
-RUNNER_LOG_DIR="${RUNNER_LOG_DIR:-{{ runner_log_dir }}}"
+RUNNER_LOG_DIR="${EMQX_LOG_DIR:-${RUNNER_LOG_DIR:-{{ runner_log_dir }}}}"
EMQX_ETC_DIR="{{ emqx_etc_dir }}"
RUNNER_USER="{{ runner_user }}"
SCHEMA_MOD="{{ emqx_schema_mod }}"
diff --git a/scripts/git-hook-post-commit.sh b/scripts/git-hook-post-commit.sh
new file mode 100755
index 000000000..cc1f079c6
--- /dev/null
+++ b/scripts/git-hook-post-commit.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+exec git config --local blame.ignoreRevsFile git-blame-ignore-revs
diff --git a/scripts/git-hooks-init.sh b/scripts/git-hooks-init.sh
index 6b85e23e3..05d76b27a 100755
--- a/scripts/git-hooks-init.sh
+++ b/scripts/git-hooks-init.sh
@@ -15,3 +15,7 @@ fi
if [ ! -L '.git/hooks/pre-commit' ]; then
ln -sf '../../scripts/git-hook-pre-commit.sh' '.git/hooks/pre-commit'
fi
+
+if [ ! -L '.git/hooks/post-commit' ]; then
+ ln -sf '../../scripts/git-hook-post-commit.sh' '.git/hooks/post-commit'
+fi
diff --git a/scripts/macos-sign-binaries.sh b/scripts/macos-sign-binaries.sh
index a69b9a49b..11b6b734d 100755
--- a/scripts/macos-sign-binaries.sh
+++ b/scripts/macos-sign-binaries.sh
@@ -52,4 +52,4 @@ codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=r
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/rocksdb-*/priv/liberocksdb.so
codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
-codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/quicer-*/priv/libquicer_nif.so
+find "${REL_DIR}/lib/" -name libquicer_nif.so -exec codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime {} \;
diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt
index 8355e3d03..065cf1a3a 100644
--- a/scripts/spellcheck/dicts/emqx.txt
+++ b/scripts/spellcheck/dicts/emqx.txt
@@ -263,3 +263,5 @@ hstreamdb
SASL
GSSAPI
keytab
+jq
+nif
diff --git a/scripts/test-node-discovery-dns.sh b/scripts/test-node-discovery-dns.sh
index 968269042..e502dbf59 100755
--- a/scripts/test-node-discovery-dns.sh
+++ b/scripts/test-node-discovery-dns.sh
@@ -1,12 +1,18 @@
#!/usr/bin/env bash
-## Test two nodes-cluster discover each other using DNS A records lookup result.
+## Test two-nodes cluster discover each other using DNS A records lookup result.
set -euo pipefail
cd -P -- "$(dirname -- "$0")/.."
-IMAGE="${1}"
+IMAGE="${1:-}"
+
+if [ -z "$IMAGE" ]; then
+ echo "Usage: $0 "
+ echo "e.g. $0 docker.io/emqx/emqx:5.0.8"
+ exit 1
+fi
NET='test_node_discovery_dns'
NODE1='emqx1'
@@ -56,7 +62,9 @@ docker run -d -t --name dnsmasq \
--cap-add=NET_ADMIN \
storytel/dnsmasq dnsmasq --no-daemon --log-queries
-start_emqx() {
+# Node names (the part before '@') should be all the same in the cluster
+# e.g. emqx@${IP}
+start_emqx_v5() {
NAME="$1"
IP="$2"
DASHBOARD_PORT="$3"
@@ -66,6 +74,7 @@ start_emqx() {
--ip "$IP" \
--dns "$IP0" \
-p "$DASHBOARD_PORT:18083" \
+ -e EMQX_NODE_NAME="emqx@${IP}" \
-e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \
-e EMQX_NODE_COOKIE="$COOKIE" \
-e EMQX_cluster__discovery_strategy='dns' \
@@ -74,5 +83,48 @@ start_emqx() {
"$IMAGE"
}
-start_emqx "$NODE1" "$IP1" 18083
-start_emqx "$NODE2" "$IP2" 18084
+## EMQX v4 has different configuration schema:
+# EMQX_NODE_NAME="emqx@${IP}":
+# This is necessary because 4.x docker entrypoint
+# by default uses docker container ID as node name
+# (the part before @ of e.g. emqx@172.18.0.101)
+# EMQX_cluster__dns__app
+# This must be the same as node name in 4.x
+# EMQX_cluster__discovery
+# This in 5.0 is EMQX_cluster__discovery_strategy
+# EMQX_cluster__dns__name
+# The DNS domain to lookup for peer nodes
+# EMQX_cluster__dns__record_type
+# The DNS record type. (only 'a' type is tested)
+start_emqx_v4() {
+ NAME="$1"
+ IP="$2"
+ APP_NAME="emqx"
+ DASHBOARD_PORT="$3"
+ docker run -d -t \
+ --name "$NAME" \
+ --net "$NET" \
+ --ip "$IP" \
+ --dns "$IP0" \
+ -p "$DASHBOARD_PORT:18083" \
+ -e EMQX_NODE_NAME="${APP_NAME}@${IP}" \
+ -e EMQX_LOG__LEVEL=debug \
+ -e EMQX_NODE_COOKIE="$COOKIE" \
+ -e EMQX_cluster__discovery='dns' \
+ -e EMQX_cluster__dns__name="$DOMAIN" \
+ -e EMQX_cluster__dns__app="${APP_NAME}" \
+ -e EMQX_cluster__dns__record_type="a" \
+ "$IMAGE"
+}
+
+case "${IMAGE}" in
+ *emqx:4.*|*emqx-ee:4.*)
+ start_emqx_v4 "$NODE1" "$IP1" 18083
+ start_emqx_v4 "$NODE2" "$IP2" 18084
+ ;;
+ *)
+ start_emqx_v5 "$NODE1" "$IP1" 18083
+ start_emqx_v5 "$NODE2" "$IP2" 18084
+ ;;
+
+esac
diff --git a/scripts/update-appup.sh b/scripts/update-appup.sh
index 1f2b23435..b962c4a9a 100755
--- a/scripts/update-appup.sh
+++ b/scripts/update-appup.sh
@@ -99,7 +99,7 @@ else
pushd "${PREV_DIR_BASE}/${PREV_TAG}"
if [ "$NEW_COPY" = 'no' ]; then
REMOTE="$(git remote -v | grep "${GIT_REPO}" | head -1 | awk '{print $1}')"
- git fetch "$REMOTE"
+ git fetch "$REMOTE" --tags --force
fi
git reset --hard
git clean -ffdx