yuanbiao
/
emqx
1
0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into authn-config-check

This commit is contained in:
zhongwencool 2022-10-14 16:36:50 +08:00 committed by GitHub
parent 5056cbebf4 6f077c47e7
commit bfc780f95b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
144 changed files with 2410 additions and 1056 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.ci/docker-compose-file/redis/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
r7000i.log
r7001i.log
r7002i.log

91
.github/actions/package-macos/action.yaml vendored Normal file
View File

@ -0,0 +1,91 @@
name: 'Create MacOS package'
inputs:
profile: # emqx, emqx-enterprise
required: true
type: string
otp: # 24.2.1-1, 23.3.4.9-3
required: true
type: string
os:
required: false
type: string
default: macos-11
apple_id_password:
required: true
type: string
apple_developer_identity:
required: true
type: string
apple_developer_id_bundle:
required: true
type: string
apple_developer_id_bundle_password:
required: true
type: string
runs:
using: composite
steps:
- name: prepare
shell: bash
run: |
brew update
brew install curl zip unzip kerl coreutils openssl@1.1
echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
echo "/usr/local/bin" >> $GITHUB_PATH
- uses: actions/cache@v2
id: cache
with:
path: ~/.kerl/${{ inputs.otp }}
key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit
- name: build erlang
if: steps.cache.outputs.cache-hit != 'true'
shell: bash
env:
KERL_BUILD_BACKEND: git
OTP_GITHUB_URL: https://github.com/emqx/otp
KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
run: |
kerl update releases
kerl build ${{ inputs.otp }}
kerl install ${{ inputs.otp }} $HOME/.kerl/${{ inputs.otp }}
- name: build ${{ inputs.profile }}
env:
AUTO_INSTALL_BUILD_DEPS: 1
APPLE_SIGN_BINARIES: 1
APPLE_ID: developers@emqx.io
APPLE_TEAM_ID: 26N6HYJLZA
APPLE_ID_PASSWORD: ${{ inputs.apple_id_password }}
APPLE_DEVELOPER_IDENTITY: ${{ inputs.apple_developer_identity }}
APPLE_DEVELOPER_ID_BUNDLE: ${{ inputs.apple_developer_id_bundle }}
APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }}
shell: bash
run: |
. $HOME/.kerl/${{ inputs.otp }}/activate
make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3
make ${{ inputs.profile }}-tgz
- name: test ${{ inputs.profile }}
shell: bash
run: |
pkg_name=$(find _packages/${{ inputs.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
mkdir emqx
unzip -d emqx $pkg_name > /dev/null
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no'
for i in {1..30}; do
if curl -fs 127.0.0.1:18083/status > /dev/null; then
ready='yes'
break
fi
sleep 1
done
if [ "$ready" != "yes" ]; then
echo "Timed out waiting for emqx to be ready"
cat emqx/log/erlang.log.1
exit 1
fi
./emqx/bin/emqx_ctl status
./emqx/bin/emqx stop
rm -rf emqx

72
.github/workflows/build_packages.yaml vendored
View File

@ -150,72 +150,26 @@ jobs:
name: source
path: .
- name: unzip source code
run: unzip -q source.zip
run: |
ln -s . source
unzip -q source.zip
rm source source.zip
- name: prepare
run: |
brew update
brew install curl zip unzip kerl coreutils
echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
echo "/usr/local/bin" >> $GITHUB_PATH
git config --global credential.helper store
- uses: actions/cache@v2
id: cache
- uses: ./.github/actions/package-macos
with:
path: ~/.kerl/${{ matrix.otp }}
key: otp-install-${{ matrix.otp }}-${{ matrix.os }}
- name: build erlang
if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60
env:
KERL_BUILD_BACKEND: git
OTP_GITHUB_URL: https://github.com/emqx/otp
run: |
kerl update releases
kerl build ${{ matrix.otp }}
kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
- name: build
working-directory: source
env:
AUTO_INSTALL_BUILD_DEPS: 1
APPLE_SIGN_BINARIES: 1
APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
run: |
. $HOME/.kerl/${{ matrix.otp }}/activate
make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3
rm -rf _build/${{ matrix.profile }}/lib
make ${{ matrix.profile }}-tgz
- name: test
working-directory: source
run: |
pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz)
mkdir -p emqx
tar -C emqx -zxf $pkg_name
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no'
for i in {1..18}; do
if curl -fs 127.0.0.1:18083/status > /dev/null; then
ready='yes'
break
fi
sleep 1
done
if [ "$ready" != "yes" ]; then
echo "Timed out waiting for emqx to be ready"
cat emqx/log/erlang.log.1
exit 1
fi
./emqx/bin/emqx_ctl status
./emqx/bin/emqx stop
rm -rf emqx
profile: ${{ matrix.profile }}
otp: ${{ matrix.otp }}
os: ${{ matrix.os }}
apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- uses: actions/upload-artifact@v1
with:
name: ${{ matrix.profile }}-${{ matrix.otp }}
path: source/_packages/${{ matrix.profile }}/.
path: _packages/${{ matrix.profile }}/.
linux:
needs: prepare

69
.github/workflows/build_slim_packages.yaml vendored
View File

@ -133,75 +133,26 @@ jobs:
- emqx-enterprise
otp:
- 24.2.1-1
macos:
os:
- macos-11
runs-on: ${{ matrix.macos }}
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
- name: prepare
run: |
brew update
brew install curl zip unzip kerl coreutils openssl@1.1
echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
echo "/usr/local/bin" >> $GITHUB_PATH
echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
echo "BUILD_WITH_QUIC=1" >> $GITHUB_ENV
- uses: actions/cache@v2
id: cache
- uses: ./.github/actions/package-macos
with:
path: ~/.kerl/${{ matrix.otp }}
key: otp-install-${{ matrix.otp }}-${{ matrix.macos }}-static-ssl-disable-hipe-disable-jit
- name: build erlang
if: steps.cache.outputs.cache-hit != 'true'
timeout-minutes: 60
env:
KERL_BUILD_BACKEND: git
OTP_GITHUB_URL: https://github.com/emqx/otp
KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
run: |
kerl update releases
kerl build ${{ matrix.otp }}
kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
- name: build ${{ matrix.profile }}
env:
AUTO_INSTALL_BUILD_DEPS: 1
APPLE_SIGN_BINARIES: 1
APPLE_ID: developers@emqx.io
APPLE_TEAM_ID: 26N6HYJLZA
APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
run: |
. $HOME/.kerl/${{ matrix.otp }}/activate
make ensure-rebar3
sudo cp rebar3 /usr/local/bin/rebar3
make ${{ matrix.profile }}-tgz
- name: test
run: |
pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
mkdir emqx
unzip -d emqx $pkg_name > /dev/null
# gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1
ready='no'
for i in {1..30}; do
if curl -fs 127.0.0.1:18083/status > /dev/null; then
ready='yes'
break
fi
sleep 1
done
if [ "$ready" != "yes" ]; then
echo "Timed out waiting for emqx to be ready"
cat emqx/log/erlang.log.1
exit 1
fi
./emqx/bin/emqx_ctl status
./emqx/bin/emqx stop
rm -rf emqx
profile: ${{ matrix.profile }}
otp: ${{ matrix.otp }}
os: ${{ matrix.os }}
apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
- uses: actions/upload-artifact@v2
with:
name: macos

1
.gitignore vendored
View File

@ -68,4 +68,3 @@ apps/emqx/test/emqx_static_checks_data/master.bpapi
# rendered configurations
*.conf.rendered
lux_logs/
.ci/docker-compose-file/redis/*.log

19
CHANGES-5.0.md
View File

@ -1,6 +1,18 @@
# 5.0.9
## Enhancements
* Add `cert_common_name` and `cert_subject` placeholder support for authz_http and authz_mongo.[#8973](https://github.com/emqx/emqx/pull/8973)
* Use milliseconds internally in emqx_delayed to store the publish time, improving precision.[#9060](https://github.com/emqx/emqx/pull/9060)
* More rigorous checking of flapping to improve stability of the system. [#9136](https://github.com/emqx/emqx/pull/9136)
## Bug fixes
* Check ACLs for last will testament topic before publishing the message. [#8930](https://github.com/emqx/emqx/pull/8930)
* Fix GET /listeners API crash When some nodes still in initial configuration. [#9002](https://github.com/emqx/emqx/pull/9002)
* Fix empty variable interpolation in authentication and authorization. Placeholders for undefined variables are rendered now as empty strings and do not cause errors anymore. [#8963](https://github.com/emqx/emqx/pull/8963)
* Fix the latency statistics error of the slow subscription module when `stats_type` is `internal` or `response`. [#8986](https://github.com/emqx/emqx/pull/8986)
* Redispatch shared subscription messages. [#9104](https://github.com/emqx/emqx/pull/9104)
* Ensure authentication type is an array, not struct. [#8923](https://github.com/emqx/emqx/pull/8923)
# 5.0.8
@ -16,14 +28,21 @@
* Fix delayed publish inaccurate caused by os time change. [#8926](https://github.com/emqx/emqx/pull/8926)
* Fix that EMQX can't start when the retainer is disabled [#8911](https://github.com/emqx/emqx/pull/8911)
* Fix that redis authn will deny the unknown users [#8934](https://github.com/emqx/emqx/pull/8934)
* Fix ExProto UDP client keepalive checking error.
This causes the clients to not expire as long as a new UDP packet arrives [#8866](https://github.com/emqx/emqx/pull/8866)
* Fix that MQTT Bridge message payload could be empty string. [#8949](https://github.com/emqx/emqx/pull/8949)
## Enhancements
* Print a warning message when boot with the default (insecure) Erlang cookie. [#8905](https://github.com/emqx/emqx/pull/8905)
* Change the `/gateway` API path to plural form. [#8823](https://github.com/emqx/emqx/pull/8823)
* Don't allow updating config items when they already exist in `local-override.conf`. [#8851](https://github.com/emqx/emqx/pull/8851)
* Remove `node.etc_dir` from emqx.conf, because it is never used.
Also allow user to customize the logging directory [#8892](https://github.com/emqx/emqx/pull/8892)
* Added a new API `POST /listeners` for creating listener. [#8876](https://github.com/emqx/emqx/pull/8876)
* Close ExProto client process immediately if it's keepalive timeouted. [#8866](https://github.com/emqx/emqx/pull/8866)
* Upgrade grpc-erl driver to 0.6.7 to support batch operation in sending stream. [#8866](https://github.com/emqx/emqx/pull/8866)
# 5.0.7

5
Dockerfile.ubuntu20.04.runner
View File

@ -27,15 +27,14 @@ VOLUME ["/opt/emqx/log", "/opt/emqx/data"]
# emqx will occupy these port:
# - 1883 port for MQTT
# - 8081 for mgmt API
# - 8083 for WebSocket/HTTP
# - 8084 for WSS/HTTPS
# - 8883 port for MQTT(SSL)
# - 11883 port for internal MQTT/TCP
# - 18083 for dashboard
# - 18083 for dashboard and API
# - 4370 default Erlang distrbution port
# - 5369 for backplain gen_rpc
EXPOSE 1883 8081 8083 8084 8883 11883 18083 4370 5369
EXPOSE 1883 8083 8084 8883 11883 18083 4370 5369
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]

2
Makefile
View File

@ -6,7 +6,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
export EMQX_DEFAULT_RUNNER = debian:11-slim
export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
export EMQX_DASHBOARD_VERSION ?= v1.0.8
export EMQX_DASHBOARD_VERSION ?= v1.0.9
export EMQX_EE_DASHBOARD_VERSION ?= e1.0.0
export EMQX_REL_FORM ?= tgz
export QUICER_DOWNLOAD_FROM_RELEASE = 1

27
README-CN.md
View File

@ -32,12 +32,6 @@ EMQX 自 2013 年在 GitHub 发布开源版本以来，获得了来自 50 多个
docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx:latest
```
或直接试用 EMQX 企业版（已内置 10 个并发连接的永不过期 License）
```
docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest
```
接下来请参考 [入门指南](https://www.emqx.io/docs/zh/v5.0/getting-started/getting-started.html#启动-emqx) 开启您的 EMQX 之旅。
#### 在 Kubernetes 上运行 EMQX 集群
@ -112,6 +106,27 @@ make
_build/emqx/rel/emqx/bin/emqx console
```
### 在 Apple 芯片（M1,M2）上编译
基于 Apple 芯片的 Homebrew 将[默认的 home 目录](https://github.com/Homebrew/brew/issues/9177)从 `/usr/local` 改成了 `/opt/homebrew`，这个改变导致了一些兼容性问题。
具体到 EMQX 来说，主要影响的是 `unixodbc`，如果使用 Homebrew 安装的 `unixodbc` 包，那么在使用 [kerl](https://github.com/kerl/kerl) 编译 Erlang/OTP 的时候，kerl 会找不到 `unixodbc`。
解决此问题的方法如下:
```bash
brew install unixodbc kerl
sudo ln -s $(realpath $(brew --prefix unixodbc)) /usr/local/odbc
export CC="/usr/bin/gcc -I$(brew --prefix unixodbc)/include"
export LDFLAGS="-L$(brew --prefix unixodbc)/lib"
kerl build 24.3
mkdir ~/.kerl/installations
kerl install 24.3 ~/.kerl/installations/24.3
. ~/.kerl/installations/24.3/activate
```
然后再使用 `make` 继续编译就可以了。
## 开源许可
详见 [LICENSE](./LICENSE)。

8
README-RU.md
View File

@ -28,13 +28,7 @@
#### Установка EMQX с помощью Docker
```
docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
```
Или запустите EMQX Enterprise со встроенной бессрочной лицензией на 10 соединений.
```
docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest
docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
```
Чтобы ознакомиться с функциональностью EMQX, пожалуйста, следуйте [руководству по началу работы](https://www.emqx.io/docs/en/v5.0/getting-started/getting-started.html#start-emqx).

6
README.md
View File

@ -33,12 +33,6 @@ The simplest way to set up EMQX is to create a managed deployment with EMQX Clou
docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx:latest
```
Or install EMQX Enterprise with a built-in license for ten connections that never expire.
```
docker run -d --name emqx-ee -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8084:8084 -p 8883:8883 -p 18083:18083 emqx/emqx-ee:latest
```
Next, please follow the [getting started guide](https://www.emqx.io/docs/en/v5.0/getting-started/getting-started.html#start-emqx) to tour the EMQX features.
#### Run EMQX cluster on kubernetes

2
apps/emqx/include/emqx_release.hrl
View File

@ -32,7 +32,7 @@
%% `apps/emqx/src/bpapi/README.md'
%% Community edition
-define(EMQX_RELEASE_CE, "5.0.7").
-define(EMQX_RELEASE_CE, "5.0.8").
%% Enterprise edition
-define(EMQX_RELEASE_EE, "5.0.0-alpha.1").

2
apps/emqx/rebar.config
View File

@ -27,7 +27,7 @@
{jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
{cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
{esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.4"}}},
{ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.5"}}},
{gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
{hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.30.0"}}},
{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},

2
apps/emqx/src/emqx.app.src
View File

@ -3,7 +3,7 @@
{id, "emqx"},
{description, "EMQX Core"},
% strict semver, bump manually!
{vsn, "5.0.8"},
{vsn, "5.0.9"},
{modules, []},
{registered, []},
{applications, [

73
apps/emqx/src/emqx_channel.erl
View File

@ -345,7 +345,8 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
fun check_connect/2,
fun enrich_client/2,
fun set_log_meta/2,
fun check_banned/2
fun check_banned/2,
fun count_flapping_event/2
],
ConnPkt,
Channel#channel{conn_state = connecting}
@ -354,12 +355,14 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
{ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} ->
?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
NChannel1 = NChannel#channel{
will_msg = emqx_packet:will_msg(NConnPkt),
alias_maximum = init_alias_maximum(NConnPkt, ClientInfo)
},
case authenticate(?CONNECT_PACKET(NConnPkt), NChannel1) of
{ok, Properties, NChannel2} ->
process_connect(Properties, NChannel2);
%% only store will_msg after successful authn
%% fix for: https://github.com/emqx/emqx/issues/8886
NChannel3 = NChannel2#channel{will_msg = emqx_packet:will_msg(NConnPkt)},
process_connect(Properties, NChannel3);
{continue, Properties, NChannel2} ->
handle_out(auth, {?RC_CONTINUE_AUTHENTICATION, Properties}, NChannel2);
{error, ReasonCode} ->
@ -995,8 +998,13 @@ maybe_nack(Delivers) ->
lists:filter(fun not_nacked/1, Delivers).
not_nacked({deliver, _Topic, Msg}) ->
not (emqx_shared_sub:is_ack_required(Msg) andalso
(ok == emqx_shared_sub:nack_no_connection(Msg))).
case emqx_shared_sub:is_ack_required(Msg) of
true ->
ok = emqx_shared_sub:nack_no_connection(Msg),
false;
false ->
true
end.
maybe_mark_as_delivered(Session, Delivers) ->
case emqx_session:info(is_persistent, Session) of
@ -1165,10 +1173,11 @@ handle_call(
Channel = #channel{
conn_state = ConnState,
will_msg = WillMsg,
clientinfo = ClientInfo,
conninfo = #{proto_ver := ProtoVer}
}
) ->
(WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
(WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg),
Channel1 =
case ConnState of
connected -> ensure_disconnected(kicked, Channel);
@ -1219,6 +1228,8 @@ handle_call(
ChanInfo1 = info(NChannel),
emqx_cm:set_chan_info(ClientId, ChanInfo1#{sockinfo => SockInfo}),
reply(ok, reset_timer(alive_timer, NChannel));
handle_call(get_mqueue, Channel) ->
reply({ok, get_mqueue(Channel)}, Channel);
handle_call(Req, Channel) ->
?SLOG(error, #{msg => "unexpected_call", call => Req}),
reply(ignored, Channel).
@ -1250,14 +1261,11 @@ handle_info(
{sock_closed, Reason},
Channel =
#channel{
conn_state = ConnState,
clientinfo = ClientInfo = #{zone := Zone}
conn_state = ConnState
}
) when
ConnState =:= connected orelse ConnState =:= reauthenticating
->
emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso
emqx_flapping:detect(ClientInfo),
Channel1 = ensure_disconnected(Reason, maybe_publish_will_msg(Channel)),
case maybe_shutdown(Reason, Channel1) of
{ok, Channel2} -> {ok, {event, disconnected}, Channel2};
@ -1359,8 +1367,10 @@ handle_timeout(
end;
handle_timeout(_TRef, expire_session, Channel) ->
shutdown(expired, Channel);
handle_timeout(_TRef, will_message, Channel = #channel{will_msg = WillMsg}) ->
(WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
handle_timeout(
_TRef, will_message, Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg}
) ->
(WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg),
{ok, clean_timer(will_timer, Channel#channel{will_msg = undefined})};
handle_timeout(
_TRef,
@ -1434,12 +1444,14 @@ terminate({shutdown, kicked}, Channel) ->
run_terminate_hook(kicked, Channel);
terminate({shutdown, Reason}, Channel) when
Reason =:= discarded;
Reason =:= takenover;
Reason =:= not_authorized
Reason =:= takenover
->
run_terminate_hook(Reason, Channel);
terminate(Reason, Channel = #channel{will_msg = WillMsg}) ->
(WillMsg =/= undefined) andalso publish_will_msg(WillMsg),
terminate(Reason, Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg}) ->
%% since will_msg is set to undefined as soon as it is published,
%% if will_msg still exists when the session is terminated, it
%% must be published immediately.
WillMsg =/= undefined andalso publish_will_msg(ClientInfo, WillMsg),
(Reason =:= expired) andalso persist_if_session(Channel),
run_terminate_hook(Reason, Channel).
@ -1622,6 +1634,14 @@ check_banned(_ConnPkt, #channel{clientinfo = ClientInfo}) ->
false -> ok
end.
%%--------------------------------------------------------------------
%% Flapping
count_flapping_event(_ConnPkt, Channel = #channel{clientinfo = ClientInfo = #{zone := Zone}}) ->
emqx_config:get_zone_conf(Zone, [flapping_detect, enable]) andalso
emqx_flapping:detect(ClientInfo),
{ok, Channel}.
%%--------------------------------------------------------------------
%% Authenticate
@ -2098,10 +2118,10 @@ ensure_disconnected(
maybe_publish_will_msg(Channel = #channel{will_msg = undefined}) ->
Channel;
maybe_publish_will_msg(Channel = #channel{will_msg = WillMsg}) ->
maybe_publish_will_msg(Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg}) ->
case will_delay_interval(WillMsg) of
0 ->
ok = publish_will_msg(WillMsg),
ok = publish_will_msg(ClientInfo, WillMsg),
Channel#channel{will_msg = undefined};
I ->
ensure_timer(will_timer, timer:seconds(I), Channel)
@ -2114,9 +2134,19 @@ will_delay_interval(WillMsg) ->
0
).
publish_will_msg(Msg) ->
publish_will_msg(ClientInfo, Msg = #message{topic = Topic}) ->
case emqx_access_control:authorize(ClientInfo, publish, Topic) of
allow ->
_ = emqx_broker:publish(Msg),
ok.
ok;
deny ->
?tp(
warning,
last_will_testament_publish_denied,
#{topic => Topic}
),
ok
end.
%%--------------------------------------------------------------------
%% Disconnect Reason
@ -2207,3 +2237,6 @@ get_mqtt_conf(Zone, Key, Default) ->
set_field(Name, Value, Channel) ->
Pos = emqx_misc:index_of(Name, record_info(fields, channel)),
setelement(Pos + 1, Channel, Value).
get_mqueue(#channel{session = Session}) ->
emqx_session:get_mqueue(Session).

2
apps/emqx/src/emqx_config.erl
View File

@ -476,7 +476,7 @@ read_override_conf(#{} = Opts) ->
override_conf_file(Opts) when is_map(Opts) ->
Key =
case maps:get(override_to, Opts, local) of
case maps:get(override_to, Opts, cluster) of
local -> local_override_conf_file;
cluster -> cluster_override_conf_file
end,

120
apps/emqx/src/emqx_config_handler.erl
View File

@ -43,6 +43,7 @@
terminate/2,
code_change/3
]).
-export([is_mutable/3]).
-define(MOD, {mod}).
-define(WKEY, '?').
@ -229,15 +230,26 @@ process_update_request([_], _Handlers, {remove, _Opts}) ->
process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) ->
OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
BinKeyPath = bin_path(ConfKeyPath),
case check_permissions(remove, BinKeyPath, OldRawConf, Opts) of
allow ->
NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf),
OverrideConf = remove_from_override_config(BinKeyPath, Opts),
{ok, NewRawConf, OverrideConf, Opts};
{deny, Reason} ->
{error, {permission_denied, Reason}}
end;
process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) ->
OldRawConf = emqx_config:get_root_raw(ConfKeyPath),
case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of
{ok, NewRawConf} ->
BinKeyPath = bin_path(ConfKeyPath),
case check_permissions(update, BinKeyPath, NewRawConf, Opts) of
allow ->
OverrideConf = update_override_config(NewRawConf, Opts),
{ok, NewRawConf, OverrideConf, Opts};
{deny, Reason} ->
{error, {permission_denied, Reason}}
end;
Error ->
Error
end.
@ -272,12 +284,11 @@ check_and_save_configs(
UpdateArgs,
Opts
) ->
OldConf = emqx_config:get_root(ConfKeyPath),
Schema = schema(SchemaModule, ConfKeyPath),
{AppEnvs, NewConf} = emqx_config:check_config(Schema, NewRawConf),
OldConf = emqx_config:get_root(ConfKeyPath),
case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, AppEnvs, UpdateArgs, #{}) of
{ok, Result0} ->
remove_from_local_if_cluster_change(ConfKeyPath, Opts),
ok = emqx_config:save_configs(AppEnvs, NewConf, NewRawConf, OverrideConf, Opts),
Result1 = return_change_result(ConfKeyPath, UpdateArgs),
{ok, Result1#{post_config_update => Result0}};
@ -430,16 +441,6 @@ merge_to_old_config(UpdateReq, RawConf) when is_map(UpdateReq), is_map(RawConf)
merge_to_old_config(UpdateReq, _RawConf) ->
{ok, UpdateReq}.
%% local-override.conf priority is higher than cluster-override.conf
%% If we want cluster to take effect, we must remove the local.
remove_from_local_if_cluster_change(BinKeyPath, #{override_to := cluster} = Opts) ->
Opts1 = Opts#{override_to => local},
Local = remove_from_override_config(BinKeyPath, Opts1),
_ = emqx_config:save_to_override_conf(Local, Opts1),
ok;
remove_from_local_if_cluster_change(_BinKeyPath, _Opts) ->
ok.
remove_from_override_config(_BinKeyPath, #{persistent := false}) ->
undefined;
remove_from_override_config(BinKeyPath, Opts) ->
@ -544,3 +545,98 @@ load_prev_handlers() ->
save_handlers(Handlers) ->
application:set_env(emqx, ?MODULE, Handlers).
check_permissions(_Action, _ConfKeyPath, _NewRawConf, #{override_to := local}) ->
allow;
check_permissions(Action, ConfKeyPath, NewRawConf, _Opts) ->
case emqx_map_lib:deep_find(ConfKeyPath, NewRawConf) of
{ok, NewRaw} ->
LocalOverride = emqx_config:read_override_conf(#{override_to => local}),
case emqx_map_lib:deep_find(ConfKeyPath, LocalOverride) of
{ok, LocalRaw} ->
case is_mutable(Action, NewRaw, LocalRaw) of
ok ->
allow;
{error, Error} ->
?SLOG(error, #{
msg => "prevent_remove_local_override_conf",
config_key_path => ConfKeyPath,
error => Error
}),
{deny, "Disable changed from local-override.conf"}
end;
{not_found, _, _} ->
allow
end;
{not_found, _, _} ->
allow
end.
is_mutable(Action, NewRaw, LocalRaw) ->
try
KeyPath = [],
is_mutable(KeyPath, Action, NewRaw, LocalRaw)
catch
throw:Error -> Error
end.
-define(REMOVE_FAILED, "remove_failed").
-define(UPDATE_FAILED, "update_failed").
is_mutable(KeyPath, Action, New = #{}, Local = #{}) ->
maps:foreach(
fun(Key, SubLocal) ->
case maps:find(Key, New) of
error -> ok;
{ok, SubNew} -> is_mutable(KeyPath ++ [Key], Action, SubNew, SubLocal)
end
end,
Local
);
is_mutable(KeyPath, remove, Update, Origin) ->
throw({error, {?REMOVE_FAILED, KeyPath, Update, Origin}});
is_mutable(_KeyPath, update, Val, Val) ->
ok;
is_mutable(KeyPath, update, Update, Origin) ->
throw({error, {?UPDATE_FAILED, KeyPath, Update, Origin}}).
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
is_mutable_update_test() ->
Action = update,
?assertEqual(ok, is_mutable(Action, #{}, #{})),
?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a => #{b => #{c => #{}}}})),
?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})),
?assertEqual(
{error, {?UPDATE_FAILED, [a, b, c], 1, 2}},
is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}})
),
?assertEqual(
{error, {?UPDATE_FAILED, [a, b, d], 2, 3}},
is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}})
),
ok.
is_mutable_remove_test() ->
Action = remove,
?assertEqual(ok, is_mutable(Action, #{}, #{})),
?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => #{}}}}, #{a1 => #{b => #{c => #{}}}})),
?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b1 => #{c => 1}}})),
?assertEqual(ok, is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c1 => 1}}})),
?assertEqual(
{error, {?REMOVE_FAILED, [a, b, c], 1, 1}},
is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 1}}})
),
?assertEqual(
{error, {?REMOVE_FAILED, [a, b, c], 1, 2}},
is_mutable(Action, #{a => #{b => #{c => 1}}}, #{a => #{b => #{c => 2}}})
),
?assertEqual(
{error, {?REMOVE_FAILED, [a, b, c], 1, 1}},
is_mutable(Action, #{a => #{b => #{c => 1, d => 2}}}, #{a => #{b => #{c => 1, d => 3}}})
),
ok.
-endif.

24
apps/emqx/src/emqx_listeners.erl
View File

@ -87,12 +87,18 @@ format_list(Listener) ->
].
do_list_raw() ->
%% GET /listeners from other nodes returns [] when init config is not loaded.
case emqx_app:get_init_config_load_done() of
true ->
Key = <<"listeners">>,
Raw = emqx_config:get_raw([Key], #{}),
SchemaMod = emqx_config:get_schema_mod(Key),
#{Key := RawWithDefault} = emqx_config:fill_defaults(SchemaMod, #{Key => Raw}, #{}),
Listeners = maps:to_list(RawWithDefault),
lists:flatmap(fun format_raw_listeners/1, Listeners).
lists:flatmap(fun format_raw_listeners/1, Listeners);
false ->
[]
end.
format_raw_listeners({Type0, Conf}) ->
Type = binary_to_atom(Type0),
@ -515,12 +521,16 @@ merge_default(Options) ->
integer() | {tuple(), integer()} | string() | binary()
) -> io_lib:chars().
format_bind(Port) when is_integer(Port) ->
%% **Note**:
%% 'For TCP, UDP and IP networks, if the host is empty or a literal
%% unspecified IP address, as in ":80", "0.0.0.0:80" or "[::]:80" for
%% TCP and UDP, "", "0.0.0.0" or "::" for IP, the local system is
%% assumed.'
%%
%% Quoted from: https://pkg.go.dev/net
%% Decided to use this format to display the bind for all interfaces and
%% IPv4/IPv6 support
io_lib:format(":~w", [Port]);
%% Print only the port number when bound on all interfaces
format_bind({{0, 0, 0, 0}, Port}) ->
format_bind(Port);
format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, Port}) ->
format_bind(Port);
format_bind({Addr, Port}) when is_list(Addr) ->
io_lib:format("~ts:~w", [Addr, Port]);
format_bind({Addr, Port}) when is_tuple(Addr), tuple_size(Addr) == 4 ->
@ -532,6 +542,8 @@ format_bind(Str) when is_list(Str) ->
case emqx_schema:to_ip_port(Str) of
{ok, {Ip, Port}} ->
format_bind({Ip, Port});
{ok, Port} ->
format_bind(Port);
{error, _} ->
format_bind(list_to_integer(Str))
end;

22
apps/emqx/src/emqx_mqueue.erl
View File

@ -66,7 +66,8 @@
in/2,
out/1,
stats/1,
dropped/1
dropped/1,
to_list/1
]).
-define(NO_PRIORITY_TABLE, disabled).
@ -109,7 +110,7 @@
dropped = 0 :: count(),
p_table = ?NO_PRIORITY_TABLE :: p_table(),
default_p = ?LOWEST_PRIORITY :: priority(),
q = ?PQUEUE:new() :: pq(),
q = emqx_pqueue:new() :: pq(),
shift_opts :: #shift_opts{},
last_prio :: non_neg_integer() | undefined,
p_credit :: non_neg_integer() | undefined
@ -118,7 +119,7 @@
-type mqueue() :: #mqueue{}.
-spec init(options()) -> mqueue().
init(Opts = #{max_len := MaxLen0, store_qos0 := QoS_0}) ->
init(Opts = #{max_len := MaxLen0, store_qos0 := Qos0}) ->
MaxLen =
case (is_integer(MaxLen0) andalso MaxLen0 > ?MAX_LEN_INFINITY) of
true -> MaxLen0;
@ -126,7 +127,7 @@ init(Opts = #{max_len := MaxLen0, store_qos0 := QoS_0}) ->
end,
#mqueue{
max_len = MaxLen,
store_qos0 = QoS_0,
store_qos0 = Qos0,
p_table = get_opt(priorities, Opts, ?NO_PRIORITY_TABLE),
default_p = get_priority_opt(Opts),
shift_opts = get_shift_opt(Opts)
@ -152,6 +153,19 @@ len(#mqueue{len = Len}) -> Len.
max_len(#mqueue{max_len = MaxLen}) -> MaxLen.
%% @doc Return all queued items in a list.
-spec to_list(mqueue()) -> list().
to_list(MQ) ->
to_list(MQ, []).
to_list(MQ, Acc) ->
case out(MQ) of
{empty, _MQ} ->
lists:reverse(Acc);
{{value, Msg}, Q1} ->
to_list(Q1, [Msg | Acc])
end.
%% @doc Return number of dropped messages.
-spec dropped(mqueue()) -> count().
dropped(#mqueue{dropped = Dropped}) -> Dropped.

65
apps/emqx/src/emqx_schema.erl
View File

@ -39,7 +39,8 @@
-type comma_separated_binary() :: [binary()].
-type comma_separated_atoms() :: [atom()].
-type bar_separated_list() :: list().
-type ip_port() :: tuple().
-type ip_port() :: tuple() | integer().
-type host_port() :: tuple().
-type cipher() :: map().
-typerefl_from_string({duration/0, emqx_schema, to_duration}).
@ -52,6 +53,7 @@
-typerefl_from_string({comma_separated_binary/0, emqx_schema, to_comma_separated_binary}).
-typerefl_from_string({bar_separated_list/0, emqx_schema, to_bar_separated_list}).
-typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}).
-typerefl_from_string({host_port/0, emqx_schema, to_host_port}).
-typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}).
-typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}).
@ -78,6 +80,7 @@
to_comma_separated_binary/1,
to_bar_separated_list/1,
to_ip_port/1,
to_host_port/1,
to_erl_cipher_suite/1,
to_comma_separated_atoms/1
]).
@ -96,6 +99,7 @@
comma_separated_binary/0,
bar_separated_list/0,
ip_port/0,
host_port/0,
cipher/0,
comma_separated_atoms/0
]).
@ -2167,33 +2171,60 @@ to_bar_separated_list(Str) ->
%% - :1883
%% - :::1883
to_ip_port(Str) ->
case split_ip_port(Str) of
{"", Port} ->
{ok, {{0, 0, 0, 0}, list_to_integer(Port)}};
{Ip, Port} ->
to_host_port(Str, ip_addr).
%% @doc support the following format:
%% - 127.0.0.1:1883
%% - ::1:1883
%% - [::1]:1883
%% - :1883
%% - :::1883
%% - example.com:80
to_host_port(Str) ->
to_host_port(Str, hostname).
%% - example.com:80
to_host_port(Str, IpOrHost) ->
case split_host_port(Str) of
{"", Port} when IpOrHost =:= ip_addr ->
%% this is a local address
{ok, list_to_integer(Port)};
{"", _Port} ->
%% must specify host part when it's a remote endpoint
{error, bad_host_port};
{MaybeIp, Port} ->
PortVal = list_to_integer(Port),
case inet:parse_address(Ip) of
{ok, R} ->
{ok, {R, PortVal}};
_ ->
case inet:parse_address(MaybeIp) of
{ok, IpTuple} ->
{ok, {IpTuple, PortVal}};
_ when IpOrHost =:= hostname ->
%% check is a rfc1035's hostname
case inet_parse:domain(Ip) of
case inet_parse:domain(MaybeIp) of
true ->
{ok, {Ip, PortVal}};
{ok, {MaybeIp, PortVal}};
_ ->
{error, Str}
end
{error, bad_hostname}
end;
_ ->
{error, Str}
{error, bad_ip_port}
end;
_ ->
{error, bad_ip_port}
end.
split_ip_port(Str0) ->
split_host_port(Str0) ->
Str = re:replace(Str0, " ", "", [{return, list}, global]),
case lists:split(string:rchr(Str, $:), Str) of
%% no port
%% no colon
{[], Str} ->
error;
try
%% if it's just a port number, then return as-is
_ = list_to_integer(Str),
{"", Str}
catch
_:_ ->
error
end;
{IpPlusColon, PortString} ->
IpStr0 = lists:droplast(IpPlusColon),
case IpStr0 of

42
apps/emqx/src/emqx_session.erl
View File

@ -60,7 +60,8 @@
info/2,
is_session/1,
stats/1,
obtain_next_pkt_id/1
obtain_next_pkt_id/1,
get_mqueue/1
]).
-export([
@ -801,6 +802,7 @@ replay(ClientInfo, Session = #session{inflight = Inflight}) ->
-spec terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok.
terminate(ClientInfo, Reason, Session) ->
run_terminate_hooks(ClientInfo, Reason, Session),
Reason =/= takenover andalso
redispatch_shared_messages(Session),
ok.
@ -811,29 +813,20 @@ run_terminate_hooks(ClientInfo, takenover, Session) ->
run_terminate_hooks(ClientInfo, Reason, Session) ->
run_hook('session.terminated', [ClientInfo, Reason, info(Session)]).
redispatch_shared_messages(#session{inflight = Inflight}) ->
InflightList = emqx_inflight:to_list(Inflight),
lists:foreach(
fun
%% Only QoS1 messages get redispatched, because QoS2 messages
%% must be sent to the same client, once they're in flight
({_, #inflight_data{message = #message{qos = ?QOS_2} = Msg}}) ->
?SLOG(warning, #{msg => qos2_lost_no_redispatch}, #{message => Msg});
({_, #inflight_data{message = #message{topic = Topic, qos = ?QOS_1} = Msg}}) ->
case emqx_shared_sub:get_group(Msg) of
{ok, Group} ->
%% Note that dispatch is called with self() in failed subs
%% This is done to avoid dispatching back to caller
Delivery = #delivery{sender = self(), message = Msg},
emqx_shared_sub:dispatch(Group, Topic, Delivery, [self()]);
_ ->
redispatch_shared_messages(#session{inflight = Inflight, mqueue = Q}) ->
AllInflights = emqx_inflight:to_list(fun sort_fun/2, Inflight),
F = fun
({_PacketId, #inflight_data{message = #message{qos = ?QOS_1} = Msg}}) ->
%% For QoS 2, here is what the spec says:
%% If the Client's Session terminates before the Client reconnects,
%% the Server MUST NOT send the Application Message to any other
%% subscribed Client [MQTT-4.8.2-5].
{true, Msg};
({_PacketId, #inflight_data{}}) ->
false
end;
(_) ->
ok
end,
InflightList
).
InflightList = lists:filtermap(F, AllInflights),
emqx_shared_sub:redispatch(InflightList ++ emqx_mqueue:to_list(Q)).
-compile({inline, [run_hook/2]}).
run_hook(Name, Args) ->
@ -892,7 +885,7 @@ on_delivery_completed(
).
mark_begin_deliver(Msg) ->
emqx_message:set_header(deliver_begin_at, erlang:system_time(second), Msg).
emqx_message:set_header(deliver_begin_at, erlang:system_time(millisecond), Msg).
%%--------------------------------------------------------------------
%% Helper functions
@ -925,3 +918,6 @@ age(Now, Ts) -> Now - Ts.
set_field(Name, Value, Session) ->
Pos = emqx_misc:index_of(Name, record_info(fields, session)),
setelement(Pos + 1, Session, Value).
get_mqueue(#session{mqueue = Q}) ->
emqx_mqueue:to_list(Q).

63
apps/emqx/src/emqx_shared_sub.erl
View File

@ -39,15 +39,15 @@
-export([
dispatch/3,
dispatch/4,
do_dispatch_with_ack/4
do_dispatch_with_ack/4,
redispatch/1
]).
-export([
maybe_ack/1,
maybe_nack_dropped/1,
nack_no_connection/1,
is_ack_required/1,
get_group/1
is_ack_required/1
]).
%% for testing
@ -96,6 +96,9 @@
-define(ACK, shared_sub_ack).
-define(NACK(Reason), {shared_sub_nack, Reason}).
-define(NO_ACK, no_ack).
-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
-type redispatch_to() :: ?REDISPATCH_TO(emqx_topic:group(), emqx_topic:topic()).
-record(state, {pmon}).
@ -144,7 +147,8 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) ->
false ->
{error, no_subscribers};
{Type, SubPid} ->
case do_dispatch(SubPid, Group, Topic, Msg, Type) of
Msg1 = with_redispatch_to(Msg, Group, Topic),
case do_dispatch(SubPid, Group, Topic, Msg1, Type) of
ok ->
{ok, 1};
{error, _Reason} ->
@ -223,16 +227,53 @@ without_group_ack(Msg) ->
get_group_ack(Msg) ->
emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK).
with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) ->
Msg;
with_redispatch_to(Msg, Group, Topic) ->
emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg).
%% @hidden Redispatch is neede only for the messages with redispatch_to header added.
is_redispatch_needed(#message{} = Msg) ->
case get_redispatch_to(Msg) of
?REDISPATCH_TO(_, _) ->
true;
_ ->
false
end.
%% @doc Redispatch shared deliveries to other members in the group.
redispatch(Messages0) ->
Messages = lists:filter(fun is_redispatch_needed/1, Messages0),
case length(Messages) of
L when L > 0 ->
?SLOG(info, #{
msg => "redispatching_shared_subscription_message",
count => L
}),
lists:foreach(fun redispatch_shared_message/1, Messages);
_ ->
ok
end.
redispatch_shared_message(#message{} = Msg) ->
%% As long as it's still a #message{} record in inflight,
%% we should try to re-dispatch
?REDISPATCH_TO(Group, Topic) = get_redispatch_to(Msg),
%% Note that dispatch is called with self() in failed subs
%% This is done to avoid dispatching back to caller
Delivery = #delivery{sender = self(), message = Msg},
dispatch(Group, Topic, Delivery, [self()]).
%% @hidden Return the `redispatch_to` group-topic in the message header.
%% `false` is returned if the message is not a shared dispatch.
%% or when it's a QoS 0 message.
-spec get_redispatch_to(emqx_types:message()) -> redispatch_to() | false.
get_redispatch_to(Msg) ->
emqx_message:get_header(redispatch_to, Msg, false).
-spec is_ack_required(emqx_types:message()) -> boolean().
is_ack_required(Msg) -> ?NO_ACK =/= get_group_ack(Msg).
-spec get_group(emqx_types:message()) -> {ok, any()} | error.
get_group(Msg) ->
case get_group_ack(Msg) of
?NO_ACK -> error;
{Group, _Sender, _Ref} -> {ok, Group}
end.
%% @doc Negative ack dropped message due to inflight window or message queue being full.
-spec maybe_nack_dropped(emqx_types:message()) -> boolean().
maybe_nack_dropped(Msg) ->

45
apps/emqx/test/emqx_channel_SUITE.erl
View File

@ -207,14 +207,6 @@ init_per_suite(Config) ->
ok = meck:new(emqx_cm, [passthrough, no_history, no_link]),
ok = meck:expect(emqx_cm, mark_channel_connected, fun(_) -> ok end),
ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end),
%% Access Control Meck
ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
ok = meck:expect(
emqx_access_control,
authenticate,
fun(_) -> {ok, #{is_superuser => false}} end
),
ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end),
%% Broker Meck
ok = meck:new(emqx_broker, [passthrough, no_history, no_link]),
%% Hooks Meck
@ -234,7 +226,6 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
meck:unload([
emqx_access_control,
emqx_metrics,
emqx_session,
emqx_broker,
@ -244,11 +235,21 @@ end_per_suite(_Config) ->
]).
init_per_testcase(_TestCase, Config) ->
%% Access Control Meck
ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
ok = meck:expect(
emqx_access_control,
authenticate,
fun(_) -> {ok, #{is_superuser => false}} end
),
ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end),
%% Set confs
OldConf = set_test_listener_confs(),
emqx_common_test_helpers:start_apps([]),
[{config, OldConf} | Config].
end_per_testcase(_TestCase, Config) ->
meck:unload([emqx_access_control]),
emqx_config:put(?config(config, Config)),
emqx_common_test_helpers:stop_apps([]),
Config.
@ -1115,6 +1116,32 @@ t_ws_cookie_init(_) ->
),
?assertMatch(#{ws_cookie := WsCookie}, emqx_channel:info(clientinfo, Channel)).
%%--------------------------------------------------------------------
%% Test cases for other mechnisms
%%--------------------------------------------------------------------
t_flapping_detect(_) ->
emqx_config:put_zone_conf(default, [flapping_detect, enable], true),
Parent = self(),
ok = meck:expect(
emqx_cm,
open_session,
fun(true, _ClientInfo, _ConnInfo) ->
{ok, #{session => session(), present => false}}
end
),
ok = meck:expect(emqx_access_control, authenticate, fun(_) -> {error, not_authorized} end),
ok = meck:expect(emqx_flapping, detect, fun(_) -> Parent ! flapping_detect end),
IdleChannel = channel(#{conn_state => idle}),
{shutdown, not_authorized, _ConnAck, _Channel} =
emqx_channel:handle_in(?CONNECT_PACKET(connpkt()), IdleChannel),
receive
flapping_detect -> ok
after 2000 ->
?assert(false, "Flapping detect should be exected in connecting progress")
end,
meck:unload([emqx_flapping]).
%%--------------------------------------------------------------------
%% Helper functions
%%--------------------------------------------------------------------

4
apps/emqx/test/emqx_common_test_helpers.erl
View File

@ -632,9 +632,9 @@ setup_node(Node, Opts) when is_map(Opts) ->
%% Here we start the apps
EnvHandlerForRpc =
fun(App) ->
%% We load configuration, and than set the special enviroment variable
%% We load configuration, and than set the special environment variable
%% which says that emqx shouldn't load configuration at startup
%% Otherwise, configuration get's loaded and all preset env in envhandler is lost
%% Otherwise, configuration gets loaded and all preset env in EnvHandler is lost
LoadSchema andalso
begin
emqx_config:init_load(SchemaMod),

66
apps/emqx/test/emqx_config_handler_SUITE.erl
View File

@ -21,6 +21,8 @@
-define(MOD, {mod}).
-define(WKEY, '?').
-define(LOCAL_CONF, "/tmp/local-override.conf").
-define(CLUSTER_CONF, "/tmp/cluster-override.conf").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
@ -36,6 +38,8 @@ end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]).
init_per_testcase(_Case, Config) ->
_ = file:delete(?LOCAL_CONF),
_ = file:delete(?CLUSTER_CONF),
Config.
end_per_testcase(_Case, _Config) ->
@ -196,6 +200,62 @@ t_sub_key_update_remove(_Config) ->
ok = emqx_config_handler:remove_handler(KeyPath2),
ok.
t_local_override_update_remove(_Config) ->
application:set_env(emqx, local_override_conf_file, ?LOCAL_CONF),
application:set_env(emqx, cluster_override_conf_file, ?CLUSTER_CONF),
KeyPath = [sysmon, os, cpu_high_watermark],
ok = emqx_config_handler:add_handler(KeyPath, ?MODULE),
LocalOpts = #{override_to => local},
{ok, Res} = emqx:update_config(KeyPath, <<"70%">>, LocalOpts),
?assertMatch(
#{
config := 0.7,
post_config_update := #{},
raw_config := <<"70%">>
},
Res
),
ClusterOpts = #{override_to => cluster},
?assertMatch(
{error, {permission_denied, _}}, emqx:update_config(KeyPath, <<"71%">>, ClusterOpts)
),
?assertMatch(0.7, emqx:get_config(KeyPath)),
KeyPath2 = [sysmon, os, cpu_low_watermark],
ok = emqx_config_handler:add_handler(KeyPath2, ?MODULE),
?assertMatch(
{error, {permission_denied, _}}, emqx:update_config(KeyPath2, <<"40%">>, ClusterOpts)
),
%% remove
?assertMatch({error, {permission_denied, _}}, emqx:remove_config(KeyPath)),
?assertEqual(
{ok, #{post_config_update => #{}}},
emqx:remove_config(KeyPath, #{override_to => local})
),
?assertEqual(
{ok, #{post_config_update => #{}}},
emqx:remove_config(KeyPath)
),
?assertError({config_not_found, KeyPath}, emqx:get_raw_config(KeyPath)),
OSKey = maps:keys(emqx:get_raw_config([sysmon, os])),
?assertEqual(false, lists:member(<<"cpu_high_watermark">>, OSKey)),
?assert(length(OSKey) > 0),
?assertEqual(
{ok, #{config => 0.8, post_config_update => #{}, raw_config => <<"80%">>}},
emqx:reset_config(KeyPath, ClusterOpts)
),
OSKey1 = maps:keys(emqx:get_raw_config([sysmon, os])),
?assertEqual(true, lists:member(<<"cpu_high_watermark">>, OSKey1)),
?assert(length(OSKey1) > 1),
ok = emqx_config_handler:remove_handler(KeyPath),
ok = emqx_config_handler:remove_handler(KeyPath2),
application:unset_env(emqx, local_override_conf_file),
application:unset_env(emqx, cluster_override_conf_file),
ok.
t_check_failed(_Config) ->
KeyPath = [sysmon, os, cpu_check_interval],
Opts = #{rawconf_with_defaults => true},
@ -219,7 +279,7 @@ t_stop(_Config) ->
ok.
t_callback_crash(_Config) ->
CrashPath = [sysmon, os, cpu_high_watermark],
CrashPath = [sysmon, os, procmem_high_watermark],
Opts = #{rawconf_with_defaults => true},
ok = emqx_config_handler:add_handler(CrashPath, ?MODULE),
Old = emqx:get_raw_config(CrashPath),
@ -334,6 +394,8 @@ pre_config_update([sysmon, os, cpu_check_interval], UpdateReq, _RawConf) ->
{ok, UpdateReq};
pre_config_update([sysmon, os, cpu_low_watermark], UpdateReq, _RawConf) ->
{ok, UpdateReq};
pre_config_update([sysmon, os, cpu_high_watermark], UpdateReq, _RawConf) ->
{ok, UpdateReq};
pre_config_update([sysmon, os, sysmem_high_watermark], UpdateReq, _RawConf) ->
{ok, UpdateReq};
pre_config_update([sysmon, os, mem_check_interval], _UpdateReq, _RawConf) ->
@ -347,6 +409,8 @@ post_config_update([sysmon, os, cpu_check_interval], _UpdateReq, _NewConf, _OldC
{ok, ok};
post_config_update([sysmon, os, cpu_low_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs) ->
ok;
post_config_update([sysmon, os, cpu_high_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs) ->
ok;
post_config_update([sysmon, os, sysmem_high_watermark], _UpdateReq, _NewConf, _OldConf, _AppEnvs) ->
{error, post_config_update_error}.

26
apps/emqx/test/emqx_listeners_SUITE.erl
View File

@ -148,6 +148,32 @@ t_wss_conn(_) ->
{ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000),
ok = ssl:close(Socket).
t_format_bind(_) ->
?assertEqual(
":1883",
lists:flatten(emqx_listeners:format_bind(1883))
),
?assertEqual(
"0.0.0.0:1883",
lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0}, 1883}))
),
?assertEqual(
"[::]:1883",
lists:flatten(emqx_listeners:format_bind({{0, 0, 0, 0, 0, 0, 0, 0}, 1883}))
),
?assertEqual(
"127.0.0.1:1883",
lists:flatten(emqx_listeners:format_bind({{127, 0, 0, 1}, 1883}))
),
?assertEqual(
":1883",
lists:flatten(emqx_listeners:format_bind("1883"))
),
?assertEqual(
":1883",
lists:flatten(emqx_listeners:format_bind(":1883"))
).
render_config_file() ->
Path = local_path(["etc", "emqx.conf"]),
{ok, Temp} = file:read_file(Path),

27
apps/emqx/test/emqx_schema_tests.erl
View File

@ -175,3 +175,30 @@ ssl_opts_gc_after_handshake_test_not_rancher_listener_test() ->
Checked
),
ok.
to_ip_port_test_() ->
Ip = fun emqx_schema:to_ip_port/1,
Host = fun(Str) ->
case Ip(Str) of
{ok, {_, _} = Res} ->
%% assert
{ok, Res} = emqx_schema:to_host_port(Str);
_ ->
emqx_schema:to_host_port(Str)
end
end,
[
?_assertEqual({ok, 80}, Ip("80")),
?_assertEqual({error, bad_host_port}, Host("80")),
?_assertEqual({ok, 80}, Ip(":80")),
?_assertEqual({error, bad_host_port}, Host(":80")),
?_assertEqual({error, bad_ip_port}, Ip("localhost:80")),
?_assertEqual({ok, {"localhost", 80}}, Host("localhost:80")),
?_assertEqual({ok, {"example.com", 80}}, Host("example.com:80")),
?_assertEqual({ok, {{127, 0, 0, 1}, 80}}, Ip("127.0.0.1:80")),
?_assertEqual({error, bad_ip_port}, Ip("$:1900")),
?_assertEqual({error, bad_hostname}, Host("$:1900")),
?_assertMatch({ok, {_, 1883}}, Ip("[::1]:1883")),
?_assertMatch({ok, {_, 1883}}, Ip("::1:1883")),
?_assertMatch({ok, {_, 1883}}, Ip(":::1883"))
].

283
apps/emqx/test/emqx_shared_sub_SUITE.erl
View File

@ -25,10 +25,20 @@
-define(SUITE, ?MODULE).
-define(wait(For, Timeout),
emqx_common_test_helpers:wait_for(
?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout
)
-define(WAIT(TIMEOUT, PATTERN, Res),
(fun() ->
receive
PATTERN ->
Res;
Other ->
ct:fail(#{
expected => ??PATTERN,
got => Other
})
after TIMEOUT ->
ct:fail({timeout, ??PATTERN})
end
end)()
).
-define(ack, shared_sub_ack).
@ -45,10 +55,26 @@ init_per_suite(Config) ->
end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([]).
t_is_ack_required(_) ->
init_per_testcase(Case, Config) ->
try
?MODULE:Case({'init', Config})
catch
error:function_clause ->
Config
end.
end_per_testcase(Case, Config) ->
try
?MODULE:Case({'end', Config})
catch
error:function_clause ->
ok
end.
t_is_ack_required(Config) when is_list(Config) ->
?assertEqual(false, emqx_shared_sub:is_ack_required(#message{headers = #{}})).
t_maybe_nack_dropped(_) ->
t_maybe_nack_dropped(Config) when is_list(Config) ->
?assertEqual(false, emqx_shared_sub:maybe_nack_dropped(#message{headers = #{}})),
Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
?assertEqual(true, emqx_shared_sub:maybe_nack_dropped(Msg)),
@ -60,7 +86,7 @@ t_maybe_nack_dropped(_) ->
end
).
t_nack_no_connection(_) ->
t_nack_no_connection(Config) when is_list(Config) ->
Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
?assertEqual(ok, emqx_shared_sub:nack_no_connection(Msg)),
?assertEqual(
@ -71,7 +97,7 @@ t_nack_no_connection(_) ->
end
).
t_maybe_ack(_) ->
t_maybe_ack(Config) when is_list(Config) ->
?assertEqual(#message{headers = #{}}, emqx_shared_sub:maybe_ack(#message{headers = #{}})),
Msg = #message{headers = #{shared_dispatch_ack => {<<"group">>, self(), for_test}}},
?assertEqual(
@ -86,10 +112,7 @@ t_maybe_ack(_) ->
end
).
% t_subscribers(_) ->
% error('TODO').
t_random_basic(_) ->
t_random_basic(Config) when is_list(Config) ->
ok = ensure_config(random),
ClientId = <<"ClientId">>,
Topic = <<"foo">>,
@ -121,7 +144,7 @@ t_random_basic(_) ->
%% After the connection for the 2nd session is also closed,
%% i.e. when all clients are offline, the following message(s)
%% should be delivered randomly.
t_no_connection_nack(_) ->
t_no_connection_nack(Config) when is_list(Config) ->
ok = ensure_config(sticky),
Publisher = <<"publisher">>,
Subscriber1 = <<"Subscriber1">>,
@ -153,54 +176,22 @@ t_no_connection_nack(_) ->
%% This is the connection which was picked by broker to dispatch (sticky) for 1st message
?assertMatch([#{packet_id := 1}], recv_msgs(1)),
%% Now kill the connection, expect all following messages to be delivered to the other
%% subscriber.
%emqx_mock_client:stop(ConnPid),
%% sleep then make synced calls to session processes to ensure that
%% the connection pid's 'EXIT' message is propagated to the session process
%% also to be sure sessions are still alive
% timer:sleep(2),
% _ = emqx_session:info(SPid1),
% _ = emqx_session:info(SPid2),
% %% Now we know what is the other still alive connection
% [TheOtherConnPid] = [SubConnPid1, SubConnPid2] -- [ConnPid],
% %% Send some more messages
% PacketIdList = lists:seq(2, 10),
% lists:foreach(fun(Id) ->
% SendF(Id),
% ?wait(Received(Id, TheOtherConnPid), 1000)
% end, PacketIdList),
% %% Now close the 2nd (last connection)
% emqx_mock_client:stop(TheOtherConnPid),
% timer:sleep(2),
% %% both sessions should have conn_pid = undefined
% ?assertEqual({conn_pid, undefined}, lists:keyfind(conn_pid, 1, emqx_session:info(SPid1))),
% ?assertEqual({conn_pid, undefined}, lists:keyfind(conn_pid, 1, emqx_session:info(SPid2))),
% %% send more messages, but all should be queued in session state
% lists:foreach(fun(Id) -> SendF(Id) end, PacketIdList),
% {_, L1} = lists:keyfind(mqueue_len, 1, emqx_session:info(SPid1)),
% {_, L2} = lists:keyfind(mqueue_len, 1, emqx_session:info(SPid2)),
% ?assertEqual(length(PacketIdList), L1 + L2),
% %% clean up
% emqx_mock_client:close_session(PubConnPid),
% emqx_sm:close_session(SPid1),
% emqx_sm:close_session(SPid2),
ok.
t_random(_) ->
t_random(Config) when is_list(Config) ->
ok = ensure_config(random, true),
test_two_messages(random).
t_round_robin(_) ->
t_round_robin(Config) when is_list(Config) ->
ok = ensure_config(round_robin, true),
test_two_messages(round_robin).
t_round_robin_per_group(_) ->
t_round_robin_per_group(Config) when is_list(Config) ->
ok = ensure_config(round_robin_per_group, true),
test_two_messages(round_robin_per_group).
%% this would fail if executed with the standard round_robin strategy
t_round_robin_per_group_even_distribution_one_group(_) ->
t_round_robin_per_group_even_distribution_one_group(Config) when is_list(Config) ->
ok = ensure_config(round_robin_per_group, true),
Topic = <<"foo/bar">>,
Group = <<"group1">>,
@ -264,7 +255,7 @@ t_round_robin_per_group_even_distribution_one_group(_) ->
),
ok.
t_round_robin_per_group_even_distribution_two_groups(_) ->
t_round_robin_per_group_even_distribution_two_groups(Config) when is_list(Config) ->
ok = ensure_config(round_robin_per_group, true),
Topic = <<"foo/bar">>,
{ok, ConnPid1} = emqtt:start_link([{clientid, <<"C0">>}]),
@ -350,19 +341,19 @@ t_round_robin_per_group_even_distribution_two_groups(_) ->
),
ok.
t_sticky(_) ->
t_sticky(Config) when is_list(Config) ->
ok = ensure_config(sticky, true),
test_two_messages(sticky).
t_hash(_) ->
t_hash(Config) when is_list(Config) ->
ok = ensure_config(hash, false),
test_two_messages(hash).
t_hash_clinetid(_) ->
t_hash_clinetid(Config) when is_list(Config) ->
ok = ensure_config(hash_clientid, false),
test_two_messages(hash_clientid).
t_hash_topic(_) ->
t_hash_topic(Config) when is_list(Config) ->
ok = ensure_config(hash_topic, false),
ClientId1 = <<"ClientId1">>,
ClientId2 = <<"ClientId2">>,
@ -407,7 +398,7 @@ t_hash_topic(_) ->
ok.
%% if the original subscriber dies, change to another one alive
t_not_so_sticky(_) ->
t_not_so_sticky(Config) when is_list(Config) ->
ok = ensure_config(sticky),
ClientId1 = <<"ClientId1">>,
ClientId2 = <<"ClientId2">>,
@ -481,7 +472,7 @@ last_message(ExpectedPayload, Pids, Timeout) ->
<<"not yet?">>
end.
t_dispatch(_) ->
t_dispatch(Config) when is_list(Config) ->
ok = ensure_config(random),
Topic = <<"foo">>,
?assertEqual(
@ -494,13 +485,13 @@ t_dispatch(_) ->
emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}})
).
t_uncovered_func(_) ->
t_uncovered_func(Config) when is_list(Config) ->
ignored = gen_server:call(emqx_shared_sub, ignored),
ok = gen_server:cast(emqx_shared_sub, ignored),
ignored = emqx_shared_sub ! ignored,
{mnesia_table_event, []} = emqx_shared_sub ! {mnesia_table_event, []}.
t_per_group_config(_) ->
t_per_group_config(Config) when is_list(Config) ->
ok = ensure_group_config(#{
<<"local_group">> => local,
<<"round_robin_group">> => round_robin,
@ -521,7 +512,7 @@ t_per_group_config(_) ->
test_two_messages(round_robin_per_group, <<"round_robin_per_group_group">>),
test_two_messages(round_robin_per_group, <<"round_robin_per_group_group">>).
t_local(_) ->
t_local(Config) when is_list(Config) ->
GroupConfig = #{
<<"local_group">> => local,
<<"round_robin_group">> => round_robin,
@ -567,7 +558,7 @@ t_local(_) ->
?assertNotEqual(UsedSubPid1, UsedSubPid2),
ok.
t_remote(_) ->
t_remote(Config) when is_list(Config) ->
%% This testcase verifies dispatching of shared messages to the remote nodes via backplane API.
%%
%% In this testcase we start two EMQX nodes: local and remote.
@ -594,7 +585,7 @@ t_remote(_) ->
try
{ok, ClientPidLocal} = emqtt:connect(ConnPidLocal),
{ok, ClientPidRemote} = emqtt:connect(ConnPidRemote),
{ok, _ClientPidRemote} = emqtt:connect(ConnPidRemote),
emqtt:subscribe(ConnPidRemote, {<<"$share/remote_group/", Topic/binary>>, 0}),
@ -620,7 +611,7 @@ t_remote(_) ->
stop_slave(Node)
end.
t_local_fallback(_) ->
t_local_fallback(Config) when is_list(Config) ->
ok = ensure_group_config(#{
<<"local_group">> => local,
<<"round_robin_group">> => round_robin,
@ -653,9 +644,14 @@ t_local_fallback(_) ->
%% This one tests that broker tries to select another shared subscriber
%% If the first one doesn't return an ACK
t_redispatch(_) ->
ok = ensure_config(sticky, true),
t_redispatch_qos1_with_ack(Config) when is_list(Config) ->
test_redispatch_qos1(Config, true).
t_redispatch_qos1_no_ack(Config) when is_list(Config) ->
test_redispatch_qos1(Config, false).
test_redispatch_qos1(_Config, AckEnabled) ->
ok = ensure_config(sticky, AckEnabled),
Group = <<"group1">>,
Topic = <<"foo/bar">>,
ClientId1 = <<"ClientId1">>,
@ -682,10 +678,169 @@ t_redispatch(_) ->
emqtt:stop(UsedSubPid2),
ok.
t_qos1_random_dispatch_if_all_members_are_down(Config) when is_list(Config) ->
ok = ensure_config(sticky, true),
Group = <<"group1">>,
Topic = <<"foo/bar">>,
ClientId1 = <<"ClientId1">>,
ClientId2 = <<"ClientId2">>,
SubOpts = [{clean_start, false}],
{ok, ConnPub} = emqtt:start_link([{clientid, <<"pub">>}]),
{ok, _} = emqtt:connect(ConnPub),
{ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1} | SubOpts]),
{ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2} | SubOpts]),
{ok, _} = emqtt:connect(ConnPid1),
{ok, _} = emqtt:connect(ConnPid2),
emqtt:subscribe(ConnPid1, {<<"$share/", Group/binary, "/foo/bar">>, 1}),
emqtt:subscribe(ConnPid2, {<<"$share/", Group/binary, "/foo/bar">>, 1}),
ok = emqtt:stop(ConnPid1),
ok = emqtt:stop(ConnPid2),
[Pid1, Pid2] = emqx_shared_sub:subscribers(Group, Topic),
?assert(is_process_alive(Pid1)),
?assert(is_process_alive(Pid2)),
{ok, _} = emqtt:publish(ConnPub, Topic, <<"hello11">>, 1),
ct:sleep(100),
{ok, Msgs1} = gen_server:call(Pid1, get_mqueue),
{ok, Msgs2} = gen_server:call(Pid2, get_mqueue),
%% assert the message is in mqueue (because socket is closed)
?assertMatch([#message{payload = <<"hello11">>}], Msgs1 ++ Msgs2),
emqtt:stop(ConnPub),
ok.
%% No ack, QoS 2 subscriptions,
%% client1 receives one message, send pubrec, then suspend
%% client2 acts normal (auto_ack=true)
%% Expected behaviour:
%% the messages sent to client1's inflight and mq are re-dispatched after client1 is down
t_dispatch_qos2({init, Config}) when is_list(Config) ->
emqx_config:put_zone_conf(default, [mqtt, max_inflight], 1),
Config;
t_dispatch_qos2({'end', Config}) when is_list(Config) ->
emqx_config:put_zone_conf(default, [mqtt, max_inflight], 0);
t_dispatch_qos2(Config) when is_list(Config) ->
ok = ensure_config(round_robin, _AckEnabled = false),
Topic = <<"foo/bar/1">>,
ClientId1 = <<"ClientId1">>,
ClientId2 = <<"ClientId2">>,
{ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]),
{ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]),
{ok, _} = emqtt:connect(ConnPid1),
{ok, _} = emqtt:connect(ConnPid2),
emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar/#">>, 2}),
emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar/#">>, 2}),
Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>),
Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>),
Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>),
Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>),
ct:sleep(100),
ok = sys:suspend(ConnPid1),
%% One message is inflight
?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message1)),
?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message2)),
?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message3)),
?assertMatch([{_, _, {ok, 1}}], emqx:publish(Message4)),
MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1),
MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2),
%% assert hello2 > hello1 or hello4 > hello3
?assert(MsgRec2 > MsgRec1),
sys:resume(ConnPid1),
%% emqtt subscriber automatically sends PUBREC, but since auto_ack is set to false
%% so it will never send PUBCOMP, hence EMQX should not attempt to send
%% the 4th message yet since max_inflight is 1.
MsgRec3 = ?WAIT(2000, {publish, #{client_pid := ConnPid1, payload := P3}}, P3),
ct:sleep(100),
%% no message expected
?assertEqual([], collect_msgs(0)),
%% now kill client 1
kill_process(ConnPid1),
%% client 2 should receive the message
MsgRec4 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P4}}, P4),
%% assert hello2 > hello1 or hello4 > hello3
?assert(MsgRec4 > MsgRec3),
emqtt:stop(ConnPid2),
ok.
t_dispatch_qos0({init, Config}) when is_list(Config) ->
Config;
t_dispatch_qos0({'end', Config}) when is_list(Config) ->
ok;
t_dispatch_qos0(Config) when is_list(Config) ->
ok = ensure_config(round_robin, _AckEnabled = false),
Topic = <<"foo/bar/1">>,
ClientId1 = <<"ClientId1">>,
ClientId2 = <<"ClientId2">>,
{ok, ConnPid1} = emqtt:start_link([{clientid, ClientId1}, {auto_ack, false}]),
{ok, ConnPid2} = emqtt:start_link([{clientid, ClientId2}, {auto_ack, true}]),
{ok, _} = emqtt:connect(ConnPid1),
{ok, _} = emqtt:connect(ConnPid2),
%% subscribe with QoS 0
emqtt:subscribe(ConnPid1, {<<"$share/group/foo/bar/#">>, 0}),
emqtt:subscribe(ConnPid2, {<<"$share/group/foo/bar/#">>, 0}),
%% publish with QoS 2, but should be downgraded to 0 as the subscribers
%% subscribe with QoS 0
Message1 = emqx_message:make(ClientId1, 2, Topic, <<"hello1">>),
Message2 = emqx_message:make(ClientId1, 2, Topic, <<"hello2">>),
Message3 = emqx_message:make(ClientId1, 2, Topic, <<"hello3">>),
Message4 = emqx_message:make(ClientId1, 2, Topic, <<"hello4">>),
ct:sleep(100),
ok = sys:suspend(ConnPid1),
?assertMatch([_], emqx:publish(Message1)),
?assertMatch([_], emqx:publish(Message2)),
?assertMatch([_], emqx:publish(Message3)),
?assertMatch([_], emqx:publish(Message4)),
MsgRec1 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P1}}, P1),
MsgRec2 = ?WAIT(2000, {publish, #{client_pid := ConnPid2, payload := P2}}, P2),
%% assert hello2 > hello1 or hello4 > hello3
?assert(MsgRec2 > MsgRec1),
kill_process(ConnPid1),
%% expect no redispatch
?assertEqual([], collect_msgs(timer:seconds(2))),
emqtt:stop(ConnPid2),
ok.
%%--------------------------------------------------------------------
%% help functions
%%--------------------------------------------------------------------
kill_process(Pid) ->
_ = unlink(Pid),
_ = monitor(process, Pid),
erlang:exit(Pid, kill),
receive
{'DOWN', _, process, Pid, _} ->
ok
end.
collect_msgs(Timeout) ->
collect_msgs([], Timeout).
collect_msgs(Acc, Timeout) ->
receive
Msg ->
collect_msgs([Msg | Acc], Timeout)
after Timeout ->
lists:reverse(Acc)
end.
ensure_config(Strategy) ->
ensure_config(Strategy, _AckEnabled = true).

4
apps/emqx_authn/include/emqx_authn.hrl
View File

@ -38,8 +38,4 @@
-define(RESOURCE_GROUP, <<"emqx_authn">>).
-define(WITH_SUCCESSFUL_RENDER(Code),
emqx_authn_utils:with_successful_render(?MODULE, fun() -> Code end)
).
-endif.

2
apps/emqx_authn/src/emqx_authn.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_authn, [
{description, "EMQX Authentication"},
{vsn, "0.1.6"},
{vsn, "0.1.7"},
{modules, []},
{registered, [emqx_authn_sup, emqx_authn_registry]},
{applications, [kernel, stdlib, emqx_resource, ehttpc, epgsql, mysql, jose]},

2
apps/emqx_authn/src/emqx_authn_api.erl
View File

@ -34,7 +34,7 @@
% Swagger
-define(API_TAGS_GLOBAL, [<<"Authentication">>]).
-define(API_TAGS_SINGLE, [<<"Listener authentication">>]).
-define(API_TAGS_SINGLE, [<<"Listener Authentication">>]).
-export([
api_spec/0,

2
apps/emqx_authn/src/emqx_authn_user_import_api.erl
View File

@ -30,7 +30,7 @@
% Swagger
-define(API_TAGS_GLOBAL, [<<"Authentication">>]).
-define(API_TAGS_SINGLE, [<<"Listener authentication">>]).
-define(API_TAGS_SINGLE, [<<"Listener Authentication">>]).
-export([
api_spec/0,

26
apps/emqx_authn/src/emqx_authn_utils.erl
View File

@ -34,8 +34,7 @@
ensure_apps_started/1,
cleanup_resources/0,
make_resource_id/1,
without_password/1,
with_successful_render/2
without_password/1
]).
-define(AUTHN_PLACEHOLDERS, [
@ -112,7 +111,8 @@ parse_sql(Template, ReplaceWith) ->
Template,
#{
replace_with => ReplaceWith,
placeholders => ?AUTHN_PLACEHOLDERS
placeholders => ?AUTHN_PLACEHOLDERS,
strip_double_quote => true
}
).
@ -137,18 +137,6 @@ render_sql_params(ParamList, Credential) ->
#{return => rawlist, var_trans => fun handle_sql_var/2}
).
with_successful_render(Provider, Fun) when is_function(Fun, 0) ->
try
Fun()
catch
error:{cannot_get_variable, Name} ->
?TRACE_AUTHN(error, "placeholder_interpolation_failed", #{
provider => Provider,
placeholder => Name
}),
ignore
end.
%% true
is_superuser(#{<<"is_superuser">> := <<"true">>}) ->
#{is_superuser => true};
@ -230,15 +218,15 @@ without_password(Credential, [Name | Rest]) ->
without_password(Credential, Rest)
end.
handle_var({var, Name}, undefined) ->
error({cannot_get_variable, Name});
handle_var({var, _Name}, undefined) ->
<<>>;
handle_var({var, <<"peerhost">>}, PeerHost) ->
emqx_placeholder:bin(inet:ntoa(PeerHost));
handle_var(_, Value) ->
emqx_placeholder:bin(Value).
handle_sql_var({var, Name}, undefined) ->
error({cannot_get_variable, Name});
handle_sql_var({var, _Name}, undefined) ->
<<>>;
handle_sql_var({var, <<"peerhost">>}, PeerHost) ->
emqx_placeholder:bin(inet:ntoa(PeerHost));
handle_sql_var(_, Value) ->

6
apps/emqx_authn/src/simple_authn/emqx_authn_http.erl
View File

@ -187,8 +187,6 @@ authenticate(
request_timeout := RequestTimeout
} = State
) ->
?WITH_SUCCESSFUL_RENDER(
begin
Request = generate_request(Credential, State),
Response = emqx_resource:query(ResourceId, {Method, Request, RequestTimeout}),
?TRACE_AUTHN_PROVIDER("http_response", #{
@ -207,9 +205,7 @@ authenticate(
ignore;
{error, _Reason} ->
ignore
end
end
).
end.
destroy(#{resource_id := ResourceId}) ->
_ = emqx_resource:remove_local(ResourceId),

6
apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl
View File

@ -162,8 +162,6 @@ authenticate(
resource_id := ResourceId
} = State
) ->
?WITH_SUCCESSFUL_RENDER(
begin
Filter = emqx_authn_utils:render_deep(FilterTemplate, Credential),
case emqx_resource:query(ResourceId, {find_one, Collection, Filter, #{}}) of
undefined ->
@ -192,9 +190,7 @@ authenticate(
{error, Reason} ->
{error, Reason}
end
end
end
).
end.
%%------------------------------------------------------------------------------
%% Internal functions

6
apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl
View File

@ -113,8 +113,6 @@ authenticate(
password_hash_algorithm := Algorithm
}
) ->
?WITH_SUCCESSFUL_RENDER(
begin
Params = emqx_authn_utils:render_sql_params(TmplToken, Credential),
case emqx_resource:query(ResourceId, {prepared_query, ?PREPARE_KEY, Params, Timeout}) of
{ok, _Columns, []} ->
@ -140,9 +138,7 @@ authenticate(
reason => Reason
}),
ignore
end
end
).
end.
parse_config(
#{

6
apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl
View File

@ -115,8 +115,6 @@ authenticate(
password_hash_algorithm := Algorithm
}
) ->
?WITH_SUCCESSFUL_RENDER(
begin
Params = emqx_authn_utils:render_sql_params(PlaceHolders, Credential),
case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Params}) of
{ok, _Columns, []} ->
@ -141,9 +139,7 @@ authenticate(
reason => Reason
}),
ignore
end
end
).
end.
parse_config(
#{

20
apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl
View File

@ -133,15 +133,14 @@ authenticate(
password_hash_algorithm := Algorithm
}
) ->
?WITH_SUCCESSFUL_RENDER(
begin
NKey = emqx_authn_utils:render_str(KeyTemplate, Credential),
Command = [CommandName, NKey | Fields],
case emqx_resource:query(ResourceId, {cmd, Command}) of
{ok, []} ->
ignore;
{ok, Values} ->
Selected = merge(Fields, Values),
case merge(Fields, Values) of
Selected when Selected =/= #{} ->
case
emqx_authn_utils:check_password_from_selected_map(
Algorithm, Selected, Password
@ -149,7 +148,16 @@ authenticate(
of
ok ->
{ok, emqx_authn_utils:is_superuser(Selected)};
{error, _Reason} ->
{error, _Reason} = Error ->
Error
end;
_ ->
?TRACE_AUTHN_PROVIDER(info, "redis_query_not_matched", #{
resource => ResourceId,
cmd => Command,
keys => NKey,
fields => Fields
}),
ignore
end;
{error, Reason} ->
@ -161,9 +169,7 @@ authenticate(
reason => Reason
}),
ignore
end
end
).
end.
%%------------------------------------------------------------------------------
%% Internal functions

84
apps/emqx_authn/test/emqx_authn_http_SUITE.erl
View File

@ -166,6 +166,49 @@ test_user_auth(#{
?GLOBAL
).
t_no_value_for_placeholder(_Config) ->
Handler = fun(Req0, State) ->
{ok, RawBody, Req1} = cowboy_req:read_body(Req0),
#{
<<"cert_subject">> := <<"">>,
<<"cert_common_name">> := <<"">>
} = jiffy:decode(RawBody, [return_maps]),
Req = cowboy_req:reply(
200,
#{<<"content-type">> => <<"application/json">>},
jiffy:encode(#{result => allow, is_superuser => false}),
Req1
),
{ok, Req, State}
end,
SpecificConfgParams = #{
<<"method">> => <<"post">>,
<<"headers">> => #{<<"content-type">> => <<"application/json">>},
<<"body">> => #{
<<"cert_subject">> => ?PH_CERT_SUBJECT,
<<"cert_common_name">> => ?PH_CERT_CN_NAME
}
},
AuthConfig = maps:merge(raw_http_auth_config(), SpecificConfgParams),
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, AuthConfig}
),
ok = emqx_authn_http_test_server:set_handler(Handler),
Credentials = maps:without([cert_subject, cert_common_name], ?CREDENTIALS),
?assertMatch({ok, _}, emqx_access_control:authenticate(Credentials)),
emqx_authn_test_lib:delete_authenticators(
[authentication],
?GLOBAL
).
t_destroy(_Config) ->
AuthConfig = raw_http_auth_config(),
@ -247,27 +290,6 @@ t_update(_Config) ->
emqx_access_control:authenticate(?CREDENTIALS)
).
t_interpolation_error(_Config) ->
{ok, _} = emqx:update_config(
?PATH,
{create_authenticator, ?GLOBAL, raw_http_auth_config()}
),
Headers = #{<<"content-type">> => <<"application/json">>},
Response = ?SERVER_RESPONSE_JSON(allow),
ok = emqx_authn_http_test_server:set_handler(
fun(Req0, State) ->
Req = cowboy_req:reply(200, Headers, Response, Req0),
{ok, Req, State}
end
),
?assertMatch(
?EXCEPTION_DENY,
emqx_access_control:authenticate(maps:without([username], ?CREDENTIALS))
).
t_is_superuser(_Config) ->
Config = raw_http_auth_config(),
{ok, _} = emqx:update_config(
@ -431,26 +453,6 @@ samples() ->
result => {ok, #{is_superuser => false, user_property => #{}}}
},
%% simple get request, no username
#{
handler => fun(Req0, State) ->
#{
username := <<"plain">>,
password := <<"plain">>
} = cowboy_req:match_qs([username, password], Req0),
Req = cowboy_req:reply(
200,
#{<<"content-type">> => <<"application/json">>},
jiffy:encode(#{result => allow, is_superuser => false}),
Req0
),
{ok, Req, State}
end,
config_params => #{},
result => {ok, #{is_superuser => false, user_property => #{}}}
},
%% get request with json body response
#{
handler => fun(Req0, State) ->

14
apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl
View File

@ -288,20 +288,6 @@ raw_mongo_auth_config() ->
user_seeds() ->
[
#{
data => #{
username => <<"plain">>,
password_hash => <<"plainsalt">>,
salt => <<"salt">>,
is_superuser => <<"1">>
},
credentials => #{
password => <<"plain">>
},
config_params => #{},
result => {error, not_authorized}
},
#{
data => #{
username => <<"plain">>,

40
apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl
View File

@ -258,20 +258,6 @@ raw_mysql_auth_config() ->
user_seeds() ->
[
#{
data => #{
username => "plain",
password_hash => "plainsalt",
salt => "salt",
is_superuser_str => "1"
},
credentials => #{
password => <<"plain">>
},
config_params => #{},
result => {error, not_authorized}
},
#{
data => #{
username => "plain",
@ -332,6 +318,32 @@ user_seeds() ->
result => {ok, #{is_superuser => true}}
},
%% strip double quote support
#{
data => #{
username => "sha256",
password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf",
salt => "salt",
is_superuser_int => 1
},
credentials => #{
username => <<"sha256">>,
password => <<"sha256">>
},
config_params => #{
<<"query">> =>
<<
"SELECT password_hash, salt, is_superuser_int as is_superuser\n"
" FROM users where username = \"${username}\" LIMIT 1"
>>,
<<"password_hash_algorithm">> => #{
<<"name">> => <<"sha256">>,
<<"salt_position">> => <<"prefix">>
}
},
result => {ok, #{is_superuser => true}}
},
#{
data => #{
username => "sha256",

40
apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl
View File

@ -320,20 +320,6 @@ raw_pgsql_auth_config() ->
user_seeds() ->
[
#{
data => #{
username => "plain",
password_hash => "plainsalt",
salt => "salt",
is_superuser_str => "1"
},
credentials => #{
password => <<"plain">>
},
config_params => #{},
result => {error, not_authorized}
},
#{
data => #{
username => "plain",
@ -394,6 +380,32 @@ user_seeds() ->
result => {ok, #{is_superuser => true}}
},
%% strip double quote support
#{
data => #{
username => "sha256",
password_hash => "ac63a624e7074776d677dd61a003b8c803eb11db004d0ec6ae032a5d7c9c5caf",
salt => "salt",
is_superuser_int => 1
},
credentials => #{
username => <<"sha256">>,
password => <<"sha256">>
},
config_params => #{
<<"query">> =>
<<
"SELECT password_hash, salt, is_superuser_int as is_superuser\n"
" FROM users where username = \"${username}\" LIMIT 1"
>>,
<<"password_hash_algorithm">> => #{
<<"name">> => <<"sha256">>,
<<"salt_position">> => <<"prefix">>
}
},
result => {ok, #{is_superuser => true}}
},
#{
data => #{
username => "sha256",

51
apps/emqx_authn/test/emqx_authn_redis_SUITE.erl
View File

@ -161,11 +161,13 @@ t_authenticate(_Config) ->
user_seeds()
).
test_user_auth(#{
test_user_auth(
#{
credentials := Credentials0,
config_params := SpecificConfigParams,
result := Result
}) ->
} = Config
) ->
AuthConfig = maps:merge(raw_redis_auth_config(), SpecificConfigParams),
{ok, _} = emqx:update_config(
@ -183,14 +185,12 @@ test_user_auth(#{
?assertEqual(Result, emqx_access_control:authenticate(Credentials)),
AuthnResult =
case Result of
{error, _} ->
ignore;
Any ->
Any
case maps:get(redis_result, Config, undefined) of
undefined ->
ok;
RedisResult ->
?assertEqual(RedisResult, emqx_authn_redis:authenticate(Credentials, State))
end,
?assertEqual(AuthnResult, emqx_authn_redis:authenticate(Credentials, State)),
emqx_authn_test_lib:delete_authenticators(
[authentication],
@ -292,20 +292,6 @@ raw_redis_auth_config() ->
user_seeds() ->
[
#{
data => #{
password_hash => <<"plainsalt">>,
salt => <<"salt">>,
is_superuser => <<"1">>
},
credentials => #{
password => <<"plain">>
},
key => <<"mqtt_user:plain">>,
config_params => #{},
result => {error, not_authorized}
},
#{
data => #{
password_hash => <<"plainsalt">>,
@ -478,7 +464,7 @@ user_seeds() ->
<<"cmd">> => <<"HMGET mqtt_user:${username} password_hash salt is_superuser">>,
<<"password_hash_algorithm">> => #{<<"name">> => <<"bcrypt">>}
},
result => {error, not_authorized}
result => {error, bad_username_or_password}
},
#{
@ -547,6 +533,23 @@ user_seeds() ->
}
},
result => {ok, #{is_superuser => true}}
},
%% user not exists
#{
data => #{
password_hash => <<"plainsalt">>,
salt => <<"salt">>,
is_superuser => <<"1">>
},
credentials => #{
username => <<"not_exists">>,
password => <<"plain">>
},
key => <<"mqtt_user:plain">>,
config_params => #{},
result => {error, not_authorized},
redis_result => ignore
}
].

2
apps/emqx_authz/src/emqx_authz.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_authz, [
{description, "An OTP application"},
{vsn, "0.1.5"},
{vsn, "0.1.6"},
{registered, []},
{mod, {emqx_authz_app, []}},
{applications, [

8
apps/emqx_authz/src/emqx_authz.erl
View File

@ -391,14 +391,6 @@ do_authorize(
Matched ->
{Matched, Type}
catch
error:{cannot_get_variable, Name} ->
emqx_metrics_worker:inc(authz_metrics, Type, nomatch),
?SLOG(warning, #{
msg => "placeholder_interpolation_failed",
placeholder => Name,
authorize_type => Type
}),
do_authorize(Client, PubSub, Topic, Tail);
Class:Reason:Stacktrace ->
emqx_metrics_worker:inc(authz_metrics, Type, nomatch),
?SLOG(warning, #{

9
apps/emqx_authz/src/emqx_authz_api_sources.erl
View File

@ -223,7 +223,7 @@ sources(get, _) ->
])
end;
(Source, AccIn) ->
lists:append(AccIn, [drop_invalid_certs(Source)])
lists:append(AccIn, [Source])
end,
[],
get_raw_sources()
@ -257,7 +257,7 @@ source(get, #{bindings := #{type := Type}}) ->
}}
end;
[Source] ->
{200, drop_invalid_certs(Source)}
{200, Source}
end;
source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file">>} = Body}) ->
update_authz_file(Body);
@ -511,11 +511,6 @@ update_config(Cmd, Sources) ->
}}
end.
drop_invalid_certs(#{<<"ssl">> := SSL} = Source) when SSL =/= undefined ->
Source#{<<"ssl">> => emqx_tls_lib:drop_invalid_certs(SSL)};
drop_invalid_certs(Source) ->
Source.
parameters_field() ->
[
{type,

4
apps/emqx_authz/src/emqx_authz_http.erl
View File

@ -45,7 +45,9 @@
?PH_PROTONAME,
?PH_MOUNTPOINT,
?PH_TOPIC,
?PH_ACTION
?PH_ACTION,
?PH_CERT_SUBJECT,
?PH_CERT_CN_NAME
]).
description() ->

4
apps/emqx_authz/src/emqx_authz_mongodb.erl
View File

@ -40,7 +40,9 @@
-define(PLACEHOLDERS, [
?PH_USERNAME,
?PH_CLIENTID,
?PH_PEERHOST
?PH_PEERHOST,
?PH_CERT_CN_NAME,
?PH_CERT_SUBJECT
]).
description() ->

11
apps/emqx_authz/src/emqx_authz_utils.erl
View File

@ -110,7 +110,8 @@ parse_sql(Template, ReplaceWith, PlaceHolders) ->
Template,
#{
replace_with => ReplaceWith,
placeholders => PlaceHolders
placeholders => PlaceHolders,
strip_double_quote => true
}
).
@ -181,15 +182,15 @@ convert_client_var({dn, DN}) -> {cert_subject, DN};
convert_client_var({protocol, Proto}) -> {proto_name, Proto};
convert_client_var(Other) -> Other.
handle_var({var, Name}, undefined) ->
error({cannot_get_variable, Name});
handle_var({var, _Name}, undefined) ->
<<>>;
handle_var({var, <<"peerhost">>}, IpAddr) ->
inet_parse:ntoa(IpAddr);
handle_var(_Name, Value) ->
emqx_placeholder:bin(Value).
handle_sql_var({var, Name}, undefined) ->
error({cannot_get_variable, Name});
handle_sql_var({var, _Name}, undefined) ->
<<>>;
handle_sql_var({var, <<"peerhost">>}, IpAddr) ->
inet_parse:ntoa(IpAddr);
handle_sql_var(_Name, Value) ->

110
apps/emqx_authz/test/emqx_authz_SUITE.erl
View File

@ -19,9 +19,12 @@
-compile(export_all).
-include("emqx_authz.hrl").
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
all() ->
emqx_common_test_helpers:all(?MODULE).
@ -61,10 +64,26 @@ end_per_suite(_Config) ->
meck:unload(emqx_resource),
ok.
init_per_testcase(TestCase, Config) when
TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament;
TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament
->
{ok, _} = emqx_authz:update(?CMD_REPLACE, []),
{ok, _} = emqx:update_config([authorization, deny_action], disconnect),
Config;
init_per_testcase(_, Config) ->
{ok, _} = emqx_authz:update(?CMD_REPLACE, []),
Config.
end_per_testcase(TestCase, _Config) when
TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament;
TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament
->
{ok, _} = emqx:update_config([authorization, deny_action], ignore),
ok;
end_per_testcase(_TestCase, _Config) ->
ok.
set_special_configs(emqx_authz) ->
{ok, _} = emqx:update_config([authorization, cache, enable], false),
{ok, _} = emqx:update_config([authorization, no_match], deny),
@ -139,6 +158,15 @@ set_special_configs(_App) ->
"\n{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}."
>>
}).
-define(SOURCE7, #{
<<"type">> => <<"file">>,
<<"enable">> => true,
<<"rules">> =>
<<
"{allow,{username,\"some_client\"},publish,[\"some_client/lwt\"]}.\n"
"{deny, all}."
>>
}).
%%------------------------------------------------------------------------------
%% Testcases
@ -287,5 +315,87 @@ t_get_enabled_authzs_some_enabled(_Config) ->
{ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE4]),
?assertEqual([postgresql], emqx_authz:get_enabled_authzs()).
t_subscribe_deny_disconnect_publishes_last_will_testament(_Config) ->
{ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE7]),
{ok, C} = emqtt:start_link([
{username, <<"some_client">>},
{will_topic, <<"some_client/lwt">>},
{will_payload, <<"should be published">>}
]),
{ok, _} = emqtt:connect(C),
ok = emqx:subscribe(<<"some_client/lwt">>),
process_flag(trap_exit, true),
try
emqtt:subscribe(C, <<"unauthorized">>),
error(should_have_disconnected)
catch
exit:{{shutdown, tcp_closed}, _} ->
ok
end,
receive
{deliver, <<"some_client/lwt">>, #message{payload = <<"should be published">>}} ->
ok
after 2_000 ->
error(lwt_not_published)
end,
ok.
t_publish_deny_disconnect_publishes_last_will_testament(_Config) ->
{ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE7]),
{ok, C} = emqtt:start_link([
{username, <<"some_client">>},
{will_topic, <<"some_client/lwt">>},
{will_payload, <<"should be published">>}
]),
{ok, _} = emqtt:connect(C),
ok = emqx:subscribe(<<"some_client/lwt">>),
process_flag(trap_exit, true),
%% disconnect is async
Ref = monitor(process, C),
emqtt:publish(C, <<"some/topic">>, <<"unauthorized">>),
receive
{'DOWN', Ref, process, C, _} ->
ok
after 1_000 ->
error(client_should_have_been_disconnected)
end,
receive
{deliver, <<"some_client/lwt">>, #message{payload = <<"should be published">>}} ->
ok
after 2_000 ->
error(lwt_not_published)
end,
ok.
t_publish_last_will_testament_denied_topic(_Config) ->
{ok, C} = emqtt:start_link([
{will_topic, <<"$SYS/lwt">>},
{will_payload, <<"should not be published">>}
]),
{ok, _} = emqtt:connect(C),
ok = emqx:subscribe(<<"$SYS/lwt">>),
unlink(C),
ok = snabbkaffe:start_trace(),
{true, {ok, _}} = ?wait_async_action(
exit(C, kill),
#{?snk_kind := last_will_testament_publish_denied},
1_000
),
ok = snabbkaffe:stop(),
receive
{deliver, <<"$SYS/lwt">>, #message{payload = <<"should not be published">>}} ->
error(lwt_should_not_be_published_to_forbidden_topic)
after 1_000 ->
ok
end,
ok.
stop_apps(Apps) ->
lists:foreach(fun application:stop/1, Apps).

53
apps/emqx_authz/test/emqx_authz_http_SUITE.erl
View File

@ -22,6 +22,7 @@
-include("emqx_authz.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
-define(HTTP_PORT, 33333).
-define(HTTP_PATH, "/authz/[...]").
@ -303,7 +304,7 @@ t_json_body(_Config) ->
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
).
t_form_body(_Config) ->
t_placeholder_and_body(_Config) ->
ok = setup_handler_and_config(
fun(Req0, State) ->
?assertEqual(
@ -321,7 +322,9 @@ t_form_body(_Config) ->
<<"proto_name">> := <<"MQTT">>,
<<"mountpoint">> := <<"MOUNTPOINT">>,
<<"topic">> := <<"t">>,
<<"action">> := <<"publish">>
<<"action">> := <<"publish">>,
<<"CN">> := ?PH_CERT_CN_NAME,
<<"CS">> := ?PH_CERT_SUBJECT
},
jiffy:decode(PostVars, [return_maps])
),
@ -336,7 +339,9 @@ t_form_body(_Config) ->
<<"proto_name">> => <<"${proto_name}">>,
<<"mountpoint">> => <<"${mountpoint}">>,
<<"topic">> => <<"${topic}">>,
<<"action">> => <<"${action}">>
<<"action">> => <<"${action}">>,
<<"CN">> => ?PH_CERT_CN_NAME,
<<"CS">> => ?PH_CERT_SUBJECT
},
<<"headers">> => #{<<"content-type">> => <<"application/x-www-form-urlencoded">>}
}
@ -349,6 +354,48 @@ t_form_body(_Config) ->
protocol => <<"MQTT">>,
mountpoint => <<"MOUNTPOINT">>,
zone => default,
listener => {tcp, default},
cn => ?PH_CERT_CN_NAME,
dn => ?PH_CERT_SUBJECT
},
?assertEqual(
allow,
emqx_access_control:authorize(ClientInfo, publish, <<"t">>)
).
t_no_value_for_placeholder(_Config) ->
ok = setup_handler_and_config(
fun(Req0, State) ->
?assertEqual(
<<"/authz/users/">>,
cowboy_req:path(Req0)
),
{ok, RawBody, Req1} = cowboy_req:read_body(Req0),
?assertMatch(
#{
<<"mountpoint">> := <<"[]">>
},
jiffy:decode(RawBody, [return_maps])
),
{ok, ?AUTHZ_HTTP_RESP(allow, Req1), State}
end,
#{
<<"method">> => <<"post">>,
<<"body">> => #{
<<"mountpoint">> => <<"[${mountpoint}]">>
}
}
),
ClientInfo = #{
clientid => <<"client id">>,
username => <<"user name">>,
peerhost => {127, 0, 0, 1},
protocol => <<"MQTT">>,
zone => default,
listener => {tcp, default}
},

42
apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl
View File

@ -18,8 +18,8 @@
-compile(nowarn_export_all).
-compile(export_all).
-include("emqx_connector.hrl").
-include("emqx_authz.hrl").
-include_lib("emqx_connector/include/emqx_connector.hrl").
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_placeholder.hrl").
@ -188,6 +188,46 @@ t_lookups(_Config) ->
#{<<"filter">> => #{<<"peerhost">> => <<"${peerhost}">>}}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
ByCN = #{
<<"CN">> => <<"cn">>,
<<"topics">> => [<<"a">>],
<<"action">> => <<"all">>,
<<"permission">> => <<"allow">>
},
ok = setup_samples([ByCN]),
ok = setup_config(
#{<<"filter">> => #{<<"CN">> => ?PH_CERT_CN_NAME}}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
ByDN = #{
<<"DN">> => <<"dn">>,
<<"topics">> => [<<"a">>],
<<"action">> => <<"all">>,
<<"permission">> => <<"allow">>
},
ok = setup_samples([ByDN]),
ok = setup_config(
#{<<"filter">> => #{<<"DN">> => ?PH_CERT_SUBJECT}}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[

28
apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl
View File

@ -202,6 +202,34 @@ t_lookups(_Config) ->
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
%% strip double quote support
ok = init_table(),
ok = q(
<<
"INSERT INTO acl(clientid, topic, permission, action)"
"VALUES(?, ?, ?, ?)"
>>,
[<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE clientid = \"${clientid}\""
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[

28
apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl
View File

@ -202,6 +202,34 @@ t_lookups(_Config) ->
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[
{allow, subscribe, <<"a">>},
{deny, subscribe, <<"b">>}
]
),
%% strip double quote support
ok = init_table(),
ok = insert(
<<
"INSERT INTO acl(clientid, topic, permission, action)"
"VALUES($1, $2, $3, $4)"
>>,
[<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>]
),
ok = setup_config(
#{
<<"query">> => <<
"SELECT permission, action, topic "
"FROM acl WHERE clientid = \"${clientid}\""
>>
}
),
ok = emqx_authz_test_lib:test_samples(
ClientInfo,
[

2
apps/emqx_auto_subscribe/src/emqx_auto_subscribe.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_auto_subscribe, [
{description, "An OTP application"},
{vsn, "0.1.1"},
{vsn, "0.1.2"},
{registered, []},
{mod, {emqx_auto_subscribe_app, []}},
{applications, [

4
apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl
View File

@ -44,14 +44,14 @@ schema("/mqtt/auto_subscribe") ->
'operationId' => auto_subscribe,
get => #{
description => ?DESC(list_auto_subscribe_api),
tags => [<<"Auto subscribe">>],
tags => [<<"Auto Subscribe">>],
responses => #{
200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe")
}
},
put => #{
description => ?DESC(update_auto_subscribe_api),
tags => [<<"Auto subscribe">>],
tags => [<<"Auto Subscribe">>],
'requestBody' => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"),
responses => #{
200 => hoconsc:ref(emqx_auto_subscribe_schema, "auto_subscribe"),

3
apps/emqx_bridge/src/emqx_bridge_api.erl
View File

@ -584,10 +584,9 @@ pick_bridges_by_id(Type, Name, BridgesAllNodes) ->
format_bridge_info([FirstBridge | _] = Bridges) ->
Res = maps:remove(node, FirstBridge),
NRes = emqx_connector_ssl:drop_invalid_certs(Res),
NodeStatus = collect_status(Bridges),
NodeMetrics = collect_metrics(Bridges),
NRes#{
Res#{
status => aggregate_status(NodeStatus),
node_status => NodeStatus,
metrics => aggregate_metrics(NodeMetrics),

2
apps/emqx_conf/src/emqx_conf.app.src
View File

@ -1,6 +1,6 @@
{application, emqx_conf, [
{description, "EMQX configuration management"},
{vsn, "0.1.4"},
{vsn, "0.1.5"},
{registered, []},
{mod, {emqx_conf_app, []}},
{applications, [kernel, stdlib]},

14
apps/emqx_conf/src/emqx_conf_app.erl
View File

@ -152,11 +152,17 @@ copy_override_conf_from_core_node() ->
_ ->
[{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
#{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
Msg = #{
?SLOG(debug, #{
msg => "copy_overide_conf_from_core_node_success",
node => Node
},
?SLOG(debug, Msg),
node => Node,
cluster_override_conf_file => application:get_env(
emqx, cluster_override_conf_file
),
local_override_conf_file => application:get_env(
emqx, local_override_conf_file
),
data_dir => emqx:data_dir()
}),
ok = emqx_config:save_to_override_conf(
RawOverrideConf,
#{override_to => cluster}

9
apps/emqx_conf/src/emqx_conf_schema.erl
View File

@ -536,6 +536,15 @@ fields("node") ->
desc => ?DESC(node_applications)
}
)},
{"etc_dir",
sc(
string(),
#{
desc => ?DESC(node_etc_dir),
'readOnly' => true,
deprecated => {since, "5.0.8"}
}
)},
{"cluster_call",
sc(
?R_REF("cluster_call"),

2
apps/emqx_connector/src/emqx_connector.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_connector, [
{description, "An OTP application"},
{vsn, "0.1.5"},
{vsn, "0.1.6"},
{registered, []},
{mod, {emqx_connector_app, []}},
{applications, [

2
apps/emqx_connector/src/emqx_connector_mongo.erl
View File

@ -345,7 +345,7 @@ init_worker_options([], Acc) ->
%% ===================================================================
%% Schema funcs
server(type) -> emqx_schema:ip_port();
server(type) -> emqx_schema:host_port();
server(required) -> true;
server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
server(converter) -> fun to_server_raw/1;

13
apps/emqx_connector/src/emqx_connector_mqtt.erl
View File

@ -222,20 +222,20 @@ make_forward_confs(undefined) ->
make_forward_confs(FrowardConf) ->
FrowardConf.
basic_config(#{
basic_config(
#{
server := Server,
reconnect_interval := ReconnIntv,
proto_ver := ProtoVer,
bridge_mode := BridgeMode,
username := User,
password := Password,
clean_start := CleanStart,
keepalive := KeepAlive,
retry_interval := RetryIntv,
max_inflight := MaxInflight,
replayq := ReplayQ,
ssl := #{enable := EnableSsl} = Ssl
}) ->
} = Conf
) ->
#{
replayq => ReplayQ,
%% connection opts
@ -251,8 +251,9 @@ basic_config(#{
%% non-standard mqtt connection packets will be filtered out by LB.
%% So let's disable bridge_mode.
bridge_mode => BridgeMode,
username => User,
password => Password,
%% should be iolist for emqtt
username => maps:get(username, Conf, <<>>),
password => maps:get(password, Conf, <<>>),
clean_start => CleanStart,
keepalive => ms_to_s(KeepAlive),
retry_interval => RetryIntv,

2
apps/emqx_connector/src/emqx_connector_mysql.erl
View File

@ -55,7 +55,7 @@ fields(config) ->
emqx_connector_schema_lib:ssl_fields() ++
emqx_connector_schema_lib:prepare_statement_fields().
server(type) -> emqx_schema:ip_port();
server(type) -> emqx_schema:host_port();
server(required) -> true;
server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
server(converter) -> fun to_server/1;

2
apps/emqx_connector/src/emqx_connector_pgsql.erl
View File

@ -58,7 +58,7 @@ fields(config) ->
emqx_connector_schema_lib:ssl_fields() ++
emqx_connector_schema_lib:prepare_statement_fields().
server(type) -> emqx_schema:ip_port();
server(type) -> emqx_schema:host_port();
server(required) -> true;
server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
server(converter) -> fun to_server/1;

2
apps/emqx_connector/src/emqx_connector_redis.erl
View File

@ -97,7 +97,7 @@ fields(sentinel) ->
redis_fields() ++
emqx_connector_schema_lib:ssl_fields().
server(type) -> emqx_schema:ip_port();
server(type) -> emqx_schema:host_port();
server(required) -> true;
server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")];
server(converter) -> fun to_server_raw/1;

23
apps/emqx_connector/src/emqx_connector_ssl.erl
View File

@ -18,7 +18,6 @@
-export([
convert_certs/2,
drop_invalid_certs/1,
clear_certs/2
]).
@ -61,28 +60,6 @@ clear_certs(RltvDir, #{ssl := OldSSL} = _Config) ->
clear_certs(_RltvDir, _) ->
ok.
drop_invalid_certs(#{<<"connector">> := Connector} = Config) when
is_map(Connector)
->
SSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
new_ssl_config(Config, NewSSL);
drop_invalid_certs(#{connector := Connector} = Config) when
is_map(Connector)
->
SSL = map_get_oneof([<<"ssl">>, ssl], Connector, undefined),
NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
new_ssl_config(Config, NewSSL);
drop_invalid_certs(#{<<"ssl">> := SSL} = Config) ->
NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
new_ssl_config(Config, NewSSL);
drop_invalid_certs(#{ssl := SSL} = Config) ->
NewSSL = emqx_tls_lib:drop_invalid_certs(SSL),
new_ssl_config(Config, NewSSL);
%% for bridges use connector name
drop_invalid_certs(Config) ->
Config.
new_ssl_config(RltvDir, Config, SSL) ->
case emqx_tls_lib:ensure_ssl_files(RltvDir, SSL) of
{ok, NewSSL} ->

6
apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl
View File

@ -55,7 +55,7 @@ fields("connector") ->
)},
{server,
sc(
emqx_schema:ip_port(),
emqx_schema:host_port(),
#{
required => true,
desc => ?DESC("server")
@ -177,7 +177,7 @@ fields("ingress") ->
sc(
binary(),
#{
default => <<"${payload}">>,
default => undefined,
desc => ?DESC("payload")
}
)}
@ -224,7 +224,7 @@ fields("egress") ->
sc(
binary(),
#{
default => <<"${payload}">>,
default => undefined,
desc => ?DESC("payload")
}
)}

2
apps/emqx_dashboard/src/emqx_dashboard.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_dashboard, [
{description, "EMQX Web Dashboard"},
% strict semver, bump manually!
{vsn, "5.0.5"},
{vsn, "5.0.6"},
{modules, []},
{registered, [emqx_dashboard_sup]},
{applications, [kernel, stdlib, mnesia, minirest, emqx]},

4
apps/emqx_dashboard/src/emqx_dashboard_error_code_api.erl
View File

@ -51,7 +51,7 @@ schema("/error_codes") ->
get => #{
security => [],
description => <<"API Error Codes">>,
tags => [<<"Error codes">>],
tags => [<<"Error Codes">>],
responses => #{
200 => hoconsc:array(hoconsc:ref(?MODULE, error_code))
}
@ -63,7 +63,7 @@ schema("/error_codes/:code") ->
get => #{
security => [],
description => <<"API Error Codes">>,
tags => [<<"Error codes">>],
tags => [<<"Error Codes">>],
parameters => [
{code,
hoconsc:mk(hoconsc:enum(emqx_dashboard_error_code:all()), #{

4
apps/emqx_dashboard/src/emqx_dashboard_swagger.erl
View File

@ -656,8 +656,8 @@ typename_to_spec("file()", _Mod) ->
#{type => string, example => <<"/path/to/file">>};
typename_to_spec("ip_port()", _Mod) ->
#{type => string, example => <<"127.0.0.1:80">>};
typename_to_spec("ip_ports()", _Mod) ->
#{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>};
typename_to_spec("host_port()", _Mod) ->
#{type => string, example => <<"example.host.domain:80">>};
typename_to_spec("url()", _Mod) ->
#{type => string, example => <<"http://127.0.0.1">>};
typename_to_spec("connect_timeout()", Mod) ->

8
apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl
View File

@ -74,14 +74,6 @@ end_per_suite(_Config) ->
emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_management]),
mria:stop().
set_special_configs(emqx_management) ->
Listeners = #{http => #{port => 8081}},
Config = #{
listeners => Listeners,
applications => [#{id => "admin", secret => "public"}]
},
emqx_config:put([emqx_management], Config),
ok;
set_special_configs(emqx_dashboard) ->
emqx_dashboard_api_test_helpers:set_default_config(),
ok;

2
apps/emqx_exhook/include/emqx_exhook.hrl
View File

@ -43,6 +43,8 @@
{'message.dropped', {emqx_exhook_handler, on_message_dropped, []}}
]).
-define(SERVER_FORCE_SHUTDOWN_TIMEOUT, 5000).
-endif.
-define(CMD_MOVE_FRONT, front).

2
apps/emqx_exhook/src/emqx_exhook.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_exhook, [
{description, "EMQX Extension for Hook"},
{vsn, "5.0.4"},
{vsn, "5.0.5"},
{modules, []},
{registered, []},
{mod, {emqx_exhook_app, []}},

9
apps/emqx_exhook/src/emqx_exhook_api.erl
View File

@ -41,7 +41,7 @@
-import(hoconsc, [mk/1, mk/2, ref/1, enum/1, array/1, map/2]).
-import(emqx_dashboard_swagger, [schema_with_example/2, error_codes/2]).
-define(TAGS, [<<"exhooks">>]).
-define(TAGS, [<<"ExHook">>]).
-define(NOT_FOURD, 'NOT_FOUND').
-define(BAD_REQUEST, 'BAD_REQUEST').
-define(BAD_RPC, 'BAD_RPC').
@ -483,16 +483,11 @@ err_msg(Msg) -> emqx_misc:readable_error_msg(Msg).
get_raw_config() ->
RawConfig = emqx:get_raw_config([exhook, servers], []),
Schema = #{roots => emqx_exhook_schema:fields(exhook), fields => #{}},
Conf = #{<<"servers">> => lists:map(fun drop_invalid_certs/1, RawConfig)},
Conf = #{<<"servers">> => RawConfig},
Options = #{only_fill_defaults => true},
#{<<"servers">> := Servers} = hocon_tconf:check_plain(Schema, Conf, Options),
Servers.
drop_invalid_certs(#{<<"ssl">> := SSL} = Conf) when SSL =/= undefined ->
Conf#{<<"ssl">> => emqx_tls_lib:drop_invalid_certs(SSL)};
drop_invalid_certs(Conf) ->
Conf.
position_example() ->
#{
front =>

6
apps/emqx_exhook/src/emqx_exhook_mgr.erl
View File

@ -21,6 +21,7 @@
-include("emqx_exhook.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% APIs
-export([start_link/0]).
@ -297,7 +298,8 @@ handle_info(refresh_tick, State) ->
handle_info(_Info, State) ->
{noreply, State}.
terminate(_Reason, State = #{servers := Servers}) ->
terminate(Reason, State = #{servers := Servers}) ->
_ = unload_exhooks(),
_ = maps:fold(
fun(Name, _, AccIn) ->
do_unload_server(Name, AccIn)
@ -305,7 +307,7 @@ terminate(_Reason, State = #{servers := Servers}) ->
State,
Servers
),
_ = unload_exhooks(),
?tp(info, exhook_mgr_terminated, #{reason => Reason, servers => Servers}),
ok.
code_change(_OldVsn, State, _Extra) ->

7
apps/emqx_exhook/src/emqx_exhook_server.erl
View File

@ -179,13 +179,16 @@ filter(Ls) ->
-spec unload(server()) -> ok.
unload(#{name := Name, options := ReqOpts, hookspec := HookSpecs}) ->
_ = do_deinit(Name, ReqOpts),
_ = may_unload_hooks(HookSpecs),
_ = do_deinit(Name, ReqOpts),
_ = emqx_exhook_sup:stop_grpc_client_channel(Name),
ok.
do_deinit(Name, ReqOpts) ->
_ = do_call(Name, undefined, 'on_provider_unloaded', #{}, ReqOpts),
%% Override the request timeout to deinit grpc server to
%% avoid emqx_exhook_mgr force killed by upper supervisor
NReqOpts = ReqOpts#{timeout => ?SERVER_FORCE_SHUTDOWN_TIMEOUT},
_ = do_call(Name, undefined, 'on_provider_unloaded', #{}, NReqOpts),
ok.
do_init(ChannName, ReqOpts) ->

16
apps/emqx_exhook/src/emqx_exhook_sup.erl
View File

@ -16,6 +16,8 @@
-module(emqx_exhook_sup).
-include("emqx_exhook.hrl").
-behaviour(supervisor).
-export([
@ -28,11 +30,13 @@
stop_grpc_client_channel/1
]).
-define(CHILD(Mod, Type, Args), #{
-define(DEFAULT_TIMEOUT, 5000).
-define(CHILD(Mod, Type, Args, Timeout), #{
id => Mod,
start => {Mod, start_link, Args},
type => Type,
shutdown => 15000
shutdown => Timeout
}).
%%--------------------------------------------------------------------
@ -45,7 +49,7 @@ start_link() ->
init([]) ->
_ = emqx_exhook_metrics:init(),
_ = emqx_exhook_mgr:init_ref_counter_table(),
Mngr = ?CHILD(emqx_exhook_mgr, worker, []),
Mngr = ?CHILD(emqx_exhook_mgr, worker, [], force_shutdown_timeout()),
{ok, {{one_for_one, 10, 100}, [Mngr]}}.
%%--------------------------------------------------------------------
@ -70,3 +74,9 @@ stop_grpc_client_channel(Name) ->
_:_:_ ->
ok
end.
%% Calculate the maximum timeout, which will help to shutdown the
%% emqx_exhook_mgr process correctly.
force_shutdown_timeout() ->
Factor = max(3, length(emqx:get_config([exhook, servers])) + 1),
Factor * ?SERVER_FORCE_SHUTDOWN_TIMEOUT.

35
apps/emqx_exhook/test/emqx_exhook_SUITE.erl
View File

@ -24,6 +24,7 @@
-include_lib("eunit/include/eunit.hrl").
-include_lib("common_test/include/ct.hrl").
-include_lib("emqx/include/emqx_hooks.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(DEFAULT_CLUSTER_NAME_ATOM, emqxcl).
@ -313,6 +314,40 @@ t_cluster_name(_) ->
),
emqx_exhook_mgr:disable(<<"default">>).
t_stop_timeout(_) ->
snabbkaffe:start_trace(),
meck:new(emqx_exhook_demo_svr, [passthrough, no_history]),
meck:expect(
emqx_exhook_demo_svr,
on_provider_unloaded,
fun(Req, Md) ->
%% ensure sleep time greater than emqx_exhook_mgr shutdown timeout
timer:sleep(20000),
meck:passthrough([Req, Md])
end
),
%% stop application
application:stop(emqx_exhook),
?block_until(#{?snk_kind := exhook_mgr_terminated}, 20000),
%% all exhook hooked point should be unloaded
Mods = lists:flatten(
lists:map(
fun({hook, _, Cbs}) ->
lists:map(fun({callback, {M, _, _}, _, _}) -> M end, Cbs)
end,
ets:tab2list(emqx_hooks)
)
),
?assertEqual(false, lists:any(fun(M) -> M == emqx_exhook_handler end, Mods)),
%% ensure started for other tests
emqx_common_test_helpers:start_apps([emqx_exhook]),
snabbkaffe:stop(),
meck:unload(emqx_exhook_demo_svr).
%%--------------------------------------------------------------------
%% Cases Helpers
%%--------------------------------------------------------------------

11
apps/emqx_exhook/test/emqx_exhook_demo_svr.erl
View File

@ -80,7 +80,16 @@ stop() ->
stop(Name) ->
grpc:stop_server(Name),
to_atom_name(Name) ! stop.
case whereis(to_atom_name(Name)) of
undefined ->
ok;
Pid ->
Ref = erlang:monitor(process, Pid),
Pid ! stop,
receive
{'DOWN', Ref, process, Pid, _Reason} -> ok
end
end.
take() ->
to_atom_name(?NAME) ! {take, self()},

39
apps/emqx_gateway/i18n/emqx_gateway_api_authn_i18n.conf
View File

@ -2,70 +2,76 @@ emqx_gateway_api_authn {
get_authn {
desc {
en: """Get the gateway authentication"""
zh: """获取指定网关认证器"""
en: """Gets the configuration of the specified gateway authenticator.</br>
Returns 404 when gateway or authentication is not enabled."""
zh: """获取指定网关认证器的配置
当网关或认证未启用时，返回 404。"""
}
}
update_authn {
desc {
en: """Update authentication for the gateway"""
zh: """更新网关认证器"""
en: """Update the configuration of the specified gateway authenticator, or disable the authenticator."""
zh: """更新指定网关认证器的配置，或停用认证器。"""
}
}
add_authn {
desc {
en: """Add authentication for the gateway"""
zh: """为指定网关新增认证器"""
en: """Enables the authenticator for client authentication for the specified gateway. <\br>
When the authenticator is not configured or turned off, all client connections are assumed to be allowed. <\br>
Note: Only one authenticator is allowed to be enabled at a time in the gateway, rather than allowing multiple authenticators to be configured to form an authentication chain as in MQTT."""
zh: """为指定网关开启认证器实现客户端认证的功能。<\br>
当未配置认证器或关闭认证器时，则认为允许所有客户端的连接。<\br>
注：在网关中仅支持添加一个认证器，而不是像 MQTT 一样允许配置多个认证器构成认证链。"""
}
}
delete_authn {
desc {
en: """Remove the gateway authentication"""
zh: """删除指定网关的认证器"""
en: """Delete the authenticator of the specified gateway."""
zh: """删除指定网关的认证器。"""
}
}
list_users {
desc {
en: """Get the users for the authentication"""
en: """Get the users for the authenticator (only supported by <code>built_in_database</code>)."""
zh: """获取用户列表（仅支持 built_in_database 类型的认证器）"""
}
}
add_user {
desc {
en: """Add user for the authentication"""
en: """Add user for the authenticator (only supports built_in_database)."""
zh: """添加用户（仅支持 built_in_database 类型的认证器）"""
}
}
get_user {
desc {
en: """Get user info from the gateway authentication"""
en: """Get user info from the gateway authenticator (only supports built_in_database)"""
zh: """获取用户信息（仅支持 built_in_database 类型的认证器）"""
}
}
update_user {
desc {
en: """Update the user info for the gateway authentication"""
en: """Update the user info for the gateway authenticator (only supports built_in_database)"""
zh: """更新用户信息（仅支持 built_in_database 类型的认证器）"""
}
}
delete_user {
desc {
en: """Delete the user for the gateway authentication"""
en: """Delete the user for the gateway authenticator (only supports built_in_database)"""
zh: """删除用户（仅支持 built_in_database 类型的认证器）"""
}
}
import_users {
desc {
en: """Import users into the gateway authentication"""
en: """Import users into the gateway authenticator (only supports built_in_database)"""
zh: """导入用户（仅支持 built_in_database 类型的认证器）"""
}
}
@ -79,8 +85,8 @@ emqx_gateway_api_authn {
like_user_id {
desc {
en: """Fuzzy search by user_id (username or clientid)"""
zh: """用户 ID （username 或 clientid）模糊搜索"""
en: """Fuzzy search using user ID (username or clientid), only supports search by substring."""
zh: """使用用户 ID （username 或 clientid）模糊搜索，仅支持按子串的方式进行搜索。"""
}
}
@ -90,5 +96,4 @@ emqx_gateway_api_authn {
zh: """是否是超级用户"""
}
}
}

42
apps/emqx_gateway/i18n/emqx_gateway_api_i18n.conf
View File

@ -2,15 +2,17 @@ emqx_gateway_api {
list_gateway {
desc {
en: """Get gateway list"""
zh: """获取网关列表"""
en: """This API returns an overview info for the specified or all gateways.
including current running status, number of connections, listener status, etc."""
zh: """该接口会返回指定或所有网关的概览状态，
包括当前状态、连接数、监听器状态等。"""
}
}
enable_gateway {
desc {
en: """Enable a gateway"""
zh: """启用某网关"""
en: """Enable a gateway by confs."""
zh: """使用配置启动某一网关。"""
}
}
@ -23,15 +25,17 @@ emqx_gateway_api {
delete_gateway {
desc {
en: """Delete/Unload the gateway"""
zh: """删除/禁用某网关"""
en: """Unload the specified gateway"""
zh: """停用指定网关"""
}
}
update_gateway {
desc {
en: """Update the gateway configurations/status"""
zh: """更新网关配置或启用状态"""
en: """Update the gateway basic configurations and running status.</br>
Note: The Authentication and Listener configurations should be updated by other special APIs. """
zh: """更新指定网关的基础配置、和启用的状态。</br>
注：认证、和监听器的配置更新需参考对应的 API 接口。"""
}
}
@ -42,13 +46,33 @@ emqx_gateway_api {
}
}
gateway_name_in_qs {
desc {
en: """Gateway Name.</br>
It's enum with `stomp`, `mqttsn`, `coap`, `lwm2m`, `exproto`
"""
zh: """网关名称.</br>
可取值为 `stomp`、`mqttsn`、`coap`、`lwm2m`、`exproto`
"""
}
}
gateway_status {
desc {
en: """Gateway Status"""
en: """Gateway status"""
zh: """网关启用状态"""
}
}
gateway_status_in_qs {
desc {
en: """Filter gateways by status.</br>
It is enum with `running`, `stopped`, `unloaded`"""
zh: """通过网关状态筛选</br>
可选值为 `running`、`stopped`、`unloaded`"""
}
}
gateway_created_at {
desc {
en: """The Gateway created datetime"""

53
apps/emqx_gateway/i18n/emqx_gateway_api_listeners_i18n.conf
View File

@ -2,105 +2,109 @@ emqx_gateway_api_listeners {
list_listeners {
desc {
en: """Get the gateway listeners"""
zh: """获取网关监听器列表"""
en: """Gets a list of gateway listeners. This interface returns all the configs of the listener (including the authenticator on that listener), as well as the status of that listener running in the cluster."""
zh: """获取网关监听器列表。该接口会返回监听器所有的配置（包括该监听器上的认证器），同时也会返回该监听器在集群中运行的状态。"""
}
}
add_listener {
desc {
en: """Create the gateway listener"""
zh: """为指定网关添加监听器"""
en: """Create the gateway listener.</br>
Note: For listener types not supported by a gateway, this API returns `400: BAD_REQUEST`."""
zh: """为指定网关添加监听器。</br>
注：对于某网关不支持的监听器类型，该接口会返回 `400: BAD_REQUEST`。"""
}
}
get_listener {
desc {
en: """Get the gateway listener configurations"""
zh: """获取指定监听器信息"""
en: """Get the gateway listener configs"""
zh: """获取指定网关监听器的配置。"""
}
}
delete_listener {
desc {
en: """Delete the gateway listener"""
zh: """删除监听器"""
en: """Delete the gateway listener. All connected clients under the deleted listener will be disconnected."""
zh: """删除指定监听器。被删除的监听器下所有已连接的客户端都会离线。"""
}
}
update_listener {
desc {
en: """Update the gateway listener"""
zh: """更新监听器"""
en: """Update the gateway listener. The listener being updated performs a restart and all clients connected to that listener will be disconnected."""
zh: """更新某网关监听器的配置。被更新的监听器会执行重启，所有已连接到该监听器上的客户端都会被断开。"""
}
}
get_listener_authn {
desc {
en: """Get the listener's authentication info"""
zh: """获取监听器的认证器信息"""
en: """Get the listener's authenticator configs."""
zh: """获取监听器的认证器配置。"""
}
}
add_listener_authn {
desc {
en: """Add authentication for the listener"""
zh: """为指定监听器添加认证器"""
en: """Enable authenticator for specified listener for client authentication.</br>
When authenticator is enabled for a listener, all clients connecting to that listener will use that authenticator for authentication."""
zh: """为指定监听器开启认证器以实现客户端认证的能力。</br>
当某一监听器开启认证后，所有连接到该监听器的客户端会使用该认证器进行认证。"""
}
}
update_listener_authn {
desc {
en: """Update authentication for the listener"""
zh: """更新指定监听上的认证器配置"""
en: """Update authenticator configs for the listener, or disable/enable it."""
zh: """更新指定监听器的认证器配置，或停用/启用该认证器。"""
}
}
delete_listener_authn {
desc {
en: """Remove authentication for the listener"""
zh: """为指定监听器移除认证器"""
en: """Remove authenticator for the listener."""
zh: """移除指定监听器的认证器。"""
}
}
list_users {
desc {
en: """Get the users for the authentication"""
en: """Get the users for the authenticator (only supported by <code>built_in_database</code>)"""
zh: """获取用户列表（仅支持 built_in_database 类型的认证器）"""
}
}
add_user {
desc {
en: """Add user for the authentication"""
en: """Add user for the authenticator (only supports built_in_database)"""
zh: """添加用户（仅支持 built_in_database 类型的认证器）"""
}
}
get_user {
desc {
en: """Get user info from the gateway authentication"""
en: """Get user info from the gateway authenticator (only supports built_in_database)"""
zh: """获取用户信息（仅支持 built_in_database 类型的认证器）"""
}
}
update_user {
desc {
en: """Update the user info for the gateway authentication"""
en: """Update the user info for the gateway authenticator (only supports built_in_database)"""
zh: """更新用户信息（仅支持 built_in_database 类型的认证器）"""
}
}
delete_user {
desc {
en: """Delete the user for the gateway authentication"""
en: """Delete the user for the gateway authenticator (only supports built_in_database)"""
zh: """删除用户（仅支持 built_in_database 类型的认证器）"""
}
}
import_users {
desc {
en: """Import users into the gateway authentication"""
en: """Import users into the gateway authenticator (only supports built_in_database)"""
zh: """导入用户（仅支持 built_in_database 类型的认证器）"""
}
}
@ -139,5 +143,4 @@ emqx_gateway_api_listeners {
zh: """当前连接数"""
}
}
}

31
apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl
View File

@ -19,6 +19,7 @@
-include_lib("emqx/include/types.hrl").
-include_lib("emqx/include/logger.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
%% API
-export([
@ -51,6 +52,9 @@
%% Internal callback
-export([wakeup_from_hib/2, recvloop/2]).
%% for channel module
-export([keepalive_stats/1]).
-record(state, {
%% TCP/SSL/UDP/DTLS Wrapped Socket
socket :: {esockd_transport, esockd:socket()} | {udp, _, _},
@ -240,6 +244,11 @@ esockd_send(Data, #state{
esockd_send(Data, #state{socket = {esockd_transport, Sock}}) ->
esockd_transport:async_send(Sock, Data).
keepalive_stats(recv) ->
emqx_pd:get_counter(recv_pkt);
keepalive_stats(send) ->
emqx_pd:get_counter(send_pkt).
is_datadram_socket({esockd_transport, _}) -> false;
is_datadram_socket({udp, _, _}) -> true.
@ -568,9 +577,15 @@ terminate(
channel = Channel
}
) ->
?SLOG(debug, #{msg => "conn_process_terminated", reason => Reason}),
_ = ChannMod:terminate(Reason, Channel),
_ = close_socket(State),
ClientId =
try ChannMod:info(clientid, Channel) of
Id -> Id
catch
_:_ -> undefined
end,
?tp(debug, conn_process_terminated, #{reason => Reason, clientid => ClientId}),
exit(Reason).
%%--------------------------------------------------------------------
@ -635,28 +650,22 @@ handle_timeout(
Keepalive,
State = #state{
chann_mod = ChannMod,
socket = Socket,
channel = Channel
}
) when
Keepalive == keepalive;
Keepalive == keepalive_send
->
Stat =
StatVal =
case Keepalive of
keepalive -> recv_oct;
keepalive_send -> send_oct
keepalive -> keepalive_stats(recv);
keepalive_send -> keepalive_stats(send)
end,
case ChannMod:info(conn_state, Channel) of
disconnected ->
{ok, State};
_ ->
case esockd_getstat(Socket, [Stat]) of
{ok, [{Stat, RecvOct}]} ->
handle_timeout(TRef, {Keepalive, RecvOct}, State);
{error, Reason} ->
handle_info({sock_error, Reason}, State)
end
handle_timeout(TRef, {Keepalive, StatVal}, State)
end;
handle_timeout(
_TRef,

3
apps/emqx_gateway/src/coap/emqx_coap_api.erl
View File

@ -48,8 +48,9 @@ schema(?PREFIX ++ "/request") ->
#{
operationId => request,
post => #{
tags => [<<"CoAP">>],
tags => [<<"Gateways">>],
desc => ?DESC(send_coap_request),
summary => <<"Send a Request to a Client">>,
parameters => request_parameters(),
requestBody => request_body(),
responses => #{

2
apps/emqx_gateway/src/emqx_gateway.app.src
View File

@ -1,7 +1,7 @@
%% -*- mode: erlang -*-
{application, emqx_gateway, [
{description, "The Gateway management application"},
{vsn, "0.1.5"},
{vsn, "0.1.6"},
{registered, []},
{mod, {emqx_gateway_app, []}},
{applications, [kernel, stdlib, grpc, emqx, emqx_authn]},

16
apps/emqx_gateway/src/emqx_gateway_api.erl
View File

@ -164,7 +164,9 @@ schema("/gateways") ->
'operationId' => gateway,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(list_gateway),
summary => <<"List All Gateways">>,
parameters => params_gateway_status_in_qs(),
responses =>
?STANDARD_RESP(
@ -178,7 +180,9 @@ schema("/gateways") ->
},
post =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(enable_gateway),
summary => <<"Enable a Gateway">>,
%% TODO: distinguish create & response swagger schema
'requestBody' => schema_gateways_conf(),
responses =>
@ -190,21 +194,27 @@ schema("/gateways/:name") ->
'operationId' => gateway_insta,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(get_gateway),
summary => <<"Get the Gateway">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(#{200 => schema_gateways_conf()})
},
delete =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(delete_gateway),
summary => <<"Unload the gateway">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(#{204 => <<"Deleted">>})
},
put =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(update_gateway),
summary => <<"Update the gateway confs">>,
parameters => params_gateway_name_in_path(),
'requestBody' => schema_update_gateways_conf(),
responses =>
@ -224,8 +234,8 @@ params_gateway_name_in_path() ->
binary(),
#{
in => path,
desc => ?DESC(gateway_name),
example => <<"">>
desc => ?DESC(gateway_name_in_qs),
example => <<"stomp">>
}
)}
].
@ -239,7 +249,7 @@ params_gateway_status_in_qs() ->
#{
in => query,
required => false,
desc => ?DESC(gateway_status),
desc => ?DESC(gateway_status_in_qs),
example => <<"">>
}
)}

29
apps/emqx_gateway/src/emqx_gateway_api_authn.erl
View File

@ -75,6 +75,7 @@ authn(get, #{bindings := #{name := Name0}}) ->
Authn -> {200, Authn}
catch
error:{config_not_found, _} ->
%% FIXME: should return 404?
{204}
end
end);
@ -181,19 +182,23 @@ schema("/gateways/:name/authentication") ->
'operationId' => authn,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(get_authn),
summary => <<"Get Authenticator Configuration">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(
#{
200 => schema_authn(),
204 => <<"Authentication does not initiated">>
204 => <<"Authenticator doesn't initiated">>
}
)
},
put =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(update_authn),
summary => <<"Update Authenticator Configuration">>,
parameters => params_gateway_name_in_path(),
'requestBody' => schema_authn(),
responses =>
@ -201,7 +206,9 @@ schema("/gateways/:name/authentication") ->
},
post =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(add_authn),
summary => <<"Create an Authenticator for a Gateway">>,
parameters => params_gateway_name_in_path(),
'requestBody' => schema_authn(),
responses =>
@ -209,7 +216,9 @@ schema("/gateways/:name/authentication") ->
},
delete =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(delete_authn),
summary => <<"Delete the Gateway Authenticator">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(#{204 => <<"Deleted">>})
@ -220,7 +229,9 @@ schema("/gateways/:name/authentication/users") ->
'operationId' => users,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(list_users),
summary => <<"List users for a Gateway Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_paging_in_qs() ++
params_fuzzy_in_qs(),
@ -236,7 +247,9 @@ schema("/gateways/:name/authentication/users") ->
},
post =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(add_user),
summary => <<"Add User for a Gateway Authenticator">>,
parameters => params_gateway_name_in_path(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
ref(emqx_authn_api, request_user_create),
@ -258,7 +271,9 @@ schema("/gateways/:name/authentication/users/:uid") ->
'operationId' => users_insta,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(get_user),
summary => <<"Get User Info for a Gateway Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_userid_in_path(),
responses =>
@ -273,7 +288,9 @@ schema("/gateways/:name/authentication/users/:uid") ->
},
put =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(update_user),
summary => <<"Update User Info for a Gateway Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_userid_in_path(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@ -292,7 +309,9 @@ schema("/gateways/:name/authentication/users/:uid") ->
},
delete =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(delete_user),
summary => <<"Delete User for a Gateway Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_userid_in_path(),
responses =>
@ -311,8 +330,8 @@ params_gateway_name_in_path() ->
binary(),
#{
in => path,
desc => ?DESC(emqx_gateway_api, gateway_name),
example => <<"">>
desc => ?DESC(emqx_gateway_api, gateway_name_in_qs),
example => <<"stomp">>
}
)}
].
@ -325,7 +344,7 @@ params_userid_in_path() ->
#{
in => path,
desc => ?DESC(user_id),
example => <<"">>
example => <<"test_username">>
}
)}
].
@ -343,7 +362,7 @@ params_fuzzy_in_qs() ->
in => query,
required => false,
desc => ?DESC(like_user_id),
example => <<"username">>
example => <<"test_">>
}
)},
{is_superuser,

7
apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl
View File

@ -122,7 +122,9 @@ schema("/gateways/:name/authentication/import_users") ->
'operationId' => import_users,
post =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(emqx_gateway_api_authn, import_users),
summary => <<"Import Users">>,
parameters => params_gateway_name_in_path(),
'requestBody' => emqx_dashboard_swagger:file_schema(filename),
responses =>
@ -134,7 +136,9 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") ->
'operationId' => import_listener_users,
post =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(emqx_gateway_api_listeners, import_users),
summary => <<"Import Users">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
'requestBody' => emqx_dashboard_swagger:file_schema(filename),
@ -144,6 +148,7 @@ schema("/gateways/:name/listeners/:id/authentication/import_users") ->
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
%%--------------------------------------------------------------------
%% params defines
%%--------------------------------------------------------------------
@ -155,7 +160,7 @@ params_gateway_name_in_path() ->
binary(),
#{
in => path,
desc => ?DESC(emqx_gateway_api, gateway_name),
desc => ?DESC(emqx_gateway_api, gateway_name_in_qs),
example => <<"stomp">>
}
)}

12
apps/emqx_gateway/src/emqx_gateway_api_clients.erl
View File

@ -467,7 +467,9 @@ schema("/gateways/:name/clients") ->
'operationId' => clients,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(list_clients),
summary => <<"List Gateway's Clients">>,
parameters => params_client_query(),
responses =>
?STANDARD_RESP(#{200 => schema_client_list()})
@ -478,14 +480,18 @@ schema("/gateways/:name/clients/:clientid") ->
'operationId' => clients_insta,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(get_client),
summary => <<"Get Client Info">>,
parameters => params_client_insta(),
responses =>
?STANDARD_RESP(#{200 => schema_client()})
},
delete =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(kick_client),
summary => <<"Kick out Client">>,
parameters => params_client_insta(),
responses =>
?STANDARD_RESP(#{204 => <<"Kicked">>})
@ -496,7 +502,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions") ->
'operationId' => subscriptions,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(list_subscriptions),
summary => <<"List Client's Subscription">>,
parameters => params_client_insta(),
responses =>
?STANDARD_RESP(
@ -510,7 +518,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions") ->
},
post =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(add_subscription),
summary => <<"Add Subscription for Client">>,
parameters => params_client_insta(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
ref(subscription),
@ -532,7 +542,9 @@ schema("/gateways/:name/clients/:clientid/subscriptions/:topic") ->
'operationId' => subscriptions,
delete =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(delete_subscription),
summary => <<"Delete Client's Subscription">>,
parameters => params_topic_name_in_path() ++ params_client_insta(),
responses =>
?STANDARD_RESP(#{204 => <<"Unsubscribed">>})

33
apps/emqx_gateway/src/emqx_gateway_api_listeners.erl
View File

@ -358,7 +358,9 @@ schema("/gateways/:name/listeners") ->
'operationId' => listeners,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(list_listeners),
summary => <<"List All Listeners">>,
parameters => params_gateway_name_in_path(),
responses =>
?STANDARD_RESP(
@ -372,7 +374,9 @@ schema("/gateways/:name/listeners") ->
},
post =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(add_listener),
summary => <<"Add a Listener">>,
parameters => params_gateway_name_in_path(),
%% XXX: How to distinguish the different listener supported by
%% different types of gateways?
@ -396,7 +400,9 @@ schema("/gateways/:name/listeners/:id") ->
'operationId' => listeners_insta,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(get_listener),
summary => <<"Get the Listener Configs">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
responses =>
@ -411,7 +417,9 @@ schema("/gateways/:name/listeners/:id") ->
},
delete =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(delete_listener),
summary => <<"Delete the Listener">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
responses =>
@ -419,7 +427,9 @@ schema("/gateways/:name/listeners/:id") ->
},
put =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(update_listener),
summary => <<"Update the Listener Configs">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@ -442,7 +452,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
'operationId' => listeners_insta_authn,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(get_listener_authn),
summary => <<"Get the Listener's Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
responses =>
@ -455,7 +467,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
},
post =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(add_listener_authn),
summary => <<"Create an Authenticator for a Listener">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
'requestBody' => schema_authn(),
@ -464,7 +478,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
},
put =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(update_listener_authn),
summary => <<"Update the Listener Authenticator configs">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
'requestBody' => schema_authn(),
@ -473,7 +489,9 @@ schema("/gateways/:name/listeners/:id/authentication") ->
},
delete =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(delete_listener_authn),
summary => <<"Delete the Listener's Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
responses =>
@ -485,7 +503,9 @@ schema("/gateways/:name/listeners/:id/authentication/users") ->
'operationId' => users,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(list_users),
summary => <<"List Authenticator's Users">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path() ++
params_paging_in_qs(),
@ -501,7 +521,9 @@ schema("/gateways/:name/listeners/:id/authentication/users") ->
},
post =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(add_user),
summary => <<"Add User for an Authenticator">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path(),
'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@ -524,7 +546,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
'operationId' => users_insta,
get =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(get_user),
summary => <<"Get User Info">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path() ++
params_userid_in_path(),
@ -540,7 +564,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
},
put =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(update_user),
summary => <<"Update User Info">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path() ++
params_userid_in_path(),
@ -560,7 +586,9 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
},
delete =>
#{
tags => [<<"Gateways">>],
desc => ?DESC(delete_user),
summary => <<"Delete User">>,
parameters => params_gateway_name_in_path() ++
params_listener_id_in_path() ++
params_userid_in_path(),
@ -570,6 +598,7 @@ schema("/gateways/:name/listeners/:id/authentication/users/:uid") ->
};
schema(Path) ->
emqx_gateway_utils:make_compatible_schema(Path, fun schema/1).
%%--------------------------------------------------------------------
%% params defines
@ -580,8 +609,8 @@ params_gateway_name_in_path() ->
binary(),
#{
in => path,
desc => ?DESC(emqx_gateway_api, gateway_name),
example => <<"">>
desc => ?DESC(emqx_gateway_api, gateway_name_in_qs),
example => <<"stomp">>
}
)}
].

2
apps/emqx_gateway/src/emqx_gateway_schema.erl
View File

@ -28,7 +28,7 @@
-include_lib("hocon/include/hoconsc.hrl").
-include_lib("typerefl/include/types.hrl").
-type ip_port() :: tuple().
-type ip_port() :: tuple() | integer().
-type duration() :: non_neg_integer().
-type duration_s() :: non_neg_integer().
-type bytesize() :: pos_integer().

62
apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl
View File

@ -78,7 +78,8 @@
-define(TIMER_TABLE, #{
alive_timer => keepalive,
force_timer => force_close
force_timer => force_close,
idle_timer => force_close_idle
}).
-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]).
@ -151,14 +152,17 @@ init(
Ctx = maps:get(ctx, Options),
GRpcChann = maps:get(handler, Options),
PoolName = maps:get(pool_name, Options),
NConnInfo = default_conninfo(ConnInfo),
IdleTimeout = emqx_gateway_utils:idle_timeout(Options),
NConnInfo = default_conninfo(ConnInfo#{idle_timeout => IdleTimeout}),
ListenerId =
case maps:get(listener, Options, undefined) of
undefined -> undefined;
{GwName, Type, LisName} -> emqx_gateway_utils:listener_id(GwName, Type, LisName)
end,
EnableAuthn = maps:get(enable_authn, Options, true),
DefaultClientInfo = default_clientinfo(ConnInfo),
DefaultClientInfo = default_clientinfo(NConnInfo),
ClientInfo = DefaultClientInfo#{
listener => ListenerId,
enable_authn => EnableAuthn
@ -183,7 +187,9 @@ init(
}
)
},
try_dispatch(on_socket_created, wrap(Req), Channel).
start_idle_checking_timer(
try_dispatch(on_socket_created, wrap(Req), Channel)
).
%% @private
peercert(NoSsl, ConnInfo) when
@ -217,6 +223,12 @@ socktype(dtls) -> 'DTLS'.
address({Host, Port}) ->
#{host => inet:ntoa(Host), port => Port}.
%% avoid udp connection process leak
start_idle_checking_timer(Channel = #channel{conninfo = #{socktype := udp}}) ->
ensure_timer(idle_timer, Channel);
start_idle_checking_timer(Channel) ->
Channel.
%%--------------------------------------------------------------------
%% Handle incoming packet
%%--------------------------------------------------------------------
@ -285,10 +297,15 @@ handle_timeout(
{ok, reset_timer(alive_timer, NChannel)};
{error, timeout} ->
Req = #{type => 'KEEPALIVE'},
{ok, try_dispatch(on_timer_timeout, wrap(Req), Channel)}
NChannel = remove_timer_ref(alive_timer, Channel),
%% close connection if keepalive timeout
Replies = [{event, disconnected}, {close, keepalive_timeout}],
{ok, Replies, try_dispatch(on_timer_timeout, wrap(Req), NChannel)}
end;
handle_timeout(_TRef, force_close, Channel = #channel{closed_reason = Reason}) ->
{shutdown, {error, {force_close, Reason}}, Channel};
handle_timeout(_TRef, force_close_idle, Channel) ->
{shutdown, idle_timeout, Channel};
handle_timeout(_TRef, Msg, Channel) ->
?SLOG(warning, #{
msg => "unexpected_timeout_signal",
@ -390,7 +407,7 @@ handle_call(
NConnInfo = ConnInfo#{keepalive => Interval},
NClientInfo = ClientInfo#{keepalive => Interval},
NChannel = Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo},
{reply, ok, ensure_keepalive(NChannel)};
{reply, ok, [{event, updated}], ensure_keepalive(cancel_timer(idle_timer, NChannel))};
handle_call(
{subscribe_from_client, TopicFilter, Qos},
_From,
@ -405,21 +422,21 @@ handle_call(
{reply, {error, ?RESP_PERMISSION_DENY, <<"Authorization deny">>}, Channel};
_ ->
{ok, _, NChannel} = do_subscribe([{TopicFilter, #{qos => Qos}}], Channel),
{reply, ok, NChannel}
{reply, ok, [{event, updated}], NChannel}
end;
handle_call({subscribe, Topic, SubOpts}, _From, Channel) ->
{ok, [{NTopicFilter, NSubOpts}], NChannel} = do_subscribe([{Topic, SubOpts}], Channel),
{reply, {ok, {NTopicFilter, NSubOpts}}, NChannel};
{reply, {ok, {NTopicFilter, NSubOpts}}, [{event, updated}], NChannel};
handle_call(
{unsubscribe_from_client, TopicFilter},
_From,
Channel = #channel{conn_state = connected}
) ->
{ok, NChannel} = do_unsubscribe([{TopicFilter, #{}}], Channel),
{reply, ok, NChannel};
{reply, ok, [{event, updated}], NChannel};
handle_call({unsubscribe, Topic}, _From, Channel) ->
{ok, NChannel} = do_unsubscribe([Topic], Channel),
{reply, ok, NChannel};
{reply, ok, [{event, update}], NChannel};
handle_call(subscriptions, _From, Channel = #channel{subscriptions = Subs}) ->
{reply, {ok, maps:to_list(Subs)}, Channel};
handle_call(
@ -446,7 +463,7 @@ handle_call(
{reply, ok, Channel}
end;
handle_call(kick, _From, Channel) ->
{shutdown, kicked, ok, ensure_disconnected(kicked, Channel)};
{reply, ok, [{event, disconnected}, {close, kicked}], Channel};
handle_call(discard, _From, Channel) ->
{shutdown, discarded, ok, Channel};
handle_call(Req, _From, Channel) ->
@ -648,7 +665,8 @@ ensure_keepalive(Channel = #channel{clientinfo = ClientInfo}) ->
ensure_keepalive_timer(Interval, Channel) when Interval =< 0 ->
Channel;
ensure_keepalive_timer(Interval, Channel) ->
Keepalive = emqx_keepalive:init(timer:seconds(Interval)),
StatVal = emqx_gateway_conn:keepalive_stats(recv),
Keepalive = emqx_keepalive:init(StatVal, timer:seconds(Interval)),
ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}).
ensure_timer(Name, Channel = #channel{timers = Timers}) ->
@ -666,11 +684,17 @@ ensure_timer(Name, Time, Channel = #channel{timers = Timers}) ->
Channel#channel{timers = Timers#{Name => TRef}}.
reset_timer(Name, Channel) ->
ensure_timer(Name, clean_timer(Name, Channel)).
ensure_timer(Name, remove_timer_ref(Name, Channel)).
clean_timer(Name, Channel = #channel{timers = Timers}) ->
cancel_timer(Name, Channel = #channel{timers = Timers}) ->
emqx_misc:cancel_timer(maps:get(Name, Timers, undefined)),
remove_timer_ref(Name, Channel).
remove_timer_ref(Name, Channel = #channel{timers = Timers}) ->
Channel#channel{timers = maps:remove(Name, Timers)}.
interval(idle_timer, #channel{conninfo = #{idle_timeout := IdleTimeout}}) ->
IdleTimeout;
interval(force_timer, _) ->
15000;
interval(alive_timer, #channel{keepalive = Keepalive}) ->
@ -725,7 +749,7 @@ enrich_clientinfo(InClientInfo = #{proto_name := ProtoName}, ClientInfo) ->
default_conninfo(ConnInfo) ->
ConnInfo#{
clean_start => true,
clientid => undefined,
clientid => anonymous_clientid(),
username => undefined,
conn_props => #{},
connected => true,
@ -739,14 +763,15 @@ default_conninfo(ConnInfo) ->
default_clientinfo(#{
peername := {PeerHost, _},
sockname := {_, SockPort}
sockname := {_, SockPort},
clientid := ClientId
}) ->
#{
zone => default,
protocol => exproto,
peerhost => PeerHost,
sockport => SockPort,
clientid => undefined,
clientid => ClientId,
username => undefined,
is_bridge => false,
is_superuser => false,
@ -764,3 +789,6 @@ proto_name_to_protocol(<<>>) ->
exproto;
proto_name_to_protocol(ProtoName) when is_binary(ProtoName) ->
binary_to_atom(ProtoName).
anonymous_clientid() ->
iolist_to_binary(["exproto-", emqx_misc:gen_id()]).

10
apps/emqx_gateway/src/exproto/emqx_exproto_gcli.erl
View File

@ -56,12 +56,19 @@ start_link(Pool, Id) ->
[]
).
-spec async_call(atom(), map(), map()) -> ok.
async_call(
FunName,
Req = #{conn := Conn},
Options = #{pool_name := PoolName}
) ->
cast(pick(PoolName, Conn), {rpc, FunName, Req, Options, self()}).
case pick(PoolName, Conn) of
false ->
reply(self(), FunName, {error, no_available_grpc_client});
Pid when is_pid(Pid) ->
cast(Pid, {rpc, FunName, Req, Options, self()})
end,
ok.
%%--------------------------------------------------------------------
%% cast, pick
@ -72,6 +79,7 @@ async_call(
cast(Deliver, Msg) ->
gen_server:cast(Deliver, Msg).
-spec pick(term(), term()) -> pid() | false.
pick(PoolName, Conn) ->
gproc_pool:pick_worker(PoolName, Conn).

12
apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl
View File

@ -45,8 +45,9 @@ schema(?PATH("/lookup")) ->
#{
'operationId' => lookup,
get => #{
tags => [<<"LwM2M">>],
tags => [<<"Gateways">>],
desc => ?DESC(lookup_resource),
summary => <<"List Client's Resources">>,
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
{path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})},
@ -69,8 +70,9 @@ schema(?PATH("/observe")) ->
#{
'operationId' => observe,
post => #{
tags => [<<"LwM2M">>],
tags => [<<"Gateways">>],
desc => ?DESC(observe_resource),
summary => <<"Observe a Resource">>,
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
{path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})},
@ -87,8 +89,9 @@ schema(?PATH("/read")) ->
#{
'operationId' => read,
post => #{
tags => [<<"LwM2M">>],
tags => [<<"Gateways">>],
desc => ?DESC(read_resource),
summary => <<"Read Value from a Resource Path">>,
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
{path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})}
@ -104,7 +107,8 @@ schema(?PATH("/write")) ->
'operationId' => write,
post => #{
desc => ?DESC(write_resource),
tags => [<<"LwM2M">>],
tags => [<<"Gateways">>],
summary => <<"Write a Value to Resource Path">>,
parameters => [
{clientid, mk(binary(), #{in => path, example => "urn:oma:lwm2m:oma:2"})},
{path, mk(binary(), #{in => query, required => true, example => "/3/0/7"})},

101
apps/emqx_gateway/test/emqx_exproto_SUITE.erl
View File

@ -20,6 +20,7 @@
-compile(nowarn_export_all).
-include_lib("emqx/include/emqx_hooks.hrl").
-include_lib("eunit/include/eunit.hrl").
-import(
emqx_exproto_echo_svr,
@ -38,6 +39,7 @@
-include_lib("emqx/include/emqx.hrl").
-include_lib("emqx/include/emqx_mqtt.hrl").
-include_lib("snabbkaffe/include/snabbkaffe.hrl").
-define(TCPOPTS, [binary, {active, false}]).
-define(DTLSOPTS, [binary, {active, false}, {protocol, dtls}]).
@ -62,6 +64,9 @@
all() ->
[{group, Name} || Name <- metrics()].
suite() ->
[{timetrap, {seconds, 30}}].
groups() ->
Cases = emqx_common_test_helpers:all(?MODULE),
[{Name, Cases} || Name <- metrics()].
@ -87,6 +92,7 @@ set_special_cfg(emqx_gateway) ->
[gateway, exproto],
#{
server => #{bind => 9100},
idle_timeout => 5000,
handler => #{address => "http://127.0.0.1:9001"},
listeners => listener_confs(LisType)
}
@ -223,14 +229,16 @@ t_acl_deny(Cfg) ->
close(Sock).
t_keepalive_timeout(Cfg) ->
ok = snabbkaffe:start_trace(),
SockType = proplists:get_value(listener_type, Cfg),
Sock = open(SockType),
ClientId1 = <<"keepalive_test_client1">>,
Client = #{
proto_name => <<"demo">>,
proto_ver => <<"v0.1">>,
clientid => <<"test_client_1">>,
keepalive => 2
clientid => ClientId1,
keepalive => 5
},
Password = <<"123456">>,
@ -238,16 +246,42 @@ t_keepalive_timeout(Cfg) ->
ConnAckBin = frame_connack(0),
send(Sock, ConnBin),
{ok, ConnAckBin} = recv(Sock, 5000),
{ok, ConnAckBin} = recv(Sock),
DisconnectBin = frame_disconnect(),
{ok, DisconnectBin} = recv(Sock, 10000),
SockType =/= udp andalso
begin
{error, closed} = recv(Sock, 5000)
case SockType of
udp ->
%% another udp client should not affect the first
%% udp client keepalive check
timer:sleep(4000),
Sock2 = open(SockType),
ConnBin2 = frame_connect(
Client#{clientid => <<"keepalive_test_client2">>},
Password
),
send(Sock2, ConnBin2),
%% first client will be keepalive timeouted in 6s
?assertMatch(
{ok, #{
clientid := ClientId1,
reason := {shutdown, {sock_closed, keepalive_timeout}}
}},
?block_until(#{?snk_kind := conn_process_terminated}, 8000)
);
_ ->
?assertMatch(
{ok, #{
clientid := ClientId1,
reason := {shutdown, {sock_closed, keepalive_timeout}}
}},
?block_until(#{?snk_kind := conn_process_terminated}, 12000)
),
Trace = snabbkaffe:collect_trace(),
%% conn process should be terminated
?assertEqual(1, length(?of_kind(conn_process_terminated, Trace))),
%% socket port should be closed
?assertEqual({error, closed}, recv(Sock, 5000))
end,
ok.
snabbkaffe:stop().
t_hook_connected_disconnected(Cfg) ->
SockType = proplists:get_value(listener_type, Cfg),
@ -337,6 +371,8 @@ t_hook_session_subscribed_unsubscribed(Cfg) ->
error(hook_is_not_running)
end,
send(Sock, frame_disconnect()),
close(Sock),
emqx_hooks:del('session.subscribed', {?MODULE, hook_fun3}),
emqx_hooks:del('session.unsubscribed', {?MODULE, hook_fun4}).
@ -373,6 +409,48 @@ t_hook_message_delivered(Cfg) ->
close(Sock),
emqx_hooks:del('message.delivered', {?MODULE, hook_fun5}).
t_idle_timeout(Cfg) ->
ok = snabbkaffe:start_trace(),
SockType = proplists:get_value(listener_type, Cfg),
Sock = open(SockType),
%% need to create udp client by sending something
case SockType of
udp ->
%% nothing to do
ok = meck:new(emqx_exproto_gcli, [passthrough, no_history]),
ok = meck:expect(
emqx_exproto_gcli,
async_call,
fun(FunName, _Req, _GClient) ->
self() ! {hreply, FunName, ok},
ok
end
),
%% send request, but nobody can respond to it
ClientId = <<"idle_test_client1">>,
Client = #{
proto_name => <<"demo">>,
proto_ver => <<"v0.1">>,
clientid => ClientId,
keepalive => 5
},
Password = <<"123456">>,
ConnBin = frame_connect(Client, Password),
send(Sock, ConnBin),
?assertMatch(
{ok, #{reason := {shutdown, idle_timeout}}},
?block_until(#{?snk_kind := conn_process_terminated}, 10000)
),
ok = meck:unload(emqx_exproto_gcli);
_ ->
?assertMatch(
{ok, #{reason := {shutdown, idle_timeout}}},
?block_until(#{?snk_kind := conn_process_terminated}, 10000)
)
end,
snabbkaffe:stop().
%%--------------------------------------------------------------------
%% Utils
@ -422,6 +500,9 @@ send({ssl, Sock}, Bin) ->
send({dtls, Sock}, Bin) ->
ssl:send(Sock, Bin).
recv(Sock) ->
recv(Sock, infinity).
recv({tcp, Sock}, Ts) ->
gen_tcp:recv(Sock, 0, Ts);
recv({udp, Sock}, Ts) ->

48
apps/emqx_management/i18n/emqx_mgmt_api_alarms_i18n.conf
View File

@ -2,78 +2,82 @@ emqx_mgmt_api_alarms {
list_alarms_api {
desc {
en: """List alarms"""
zh: """列出告警，获取告警列表"""
en: """List currently activated alarms or historical alarms, determined by query parameters."""
zh: """列出当前激活的告警或历史告警，由查询参数决定。"""
}
}
delete_alarms_api {
desc {
en: """Remove all deactivated alarms"""
zh: """删除所有历史告警（非活跃告警）"""
en: """Remove all historical alarms."""
zh: """删除所有历史告警。"""
}
}
delete_alarms_api_response204 {
desc {
en: """Remove all deactivated alarms ok"""
zh: """删除所有历史告警（非活跃告警）成功"""
en: """Historical alarms have been cleared successfully."""
zh: """历史告警已成功清除。"""
}
}
get_alarms_qs_activated {
desc {
en: """Activate alarms, or deactivate alarms. Default is false"""
zh: """活跃中的告警，或历史告警（非活跃告警），默认为 false"""
en: """It is used to specify the alarm type of the query.
When true, it returns the currently activated alarm,
and when it is false, it returns the historical alarm.
The default is false."""
zh: """用于指定查询的告警类型，
为 true 时返回当前激活的告警，为 false 时返回历史告警，默认为 false。"""
}
}
node {
desc {
en: """Alarm in node"""
zh: """告警节点名称"""
en: """The name of the node that triggered this alarm."""
zh: """触发此告警的节点名称。"""
}
}
name {
desc {
en: """Alarm name"""
zh: """告警名称"""
en: """Alarm name, used to distinguish different alarms."""
zh: """告警名称，用于区分不同的告警。"""
}
}
message {
desc {
en: """Alarm readable information"""
zh: """告警信息"""
en: """Alarm message, which describes the alarm content in a human-readable format."""
zh: """告警消息，以人类可读的方式描述告警内容。"""
}
}
details {
desc {
en: """Alarm details information"""
zh: """告警详细信息"""
en: """Alarm details, provides more alarm information, mainly for program processing."""
zh: """告警详情，提供了更多的告警信息，主要提供给程序处理。"""
}
}
duration {
desc {
en: """Alarms duration time; UNIX time stamp, millisecond"""
zh: """告警持续时间，单位：毫秒"""
en: """Indicates how long the alarm has lasted, in milliseconds."""
zh: """表明告警已经持续了多久，单位：毫秒。"""
}
}
activate_at {
desc {
en: """Alarms activate time, RFC 3339"""
zh: """告警开始时间，使用 rfc3339 标准时间格式"""
en: """Alarm start time, using rfc3339 standard time format."""
zh: """告警开始时间，使用 rfc3339 标准时间格式。"""
}
}
deactivate_at {
desc {
en: """Alarms deactivate time, RFC 3339"""
zh: """告警结束时间，使用 rfc3339 标准时间格式"""
en: """Alarm end time, using rfc3339 standard time format."""
zh: """告警结束时间，使用 rfc3339 标准时间格式。"""
}
}

83
apps/emqx_management/i18n/emqx_mgmt_api_banned_i18n.conf
View File

@ -2,112 +2,97 @@ emqx_mgmt_api_banned {
list_banned_api {
desc {
en: """List banned."""
zh: """列出黑名单"""
}
label {
en: """List Banned"""
zh: """列出黑名单"""
en: """List all currently banned client IDs, usernames and IP addresses."""
zh: """列出目前所有被封禁的客户端 ID、用户名和 IP 地址。"""
}
}
create_banned_api {
desc {
en: """Create banned."""
zh: """创建黑名单"""
en: """Add a client ID, username or IP address to the blacklist."""
zh: """添加一个客户端 ID、用户名或者 IP 地址到黑名单。"""
}
}
create_banned_api_response400 {
desc {
en: """Banned already existed, or bad args."""
zh: """黑名单已存在，或参数格式有错误"""
en: """Bad request, possibly due to wrong parameters or the existence of a banned object."""
zh: """错误的请求，可能是参数错误或封禁对象已存在等原因。"""
}
}
delete_banned_api {
desc {
en: """Delete banned"""
zh: """删除黑名单"""
en: """Remove a client ID, username or IP address from the blacklist."""
zh: """将一个客户端 ID、用户名或者 IP 地址从黑名单中删除。"""
}
}
delete_banned_api_response404 {
desc {
en: """Banned not found. May be the banned time has been exceeded"""
zh: """黑名单未找到，可能为已经超期失效"""
}
}
create_banned {
desc {
en: """List banned."""
zh: """列出黑名单"""
}
label {
en: """List Banned"""
zh: """列出黑名单"""
en: """The banned object was not found in the blacklist."""
zh: """未在黑名单中找到该封禁对象。"""
}
}
as {
desc {
en: """Banned type clientid, username, peerhost"""
zh: """黑名单类型，可选 clientid、username、peerhost"""
en: """Ban method, which can be client ID, username or IP address."""
zh: """封禁方式，可以通过客户端 ID、用户名或者 IP 地址等方式进行封禁。"""
}
label {
en: """Banned Type"""
zh: """黑名单类型"""
en: """Ban Method"""
zh: """封禁方式"""
}
}
who {
desc {
en: """Client info as banned type"""
zh: """设备信息"""
en: """Ban object, specific client ID, username or IP address."""
zh: """封禁对象，具体的客户端 ID、用户名或者 IP 地址。"""
}
label {
en: """Banned Info"""
zh: """黑名单信息"""
en: """Ban Object"""
zh: """封禁对象"""
}
}
by {
desc {
en: """Commander"""
zh: """黑名单创建者"""
en: """Initiator of the ban."""
zh: """封禁的发起者。"""
}
label {
en: """Commander"""
zh: """黑名单创建者"""
en: """Ban Initiator"""
zh: """封禁发起者"""
}
}
reason {
desc {
en: """Banned reason"""
zh: """黑名单创建原因"""
en: """Ban reason, record the reason why the current object was banned."""
zh: """封禁原因，记录当前对象被封禁的原因。"""
}
label {
en: """Reason"""
zh: """原因"""
en: """Ban Reason"""
zh: """封禁原因"""
}
}
at {
desc {
en: """Create banned time, rfc3339, now if not specified"""
zh: """黑名单创建时间，默认为当前"""
en: """The start time of the ban, the format is rfc3339, the default is the time when the operation was initiated."""
zh: """封禁的起始时间，格式为 rfc3339，默认为发起操作的时间。"""
}
label {
en: """Create banned time"""
zh: """黑名单创建时间"""
en: """Ban Time"""
zh: """封禁时间"""
}
}
until {
desc {
en: """Cancel banned time, rfc3339, now + 5 minute if not specified"""
zh: """黑名单结束时间，默认为创建时间 + 5 分钟"""
en: """The end time of the ban, the format is rfc3339, the default is the time when the operation was initiated + 5 minutes."""
zh: """封禁的结束时间，式为 rfc3339，默认为发起操作的时间 + 5 分钟。"""
}
label {
en: """Cancel banned time"""
zh: """黑名单结束时间"""
en: """Ban End Time"""
zh: """封禁结束时间"""
}
}
}

2
apps/emqx_management/src/emqx_management.app.src
View File

@ -2,7 +2,7 @@
{application, emqx_management, [
{description, "EMQX Management API and CLI"},
% strict semver, bump manually!
{vsn, "5.0.5"},
{vsn, "5.0.6"},
{modules, []},
{registered, [emqx_management_sup]},
{applications, [kernel, stdlib, emqx_plugins, minirest, emqx]},

2
apps/emqx_management/src/emqx_mgmt_api_app.erl
View File

@ -22,7 +22,7 @@
-export([api_spec/0, fields/1, paths/0, schema/1, namespace/0]).
-export([api_key/2, api_key_by_name/2]).
-export([validate_name/1]).
-define(TAGS, [<<"API keys">>]).
-define(TAGS, [<<"API Keys">>]).
namespace() -> "api_key".

Some files were not shown because too many files have changed in this diff Show More